{ "benchmark": "JevBench v1", "cohorts": { "heldout-private": "24 decisions we wrote and keep private, so the suite cannot be trained on", "imported-public-source": "146 decisions imported from our auto-router job's labelled tasks (source text not redistributed)", "original-public": "72 decisions we wrote and publish in the repo" }, "dataset": { "frozen_at": "2026-09-18T21:20:30.887054+00:00", "hardware": { "concurrency": 1, "cpu": "AMD Ryzen 5 3600 6-Core Processor", "local_threads": 2, "origin": "Sandy Hetzner server", "platform": "Linux-5.15.0-143-generic-x86_64-with-glibc2.35" }, "labels_changed_after_inference": false, "pilot": true, "protocol": "jevbench::v1", "splits": [ { "canonical_sha256": "914276022fa5f0ea70e4231f2af98e68c67b274b61bea8c440b33890401cd2f9", "n": 72, "name": "original", "sha256": "5c2414edb3006b8bfcb70fda433f0f9ca015759433849f8d3104328a1f7c4180" }, { "canonical_sha256": "0648e6b956d579250a606579ab701481639b831dca125ddc84842841817191c7", "n": 24, "name": "heldout", "sha256": "8766a0cccb65b3857b1d13c1fee541bed8f3b82abb6d3ab6dced31161fadd1cf" }, { "canonical_sha256": "08df3667a5b7419a8a89023840835c059cb1f6c1e6414de9fa6d0569ce595098", "n": 78, "name": "router", "sha256": "d845fe36727df2d183352d87f18c6fd9279317c1a06fad90abfbbfe4518acf33" }, { "canonical_sha256": "2d3a951107be250172d9cd6830602f8bed409ed85be4876f92de12a4ada18ca7", "n": 68, "name": "judge", "sha256": "e0b9805689767ca8e5ec09d9c4cd9b5f6f633882d8ba91a329fe644c9a85e20b" } ] }, "generated_utc": "2026-09-19T07:07:26.728330+00:00", "legacy_appendix": { "results": [ { "accuracy_exact": 0.0, "accuracy_router_equivalence": 0.0, "basis": "measured", "calibration": null, "calibration_reason": "No full category distribution retained; confidence is not a category probability.", "failures": 45, "id": "needle-routing", "latency": { "n": 78, "p50_s": 4.205, "p95_s": 17.52549999999998 }, "n": 78, "not_jevbench_v1": true, "protocol": "legacy-auto-router-20260918", "sha256": "efd3b6cfddb25f39924d0cbf05c0c96bd0eddff4c23ad93af3d9ff1ac11a86fb", "source_mtime_utc": "2026-09-18T18:49:56.988661+00:00", "source_path": "/home/flori/jobs/needle3-vs-jev-20260918/runs/needle_classify.jsonl" }, { "accuracy_exact": 0.8461538461538461, "accuracy_router_equivalence": 1.0, "basis": "measured", "calibration": null, "calibration_reason": "No full category distribution retained; confidence is not a category probability.", "failures": 0, "id": "jev-routing", "latency": { "n": 78, "p50_s": 0.62, "p95_s": 0.7804999999999994 }, "n": 78, "not_jevbench_v1": true, "protocol": "legacy-auto-router-20260918", "sha256": "dccb2677b47efcbe1a9713d63ec430c01f688235baee508709be8da8ce87120a", "source_mtime_utc": "2026-09-17T14:43:19.236974+00:00", "source_path": "/home/flori/jobs/auto-model-router-20260917/runs/jev_classify.jsonl" }, { "basis": "measured", "comparison_limits": "Needle and Jev judge answers came from different generations: not paired. Legacy truncation, contradictory specs/tests and missing-document confounds remain; do not headline-rank.", "families": { "coding": { "binary_brier": null, "n": 36, "n_valid": 36, "n_wrong": 20 }, "long_context": { "binary_brier": null, "n": 24, "n_valid": 24, "n_wrong": 3 }, "math": { "binary_brier": null, "n": 36, "n_valid": 36, "n_wrong": 8 } }, "id": "needle-judge", "latency": { "n": 96, "p50_s": 3.465, "p95_s": 16.835 }, "n": 96, "not_jevbench_v1": true, "protocol": "legacy-auto-router-20260918", "sha256": "ee28443989dad1962f8c93415a718d6e67122d6fadc5f382c0e761ffba9d112b", "source_mtime_utc": "2026-09-18T19:19:24.622084+00:00", "source_path": "/home/flori/jobs/needle3-vs-jev-20260918/runs/needle_judge.jsonl" }, { "basis": "measured", "comparison_limits": "Needle and Jev judge answers came from different generations: not paired. Legacy truncation, contradictory specs/tests and missing-document confounds remain; do not headline-rank.", "families": { "coding": { "binary_brier": 0.11525694444444433, "n": 72, "n_valid": 72, "n_wrong": 33 }, "long_context": { "binary_brier": 0.7765229166666665, "n": 48, "n_valid": 48, "n_wrong": 2 }, "math": { "binary_brier": 0.0410875, "n": 72, "n_valid": 72, "n_wrong": 11 } }, "id": "jev-judge", "latency": { "n": 192, "p50_s": 0.69, "p95_s": 0.8544999999999998 }, "n": 192, "not_jevbench_v1": true, "protocol": "legacy-auto-router-20260918", "sha256": "8f73bf273647b5ccae7e01bd18b9b548b143b9aeb20979dd99b9e5ef49ea99a1", "source_mtime_utc": "2026-09-18T17:33:16.362551+00:00", "source_path": "/home/flori/jobs/auto-model-router-20260917/runs/verify-20260918/judged.jsonl" } ], "reuse_only": true }, "method_notes": [ "Every model answers the same state, the same rubric and the same exact label set.", "Native adapters read the model's own probability distribution; LLM baselines verbalize one under a JSON schema. The two are labelled and never mixed silently.", "Serial, one request at a time, no retries, no concurrency: latency includes the network from a Hetzner server in Germany.", "Accuracy is argmax. Ordinal questions also report expected-value MAE.", "Brier is the multi-class sum over the exact label set; ECE is top-label with 10 equal-width bins.", "Confidence intervals resample whole scenarios, because paraphrases of one scenario are not independent.", "A route with no billable account (a public demo, our own CPU, a flat-rate subscription) reports a null price, never a zero." ], "n_decisions_per_model": 242, "owner": "Benchmark Heaven (benchmarkheaven.com) - our own independent benchmark", "protocol": "jevbench::v1", "repeatability": { "accuracy_run_1": 0.9669421487603306, "accuracy_run_2": 0.9628099173553719, "disagreement_rate": 0.012396694214876033, "families_of_differences": [ "routing" ], "n_compared": 242, "n_different_prediction": 3, "note": "Two complete runs of the same 242 decisions, about 16 minutes apart, same request bodies, temperature not exposed by this API.", "system": "jev-1.13.0" }, "systems": [ { "adapter": "typesafe", "author": "TypeSafe AI", "by_cohort": { "heldout-private": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 24, "n_scenarios": 24, "point": 1.0 }, "brier_mean": 0.0028416666666666664, "calibration_n": 24, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.014679, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.89, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9839130434782607, "n": 23 } ], "ece": 0.020000000000000167, "n": 24 }, "first_request_s": 0.6774457022547722, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7488722428679466, "latency_ok_excl_first_s": { "n": 23, "p50_s": 0.6466918364167213, "p95_s": 0.694901406019926 }, "latency_ok_s": { "n": 24, "p50_s": 0.647936575114727, "p95_s": 0.6941410589963197 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 349.5, "mean_output_tokens": 33.041666666666664, "n_attempted": 24, "n_cost_known": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.9452054794520548, "accuracy_ci": { "hi95": 0.9794520547945206, "lo95": 0.9041095890410958, "n_decisions": 146, "n_scenarios": 146, "point": 0.9452054794520548 }, "brier_mean": 0.07813698630136988, "calibration_n": 146, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.03544541095890411, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.44, "n": 1 }, { "accuracy": 0.5454545454545454, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5445454545454546, "n": 11 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.61, "n": 1 }, { "accuracy": 0.8, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.764, "n": 5 }, { "accuracy": 0.9166666666666666, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8366666666666668, "n": 12 }, { "accuracy": 0.9913793103448276, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9783620689655173, "n": 116 } ], "ece": 0.02472602739726028, "n": 146 }, "first_request_s": 0.6713059619069099, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7632215991616249, "latency_ok_excl_first_s": { "n": 145, "p50_s": 0.6527933329343796, "p95_s": 0.7236708611249923 }, "latency_ok_s": { "n": 146, "p50_s": 0.6531931534409523, "p95_s": 0.7235849276185036 }, "majority_class_accuracy": 0.4178082191780822, "mean_input_tokens": 843.9383561643835, "mean_output_tokens": 54.986301369863014, "n_attempted": 146, "n_cost_known": 146, "n_planned": 146, "n_renormalized": 0, "n_scorable": 146, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "original-public": { "accuracy": 0.9861111111111112, "accuracy_ci": { "hi95": 1.0, "lo95": 0.9583333333333334, "n_decisions": 72, "n_scenarios": 36, "point": 0.9861111111111112 }, "brier_mean": 0.02744166666666666, "calibration_n": 72, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.01543616666666666, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.55, "n": 1 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.69, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.73, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8233333333333333, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9827692307692307, "n": 65 } ], "ece": 0.04625000000000004, "n": 72 }, "first_request_s": 0.6367422714829445, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.8036087676882744, "latency_ok_excl_first_s": { "n": 71, "p50_s": 0.6531622931361198, "p95_s": 0.7136913202702999 }, "latency_ok_s": { "n": 72, "p50_s": 0.652760449796915, "p95_s": 0.7136789243668318 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 367.52777777777777, "mean_output_tokens": 36.111111111111114, "n_attempted": 72, "n_cost_known": 72, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0025, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "by_family": { "adequacy": { "accuracy": 0.9523809523809523, "accuracy_ci": { "hi95": 0.9885057471264368, "lo95": 0.9036144578313253, "n_decisions": 84, "n_scenarios": 78, "point": 0.9523809523809523 }, "brier_mean": 0.07387857142857142, "calibration_n": 84, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.028807500000000007, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.57, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 0.8, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.758, "n": 5 }, { "accuracy": 0.9230769230769231, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8376923076923077, "n": 13 }, { "accuracy": 0.9846153846153847, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9609230769230768, "n": 65 } ], "ece": 0.0408333333333335, "n": 84 }, "first_request_s": 0.6725019738078117, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7632215991616249, "latency_ok_excl_first_s": { "n": 83, "p50_s": 0.6556308418512344, "p95_s": 0.7183704517781734 }, "latency_ok_s": { "n": 84, "p50_s": 0.6565632931888103, "p95_s": 0.7183260310441255 }, "majority_class_accuracy": 0.8214285714285714, "mean_input_tokens": 685.8928571428571, "mean_output_tokens": 20.0, "n_attempted": 84, "n_cost_known": 84, "n_planned": 84, "n_renormalized": 0, "n_scorable": 84, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 1.2500000000000012e-05, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.015416625000000003, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.999375, "n": 16 } ], "ece": 0.0006249999999999867, "n": 16 }, "first_request_s": 0.6491497829556465, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7462153136730194, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6536549925804138, "p95_s": 0.7093057468533516 }, "latency_ok_s": { "n": 16, "p50_s": 0.652135293930769, "p95_s": 0.7066693492233753 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 367.0625, "mean_output_tokens": 46.9375, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.00026250000000000025, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.015815625, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.993125, "n": 16 } ], "ece": 0.0068749999999999645, "n": 16 }, "first_request_s": 0.6666643172502518, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7488722428679466, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6654286459088326, "p95_s": 0.721465715020895 }, "latency_ok_s": { "n": 16, "p50_s": 0.6660464815795422, "p95_s": 0.7195081058889627 }, "majority_class_accuracy": 0.25, "mean_input_tokens": 376.5625, "mean_output_tokens": 51.125, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 3.750000000000004e-05, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.014946750000000002, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.998125, "n": 16 } ], "ece": 0.00187499999999996, "n": 16 }, "first_request_s": 0.6397505402565002, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7405179962515831, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6625630706548691, "p95_s": 0.721652551740408 }, "latency_ok_s": { "n": 16, "p50_s": 0.6574608385562897, "p95_s": 0.7203050199896097 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 355.875, "mean_output_tokens": 17.0, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.001875, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.9375, "accuracy_ci": { "hi95": 1.0, "lo95": 0.8125, "n_decisions": 16, "n_scenarios": 10, "point": 0.9375 }, "brier_mean": 0.1045125, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.014427, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.55, "n": 1 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.69, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.87, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9383333333333334, "n": 12 } ], "ece": 0.14437499999999998, "n": 16 }, "first_request_s": 0.6367422714829445, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.8036087676882744, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6366338729858398, "p95_s": 0.7407533258199691 }, "latency_ok_s": { "n": 16, "p50_s": 0.6366880722343922, "p95_s": 0.7362636514008045 }, "majority_class_accuracy": 0.5, "mean_input_tokens": 343.5, "mean_output_tokens": 20.0, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 0.9574468085106383, "accuracy_ci": { "hi95": 0.9896907216494846, "lo95": 0.9130434782608695, "n_decisions": 94, "n_scenarios": 88, "point": 0.9574468085106383 }, "brier_mean": 0.05924468085106384, "calibration_n": 94, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.034565999999999986, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.44, "n": 1 }, { "accuracy": 0.6, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.542, "n": 10 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.61, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.79, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8200000000000001, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9927848101265824, "n": 79 } ], "ece": 0.02840425531914885, "n": 94 }, "first_request_s": 0.6110476851463318, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7569568827748299, "latency_ok_excl_first_s": { "n": 93, "p50_s": 0.6497085839509964, "p95_s": 0.7222160667181015 }, "latency_ok_s": { "n": 94, "p50_s": 0.6490628123283386, "p95_s": 0.7219626244157552 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": 823.0, "mean_output_tokens": 80.63829787234043, "n_attempted": 94, "n_cost_known": 94, "n_planned": 94, "n_renormalized": 0, "n_scorable": 94, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "class": "jev", "complete": true, "display": "Jev 1.13.0 (TypeSafe AI)", "key": "jev-1.13.0", "licence": "proprietary API", "open": "no", "overall": { "accuracy": 0.9628099173553719, "accuracy_ci": { "hi95": 0.984313725490196, "lo95": 0.9367088607594937, "n_decisions": 242, "n_scenarios": 206, "point": 0.9628099173553719 }, "brier_mean": 0.05558677685950413, "calibration_n": 242, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.027432768595041332, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.44, "n": 1 }, { "accuracy": 0.5833333333333334, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.545, "n": 12 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6499999999999999, "n": 2 }, { "accuracy": 0.8571428571428571, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7542857142857143, "n": 7 }, { "accuracy": 0.9375, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8375000000000001, "n": 16 }, { "accuracy": 0.9950980392156863, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9803921568627444, "n": 204 } ], "ece": 0.027438016528926218, "n": 242 }, "first_request_s": 0.6367422714829445, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.8036087676882744, "latency_ok_excl_first_s": { "n": 241, "p50_s": 0.6524642184376717, "p95_s": 0.7222959250211716 }, "latency_ok_s": { "n": 242, "p50_s": 0.6524335257709026, "p95_s": 0.7221905551850795 }, "majority_class_accuracy": 0.3181818181818182, "mean_input_tokens": 653.1611570247934, "mean_output_tokens": 47.19421487603306, "n_attempted": 242, "n_cost_known": 242, "n_planned": 242, "n_renormalized": 0, "n_scorable": 242, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.001875, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "paraphrase_consistency": { "both_correct_rate": 0.9722222222222222, "both_valid": 36, "pairs": 36, "same_answer_rate": 0.9722222222222222 }, "price_input_per_m": 0.042, "price_output_per_m": 0.0, "price_source": "https://docs.typesafe.ai/models (output tokens not billed)", "probability_source": [ "native" ], "ranked": true, "repo": "https://docs.typesafe.ai", "request_options": {}, "resolved_model_identities": [ "jev-1.13.0" ], "run_finished_utc": "2026-09-19T05:58:46.517886+00:00", "run_started_utc": "2026-09-19T05:55:22.420840+00:00", "stop_reason": null, "underlying": "closed" }, { "adapter": "typesafe", "author": "ekzhang", "by_cohort": { "heldout-private": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 24, "n_scenarios": 24, "point": 1.0 }, "brier_mean": 0.030465693961602647, "calibration_n": 24, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6224593031872285, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7310585610515279, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8675567210710075, "n": 5 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9907082162896014, "n": 16 } ], "ece": 0.07192952119671069, "n": 24 }, "first_request_s": 0.6517126709222794, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7090421691536903, "latency_ok_excl_first_s": { "n": 23, "p50_s": 0.6392186358571053, "p95_s": 0.6978873796761036 }, "latency_ok_s": { "n": 24, "p50_s": 0.6454656533896923, "p95_s": 0.6977766174823046 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 169.25, "mean_output_tokens": 2.0, "n_attempted": 24, "n_cost_known": 0, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.03894879187433612, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.952054794520548, "accuracy_ci": { "hi95": 0.9863013698630136, "lo95": 0.910958904109589, "n_decisions": 146, "n_scenarios": 146, "point": 0.952054794520548 }, "brier_mean": 0.09427087008094001, "calibration_n": 146, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5310882431075317, "n": 2 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6554254860479014, "n": 5 }, { "accuracy": 0.8666666666666667, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7517591925117498, "n": 15 }, { "accuracy": 0.9230769230769231, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8529188136544228, "n": 26 }, { "accuracy": 0.9795918367346939, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9759032333781944, "n": 98 } ], "ece": 0.03900174754946071, "n": 146 }, "first_request_s": 0.6167056486010551, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.851484477519989, "latency_ok_excl_first_s": { "n": 145, "p50_s": 0.6492418646812439, "p95_s": 0.7144190564751625 }, "latency_ok_s": { "n": 146, "p50_s": 0.6490314826369286, "p95_s": 0.7141148019582033 }, "majority_class_accuracy": 0.4178082191780822, "mean_input_tokens": 1108.6986301369864, "mean_output_tokens": 2.0, "n_attempted": 146, "n_cost_known": 0, "n_planned": 146, "n_renormalized": 0, "n_scorable": 146, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "original-public": { "accuracy": 0.9444444444444444, "accuracy_ci": { "hi95": 0.9861111111111112, "lo95": 0.8888888888888888, "n_decisions": 72, "n_scenarios": 36, "point": 0.9444444444444444 }, "brier_mean": 0.0854948785086843, "calibration_n": 72, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.75, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5389511552517667, "n": 4 }, { "accuracy": 0.3333333333333333, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6509976225000895, "n": 3 }, { "accuracy": 0.8, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7557263147670344, "n": 5 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8490495292202918, "n": 10 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9766282676431166, "n": 50 } ], "ece": 0.06523122219860114, "n": 72 }, "first_request_s": 0.851369246840477, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0092222914099693, "latency_ok_excl_first_s": { "n": 71, "p50_s": 0.685624286532402, "p95_s": 0.7656357437372208 }, "latency_ok_s": { "n": 72, "p50_s": 0.6858020424842834, "p95_s": 0.7821974605321884 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 185.30555555555554, "mean_output_tokens": 2.0, "n_attempted": 72, "n_cost_known": 0, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.03829121478834396, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "by_family": { "adequacy": { "accuracy": 0.8928571428571429, "accuracy_ci": { "hi95": 0.9523809523809523, "lo95": 0.8192771084337349, "n_decisions": 84, "n_scenarios": 78, "point": 0.8928571428571429 }, "brier_mean": 0.16597734217910332, "calibration_n": 84, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.6, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5373543497382958, "n": 5 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6652667686590864, "n": 2 }, { "accuracy": 0.8, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7485360520891449, "n": 10 }, { "accuracy": 0.8, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8654331077540751, "n": 10 }, { "accuracy": 0.9649122807017544, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9770161189101827, "n": 57 } ], "ece": 0.029793465751862022, "n": 84 }, "first_request_s": 0.6882655248045921, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.851484477519989, "latency_ok_excl_first_s": { "n": 83, "p50_s": 0.6767364293336868, "p95_s": 0.7168036870658397 }, "latency_ok_s": { "n": 84, "p50_s": 0.6768096536397934, "p95_s": 0.7166170362383127 }, "majority_class_accuracy": 0.8214285714285714, "mean_input_tokens": 928.952380952381, "mean_output_tokens": 2.0, "n_attempted": 84, "n_cost_known": 0, "n_planned": 84, "n_renormalized": 0, "n_scorable": 84, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0041436536722160595, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8185299935881283, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.982323318964479, "n": 15 } ], "ece": 0.027913763871542922, "n": 16 }, "first_request_s": 0.6127750724554062, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.701769195497036, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6784608215093613, "p95_s": 0.6968614041805268 }, "latency_ok_s": { "n": 16, "p50_s": 0.6588397286832333, "p95_s": 0.6965108476579189 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 169.5, "mean_output_tokens": 2.0, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.021972654414114175, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7476894558361015, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8620709929338618, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9850724390976314, "n": 11 } ], "ece": 0.06766320496576662, "n": 16 }, "first_request_s": 0.6879012137651443, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7262816950678825, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6856115162372589, "p95_s": 0.7176281049847603 }, "latency_ok_s": { "n": 16, "p50_s": 0.6867563650012016, "p95_s": 0.7170099914073944 }, "majority_class_accuracy": 0.25, "mean_input_tokens": 173.0, "mean_output_tokens": 2.0, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.005303565220641863, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8562452096337188, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9869977822356903, "n": 14 } ], "ece": 0.029346289339556103, "n": 16 }, "first_request_s": 0.7207812294363976, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7207812294363976, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6476355567574501, "p95_s": 0.6938971303403377 }, "latency_ok_s": { "n": 16, "p50_s": 0.6490505672991276, "p95_s": 0.7038774620741606 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 187.375, "mean_output_tokens": 2.0, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.038455609059841994, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.875, "accuracy_ci": { "hi95": 1.0, "lo95": 0.7222222222222222, "n_decisions": 16, "n_scenarios": 10, "point": 0.875 }, "brier_mean": 0.21029758331603057, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5312093585306514, "n": 1 }, { "accuracy": 0.6666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6413657731857211, "n": 3 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7541792026741546, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8354835493924953, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9738223146180603, "n": 9 } ], "ece": 0.090822959144091, "n": 16 }, "first_request_s": 0.851369246840477, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0092222914099693, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6859797984361649, "p95_s": 0.8778621733188627 }, "latency_ok_s": { "n": 16, "p50_s": 0.6876388341188431, "p95_s": 0.8908325079828501 }, "majority_class_accuracy": 0.5, "mean_input_tokens": 191.875, "mean_output_tokens": 2.0, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 94, "n_scenarios": 88, "point": 1.0 }, "brier_mean": 0.030221052026683553, "calibration_n": 94, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.649487186012917, "n": 4 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7535048432128142, "n": 8 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8498800014159397, "n": 24 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9737642506649791, "n": 58 } ], "ece": 0.09041027593291999, "n": 94 }, "first_request_s": 0.6905875578522682, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7871266081929207, "latency_ok_excl_first_s": { "n": 93, "p50_s": 0.6546039655804634, "p95_s": 0.7171858698129654 }, "latency_ok_s": { "n": 94, "p50_s": 0.6560658626258373, "p95_s": 0.7169921454042196 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": 954.1914893617021, "mean_output_tokens": 2.0, "n_attempted": 94, "n_cost_known": 0, "n_planned": 94, "n_renormalized": 0, "n_scorable": 94, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "class": "jev-rebuild", "complete": true, "display": "openjev-sglang (Qwen3.6-35B-A3B on SGLang)", "key": "openjev-sglang", "licence": "no licence file in the repository as of 2026-09-19; Qwen3.6 weights keep their own terms", "open": "yes", "overall": { "accuracy": 0.9545454545454546, "accuracy_ci": { "hi95": 0.978448275862069, "lo95": 0.9282868525896414, "n_decisions": 242, "n_scenarios": 206, "point": 0.9545454545454546 }, "brier_mean": 0.0853320452046322, "calibration_n": 242, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.6666666666666666, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5363301845370217, "n": 6 }, { "accuracy": 0.7777777777777778, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6502866223252226, "n": 9 }, { "accuracy": 0.8636363636363636, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7507789356188397, "n": 22 }, { "accuracy": 0.9512195121951219, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8537601964042186, "n": 41 }, { "accuracy": 0.9878048780487805, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.977568668986906, "n": 164 } ], "ece": 0.0416812931479787, "n": 242 }, "first_request_s": 0.851369246840477, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0092222914099693, "latency_ok_excl_first_s": { "n": 241, "p50_s": 0.6768828779459, "p95_s": 0.7207812294363976 }, "latency_ok_s": { "n": 242, "p50_s": 0.6776718497276306, "p95_s": 0.7260066717863083 }, "majority_class_accuracy": 0.3181818181818182, "mean_input_tokens": 740.801652892562, "mean_output_tokens": 2.0, "n_attempted": 242, "n_cost_known": 0, "n_planned": 242, "n_renormalized": 0, "n_scorable": 242, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.038455609059841994, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "paraphrase_consistency": { "both_correct_rate": 0.8888888888888888, "both_valid": 36, "pairs": 36, "same_answer_rate": 0.8888888888888888 }, "price_input_per_m": null, "price_output_per_m": null, "price_source": "author's own public Modal deployment; we hold no billable account", "probability_source": [ "native" ], "ranked": true, "repo": "https://github.com/ekzhang/openjev-sglang", "request_options": {}, "resolved_model_identities": [ "jev-latest" ], "run_finished_utc": "2026-09-19T06:02:37.968792+00:00", "run_started_utc": "2026-09-19T05:58:46.631025+00:00", "stop_reason": null, "underlying": "Qwen3.6-35B-A3B" }, { "adapter": "systemone_list", "author": "mithalouni", "by_cohort": { "heldout-private": { "accuracy": 0.9583333333333334, "accuracy_ci": { "hi95": 1.0, "lo95": 0.875, "n_decisions": 24, "n_scenarios": 24, "point": 0.9583333333333334 }, "brier_mean": 0.08513520728329144, "calibration_n": 24, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5472703576087952, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7582655549049377, "n": 2 }, { "accuracy": 0.75, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8536906652152538, "n": 4 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9750713291851913, "n": 17 } ], "ece": 0.07394785822058715, "n": 24 }, "first_request_s": 0.7457669973373413, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7457669973373413, "latency_ok_excl_first_s": { "n": 23, "p50_s": 0.6442036405205727, "p95_s": 0.6626570403575898 }, "latency_ok_s": { "n": 24, "p50_s": 0.6442981995642185, "p95_s": 0.7091682694852351 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 24, "n_cost_known": 0, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.09122845447564032, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.8767123287671232, "accuracy_ci": { "hi95": 0.9246575342465754, "lo95": 0.821917808219178, "n_decisions": 146, "n_scenarios": 146, "point": 0.8767123287671232 }, "brier_mean": 0.16311421497942305, "calibration_n": 146, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3694673180580139, "n": 1 }, { "accuracy": 0.3333333333333333, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.47420785824457806, "n": 3 }, { "accuracy": 0.6666666666666666, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5378236174583435, "n": 15 }, { "accuracy": 0.5263157894736842, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6462128632947018, "n": 19 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7471469101451692, "n": 21 }, { "accuracy": 0.9333333333333333, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8594711105028788, "n": 15 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9629874360851116, "n": 72 } ], "ece": 0.0964762593479189, "n": 146 }, "first_request_s": 0.6564928069710732, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0980295613408089, "latency_ok_excl_first_s": { "n": 145, "p50_s": 0.661714680492878, "p95_s": 0.7769349798560142 }, "latency_ok_s": { "n": 146, "p50_s": 0.6606850996613503, "p95_s": 0.7768471296876669 }, "majority_class_accuracy": 0.4178082191780822, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 146, "n_cost_known": 0, "n_planned": 146, "n_renormalized": 0, "n_scorable": 146, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "original-public": { "accuracy": 0.9305555555555556, "accuracy_ci": { "hi95": 0.9861111111111112, "lo95": 0.875, "n_decisions": 72, "n_scenarios": 36, "point": 0.9305555555555556 }, "brier_mean": 0.10523597297761339, "calibration_n": 72, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.451519250869751, "n": 1 }, { "accuracy": 0.4, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5532657146453858, "n": 5 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6562116146087646, "n": 2 }, { "accuracy": 0.875, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7622018810361624, "n": 8 }, { "accuracy": 0.9333333333333333, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8537419974803925, "n": 15 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9736400029853429, "n": 41 } ], "ece": 0.07193612444785281, "n": 72 }, "first_request_s": 29.40474732220173, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 29.40474732220173, "latency_ok_excl_first_s": { "n": 71, "p50_s": 0.6478558778762817, "p95_s": 0.741743665188551 }, "latency_ok_s": { "n": 72, "p50_s": 0.6482186019420624, "p95_s": 0.747379744425416 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 72, "n_cost_known": 0, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0689091139380859, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "by_family": { "adequacy": { "accuracy": 0.8809523809523809, "accuracy_ci": { "hi95": 0.9425287356321839, "lo95": 0.8072289156626506, "n_decisions": 84, "n_scenarios": 78, "point": 0.8809523809523809 }, "brier_mean": 0.18383072888461086, "calibration_n": 84, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.6666666666666666, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5399758766094843, "n": 12 }, { "accuracy": 0.45454545454545453, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.64674250646071, "n": 11 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7549974411725998, "n": 20 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8588520292388765, "n": 19 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9427740800363774, "n": 22 } ], "ece": 0.14851535609479816, "n": 84 }, "first_request_s": 0.6276316866278648, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0980295613408089, "latency_ok_excl_first_s": { "n": 83, "p50_s": 0.6492264717817307, "p95_s": 0.7529947854578496 }, "latency_ok_s": { "n": 84, "p50_s": 0.6490224711596966, "p95_s": 0.7529928121715784 }, "majority_class_accuracy": 0.8214285714285714, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 84, "n_cost_known": 0, "n_planned": 84, "n_renormalized": 0, "n_scorable": 84, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 0.9375, "accuracy_ci": { "hi95": 1.0, "lo95": 0.8125, "n_decisions": 16, "n_scenarios": 10, "point": 0.9375 }, "brier_mean": 0.045870330243962885, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5846080183982849, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9857980648676554, "n": 15 } ], "ece": 0.04985231533646585, "n": 16 }, "first_request_s": 0.6447048559784889, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.6548216566443443, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6372598186135292, "p95_s": 0.6522322542965412 }, "latency_ok_s": { "n": 16, "p50_s": 0.6388817206025124, "p95_s": 0.6520472969859838 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 0.9375, "accuracy_ci": { "hi95": 1.0, "lo95": 0.8125, "n_decisions": 16, "n_scenarios": 10, "point": 0.9375 }, "brier_mean": 0.1405330203724347, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.451519250869751, "n": 1 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.582737147808075, "n": 1 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6916282773017883, "n": 1 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7300671935081482, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8456628024578094, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9853126704692841, "n": 10 } ], "ece": 0.15373313799500465, "n": 16 }, "first_request_s": 0.7468994110822678, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.7468994110822678, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6514480710029602, "p95_s": 0.7198584035038949 }, "latency_ok_s": { "n": 16, "p50_s": 0.6517680287361145, "p95_s": 0.7310176994651556 }, "majority_class_accuracy": 0.25, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.017519824326784308, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7749395668506622, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8616498907407125, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9729133627631448, "n": 11 } ], "ece": 0.07269526273012158, "n": 16 }, "first_request_s": 0.648581326007843, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.747966818511486, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6448771506547928, "p95_s": 0.6999030202627181 }, "latency_ok_s": { "n": 16, "p50_s": 0.6467292383313179, "p95_s": 0.6964698918163776 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0744889490724745, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.8125, "accuracy_ci": { "hi95": 1.0, "lo95": 0.625, "n_decisions": 16, "n_scenarios": 10, "point": 0.8125 }, "brier_mean": 0.2838520999584748, "calibration_n": 16, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.6666666666666666, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.542025069395701, "n": 3 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.777439092596372, "n": 3 }, { "accuracy": 0.5, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8357955981045961, "n": 4 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9743625699726787, "n": 6 } ], "ece": 0.15866340541288082, "n": 16 }, "first_request_s": 29.40474732220173, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 29.40474732220173, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6723669469356537, "p95_s": 0.7489164359867573 }, "latency_ok_s": { "n": 16, "p50_s": 0.6805610097944736, "p95_s": 7.918385675176978 }, "majority_class_accuracy": 0.5, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 0.9042553191489362, "accuracy_ci": { "hi95": 0.9578947368421052, "lo95": 0.8444444444444444, "n_decisions": 94, "n_scenarios": 88, "point": 0.9042553191489362 }, "brier_mean": 0.10839068927073518, "calibration_n": 94, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3694673180580139, "n": 1 }, { "accuracy": 0.3333333333333333, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.47420785824457806, "n": 3 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5269555747509003, "n": 4 }, { "accuracy": 0.6666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6427413092719184, "n": 9 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7184037685394287, "n": 5 }, { "accuracy": 0.8333333333333334, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8625521759192148, "n": 6 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9681999737566168, "n": 66 } ], "ece": 0.051035503757760836, "n": 94 }, "first_request_s": 0.6638940051198006, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.8372753858566284, "latency_ok_excl_first_s": { "n": 93, "p50_s": 0.6596555188298225, "p95_s": 0.7769559279084206 }, "latency_ok_s": { "n": 94, "p50_s": 0.6607971265912056, "p95_s": 0.776777609065175 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 94, "n_cost_known": 0, "n_planned": 94, "n_renormalized": 0, "n_scorable": 94, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "class": "jev-rebuild", "complete": true, "display": "system-one-open (Gemma 4 E2B LoRA on an L4)", "key": "system-one-open", "licence": "MIT (repository LICENSE; Gemma weights keep Google\u2019s terms)", "open": "yes", "overall": { "accuracy": 0.9008264462809917, "accuracy_ci": { "hi95": 0.9365079365079365, "lo95": 0.8630705394190872, "n_decisions": 242, "n_scenarios": 206, "point": 0.9008264462809917 }, "brier_mean": 0.13816078684373112, "calibration_n": 242, "cost_basis": [ "no_billable_account_public_endpoint" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3694673180580139, "n": 1 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4685357064008713, "n": 4 }, { "accuracy": 0.6190476190476191, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5419501520338512, "n": 21 }, { "accuracy": 0.5714285714285714, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6471651253246126, "n": 21 }, { "accuracy": 0.967741935483871, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7517493958434751, "n": 31 }, { "accuracy": 0.9117647058823529, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8562635082532378, "n": 34 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.967927293205195, "n": 130 } ], "ece": 0.06800454654184858, "n": 242 }, "first_request_s": 29.40474732220173, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 29.40474732220173, "latency_ok_excl_first_s": { "n": 241, "p50_s": 0.6515424996614456, "p95_s": 0.7644615098834038 }, "latency_ok_s": { "n": 242, "p50_s": 0.6517308317124844, "p95_s": 0.7724301926791667 }, "majority_class_accuracy": 0.3181818181818182, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 242, "n_cost_known": 0, "n_planned": 242, "n_renormalized": 0, "n_scorable": 242, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0744889490724745, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "paraphrase_consistency": { "both_correct_rate": 0.8611111111111112, "both_valid": 36, "pairs": 36, "same_answer_rate": 0.8611111111111112 }, "price_input_per_m": null, "price_output_per_m": null, "price_source": "author's own public Modal deployment; we hold no billable account", "probability_source": [ "native" ], "ranked": true, "repo": "https://github.com/mithalouni/system-one-open", "request_options": {}, "resolved_model_identities": [ "e2b-full" ], "run_finished_utc": "2026-09-19T06:07:00.564116+00:00", "run_started_utc": "2026-09-19T06:02:38.089539+00:00", "stop_reason": null, "underlying": "google/gemma-4-E2B-it + LoRA" }, { "adapter": "gradio_space", "author": "IkerMoel", "by_cohort": { "heldout-private": { "accuracy": null, "n_attempted": 0, "n_planned": 24 }, "imported-public-source": { "accuracy": null, "n_attempted": 0, "n_planned": 146 }, "original-public": { "accuracy": 0.3333333333333333, "accuracy_ci": { "hi95": 1.0, "lo95": 0.0, "n_decisions": 6, "n_scenarios": 3, "point": 0.3333333333333333 }, "brier_mean": 1.0009116303189316, "calibration_n": 5, "cost_basis": [ "no_billable_account_public_demo" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7310585786300049, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9949403848203854, "n": 4 } ], "ece": 0.5421640235823093, "n": 5 }, "first_request_s": 2.0179919451475143, "latency_failed_s": { "n": 1, "p50_s": 0.9080749228596687, "p95_s": 0.9080749228596687 }, "latency_max_s": 2.0179919451475143, "latency_ok_excl_first_s": { "n": 4, "p50_s": 0.8497459217905998, "p95_s": 0.8901171565055848 }, "latency_ok_s": { "n": 5, "p50_s": 0.8515224754810333, "p95_s": 1.7937791526317595 }, "majority_class_accuracy": 0.6666666666666666, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 6, "n_cost_known": 0, "n_planned": 72, "n_renormalized": 0, "n_scorable": 6, "n_state_truncated": 0, "operational_success": 0.8333333333333334, "ordinal_mae": null, "schema_validity": 0.8333333333333334, "schema_validity_strict": 0.8333333333333334 } }, "by_family": { "adequacy": { "accuracy": null, "n_attempted": 0, "n_planned": 84 }, "extraction": { "accuracy": null, "n_attempted": 0, "n_planned": 16 }, "intent": { "accuracy": null, "n_attempted": 0, "n_planned": 16 }, "ordinal": { "accuracy": null, "n_attempted": 0, "n_planned": 16 }, "policy": { "accuracy": 0.3333333333333333, "accuracy_ci": { "hi95": 1.0, "lo95": 0.0, "n_decisions": 6, "n_scenarios": 3, "point": 0.3333333333333333 }, "brier_mean": 1.0009116303189316, "calibration_n": 5, "cost_basis": [ "no_billable_account_public_demo" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7310585786300049, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9949403848203854, "n": 4 } ], "ece": 0.5421640235823093, "n": 5 }, "first_request_s": 2.0179919451475143, "latency_failed_s": { "n": 1, "p50_s": 0.9080749228596687, "p95_s": 0.9080749228596687 }, "latency_max_s": 2.0179919451475143, "latency_ok_excl_first_s": { "n": 4, "p50_s": 0.8497459217905998, "p95_s": 0.8901171565055848 }, "latency_ok_s": { "n": 5, "p50_s": 0.8515224754810333, "p95_s": 1.7937791526317595 }, "majority_class_accuracy": 0.6666666666666666, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 6, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 6, "n_state_truncated": 0, "operational_success": 0.8333333333333334, "ordinal_mae": null, "schema_validity": 0.8333333333333334, "schema_validity_strict": 0.8333333333333334 }, "routing": { "accuracy": null, "n_attempted": 0, "n_planned": 94 } }, "class": "jev-rebuild", "complete": false, "display": "open-alternative-jev (Qwen3.5-4B, HF Space)", "key": "open-alternative-jev", "licence": "Apache-2.0 (repository); Qwen3.5 weights keep their own terms", "open": "yes", "overall": { "accuracy": 0.3333333333333333, "accuracy_ci": { "hi95": 1.0, "lo95": 0.0, "n_decisions": 6, "n_scenarios": 3, "point": 0.3333333333333333 }, "brier_mean": 1.0009116303189316, "calibration_n": 5, "cost_basis": [ "no_billable_account_public_demo" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7310585786300049, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9949403848203854, "n": 4 } ], "ece": 0.5421640235823093, "n": 5 }, "first_request_s": 2.0179919451475143, "latency_failed_s": { "n": 1, "p50_s": 0.9080749228596687, "p95_s": 0.9080749228596687 }, "latency_max_s": 2.0179919451475143, "latency_ok_excl_first_s": { "n": 4, "p50_s": 0.8497459217905998, "p95_s": 0.8901171565055848 }, "latency_ok_s": { "n": 5, "p50_s": 0.8515224754810333, "p95_s": 1.7937791526317595 }, "majority_class_accuracy": 0.6666666666666666, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 6, "n_cost_known": 0, "n_planned": 242, "n_renormalized": 0, "n_scorable": 6, "n_state_truncated": 0, "operational_success": 0.8333333333333334, "ordinal_mae": null, "schema_validity": 0.8333333333333334, "schema_validity_strict": 0.8333333333333334 }, "paraphrase_consistency": { "both_correct_rate": 0.027777777777777776, "both_valid": 2, "pairs": 36, "same_answer_rate": 1.0 }, "price_input_per_m": null, "price_output_per_m": null, "price_source": "author's free Hugging Face Space; we hold no billable account", "probability_source": [ "native" ], "ranked": false, "repo": "https://github.com/ikermoel/open-alternative-jev", "request_options": {}, "resolved_model_identities": [ "IkerMoel/open-alternative-jev (Qwen3.5-4B)" ], "run_finished_utc": "2026-09-19T07:04:02.377023+00:00", "run_started_utc": "2026-09-19T07:03:52.711936+00:00", "stop_reason": "AppError: You have exceeded your ZeroGPU runs limit. Subscribe to Hugging Face PRO to get 40 min of ZeroGPU quota a day - https://huggingface.co/subscribe/pro?from=ZeroGPU", "underlying": "Qwen/Qwen3.5-4B" }, { "adapter": "local_openjev", "author": "Kotoba Labs", "by_cohort": { "heldout-private": { "accuracy": 0.6666666666666666, "accuracy_ci": { "hi95": 0.8333333333333334, "lo95": 0.4583333333333333, "n_decisions": 24, "n_scenarios": 24, "point": 0.6666666666666666 }, "brier_mean": 0.4769306132940074, "calibration_n": 24, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.25, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.35259532183408737, "n": 4 }, { "accuracy": 0.6666666666666666, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.44052382310231525, "n": 3 }, { "accuracy": 0.75, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5396048165857792, "n": 8 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6714436858892441, "n": 2 }, { "accuracy": 0.6, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7680855721235276, "n": 5 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.873363733291626, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9672161936759949, "n": 1 } ], "ece": 0.18453882696727913, "n": 24 }, "first_request_s": 1.0502445846796036, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0502445846796036, "latency_ok_excl_first_s": { "n": 23, "p50_s": 0.8906097114086151, "p95_s": 0.9767319083213806 }, "latency_ok_s": { "n": 24, "p50_s": 0.8954557552933693, "p95_s": 1.0167366128414868 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 24, "n_cost_known": 0, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.6549376994371414, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.5342465753424658, "accuracy_ci": { "hi95": 0.6164383561643836, "lo95": 0.4589041095890411, "n_decisions": 146, "n_scenarios": 146, "point": 0.5342465753424658 }, "brier_mean": 0.7022386001356168, "calibration_n": 146, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3776950389146805, "n": 2 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4665033183991909, "n": 24 }, { "accuracy": 0.46808510638297873, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5661645595063555, "n": 47 }, { "accuracy": 0.8461538461538461, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6442159521274078, "n": 39 }, { "accuracy": 0.5454545454545454, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7588141885670748, "n": 11 }, { "accuracy": 0.7142857142857143, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8566364049911499, "n": 21 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9011107087135315, "n": 2 } ], "ece": 0.2052800071566072, "n": 146 }, "first_request_s": 3.487430118024349, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 3.590568356215954, "latency_ok_excl_first_s": { "n": 145, "p50_s": 2.0872722193598747, "p95_s": 3.3964938983321193 }, "latency_ok_s": { "n": 146, "p50_s": 2.0961069390177727, "p95_s": 3.414716958999634 }, "majority_class_accuracy": 0.4178082191780822, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 146, "n_cost_known": 0, "n_planned": 146, "n_renormalized": 0, "n_scorable": 146, "n_state_truncated": 80, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "original-public": { "accuracy": 0.4305555555555556, "accuracy_ci": { "hi95": 0.5833333333333334, "lo95": 0.2916666666666667, "n_decisions": 72, "n_scenarios": 36, "point": 0.4305555555555556 }, "brier_mean": 0.6058420869523523, "calibration_n": 72, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.2857142857142857, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3694280811718532, "n": 7 }, { "accuracy": 0.18181818181818182, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.43857687982645904, "n": 11 }, { "accuracy": 0.375, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5394797921180725, "n": 24 }, { "accuracy": 0.4666666666666667, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.644039922952652, "n": 15 }, { "accuracy": 0.42857142857142855, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7525223983185632, "n": 7 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8556444197893143, "n": 6 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.918424591422081, "n": 2 } ], "ece": 0.18493607205649218, "n": 72 }, "first_request_s": 5.622470483183861, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 5.622470483183861, "latency_ok_excl_first_s": { "n": 71, "p50_s": 1.0322956442832947, "p95_s": 1.3372715301811695 }, "latency_ok_s": { "n": 72, "p50_s": 1.0368935875594616, "p95_s": 1.343157370015979 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 72, "n_cost_known": 0, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.8699293521543344, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "by_family": { "adequacy": { "accuracy": 0.7738095238095238, "accuracy_ci": { "hi95": 0.8734177215189873, "lo95": 0.6629213483146067, "n_decisions": 84, "n_scenarios": 78, "point": 0.7738095238095238 }, "brier_mean": 0.3915898500402369, "calibration_n": 84, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.7352941176470589, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5630775646251791, "n": 34 }, { "accuracy": 0.8095238095238095, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.643436209786506, "n": 42 }, { "accuracy": 0.7142857142857143, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7429683357477188, "n": 7 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9365394338965416, "n": 1 } ], "ece": 0.1558962012862875, "n": 84 }, "first_request_s": 0.8412344828248024, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 2.65216051787138, "latency_ok_excl_first_s": { "n": 83, "p50_s": 1.910632073879242, "p95_s": 2.3201519511640067 }, "latency_ok_s": { "n": 84, "p50_s": 1.8994696103036404, "p95_s": 2.3188297394663095 }, "majority_class_accuracy": 0.8214285714285714, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 84, "n_cost_known": 0, "n_planned": 84, "n_renormalized": 0, "n_scorable": 84, "n_state_truncated": 41, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 0.6875, "accuracy_ci": { "hi95": 1.0, "lo95": 0.4117647058823529, "n_decisions": 16, "n_scenarios": 10, "point": 0.6875 }, "brier_mean": 0.4700406147954016, "calibration_n": 16, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.34768742322921753, "n": 4 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.43722112476825714, "n": 2 }, { "accuracy": 0.4, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5479718565940856, "n": 5 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.647879958152771, "n": 4 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7839838862419128, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.25619772635400295, "n": 16 }, "first_request_s": 0.9637938514351845, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0110932439565659, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.9406859278678894, "p95_s": 1.0032816290855409 }, "latency_ok_s": { "n": 16, "p50_s": 0.9471648931503296, "p95_s": 1.0027236565947533 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 0.5625, "accuracy_ci": { "hi95": 0.8461538461538461, "lo95": 0.29411764705882354, "n_decisions": 16, "n_scenarios": 10, "point": 0.5625 }, "brier_mean": 0.501143138107421, "calibration_n": 16, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4220384359359741, "n": 2 }, { "accuracy": 0.25, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5255116671323776, "n": 4 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6444200873374939, "n": 1 }, { "accuracy": 0.3333333333333333, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7402561704317728, "n": 3 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.860925042629242, "n": 5 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9672161936759949, "n": 1 } ], "ece": 0.26566440984606743, "n": 16 }, "first_request_s": 1.0414915308356285, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.1967656314373016, "latency_ok_excl_first_s": { "n": 15, "p50_s": 1.0467617586255074, "p95_s": 1.1327185213565825 }, "latency_ok_s": { "n": 16, "p50_s": 1.044126644730568, "p95_s": 1.1281437277793884 }, "majority_class_accuracy": 0.25, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 0.5, "accuracy_ci": { "hi95": 0.8, "lo95": 0.2222222222222222, "n_decisions": 16, "n_scenarios": 10, "point": 0.5 }, "brier_mean": 0.627201079802534, "calibration_n": 16, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.3333333333333333, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3723172942797343, "n": 3 }, { "accuracy": 0.2857142857142857, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4440807104110718, "n": 7 }, { "accuracy": 0.8333333333333334, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5426406363646189, "n": 6 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.185604564845562, "n": 16 }, "first_request_s": 1.0192979872226715, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.190320409834385, "latency_ok_excl_first_s": { "n": 15, "p50_s": 1.0681502372026443, "p95_s": 1.1730399534106255 }, "latency_ok_s": { "n": 16, "p50_s": 1.0573878064751625, "p95_s": 1.1718056350946426 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.8161814389750361, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.5, "accuracy_ci": { "hi95": 0.8125, "lo95": 0.1875, "n_decisions": 16, "n_scenarios": 10, "point": 0.5 }, "brier_mean": 0.5148655984713457, "calibration_n": 16, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.3333333333333333, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5657494266827902, "n": 3 }, { "accuracy": 0.3333333333333333, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6514290521542231, "n": 6 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7586283944547176, "n": 4 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8513025194406509, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9003097489476204, "n": 1 } ], "ece": 0.2523388364352286, "n": 16 }, "first_request_s": 5.622470483183861, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 5.622470483183861, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.9455523118376732, "p95_s": 1.0152896009385586 }, "latency_ok_s": { "n": 16, "p50_s": 0.9494975917041302, "p95_s": 2.193301059305668 }, "majority_class_accuracy": 0.5, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 0.2553191489361702, "accuracy_ci": { "hi95": 0.35789473684210527, "lo95": 0.17204301075268819, "n_decisions": 94, "n_scenarios": 88, "point": 0.2553191489361702 }, "brier_mean": 0.9668962372044543, "calibration_n": 94, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3740110596021016, "n": 6 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4635152849886153, "n": 27 }, { "accuracy": 0.1111111111111111, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.55312787382691, "n": 27 }, { "accuracy": 0.3333333333333333, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.653024435043335, "n": 3 }, { "accuracy": 0.375, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7768745496869087, "n": 8 }, { "accuracy": 0.7142857142857143, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8566364049911499, "n": 21 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9011107087135315, "n": 2 } ], "ece": 0.36228345144302293, "n": 94 }, "first_request_s": 1.214595265686512, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 3.590568356215954, "latency_ok_excl_first_s": { "n": 93, "p50_s": 2.5646011531352997, "p95_s": 3.4592101618647577 }, "latency_ok_s": { "n": 94, "p50_s": 2.537208341062069, "p95_s": 3.4591447368264197 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 94, "n_cost_known": 0, "n_planned": 94, "n_renormalized": 0, "n_scorable": 94, "n_state_truncated": 39, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "class": "jev-rebuild", "complete": true, "display": "open-jev-deberta-v3-large (local CPU)", "key": "open-jev-deberta-v3-large", "licence": "Apache-2.0 (model card); DeBERTa-v3 keeps its own terms", "open": "yes", "overall": { "accuracy": 0.5165289256198347, "accuracy_ci": { "hi95": 0.5840336134453782, "lo95": 0.45, "n_decisions": 242, "n_scenarios": 206, "point": 0.5165289256198347 }, "brier_mean": 0.6512140520637424, "calibration_n": 242, "cost_basis": [ "local_cpu_no_provider_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.23076923076923078, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.36552061025912946, "n": 13 }, { "accuracy": 0.10526315789473684, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4563683365520678, "n": 38 }, { "accuracy": 0.46835443037974683, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5553682005103631, "n": 79 }, { "accuracy": 0.75, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6451412205185209, "n": 56 }, { "accuracy": 0.5217391304347826, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7589148140471914, "n": 23 }, { "accuracy": 0.7857142857142857, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8570212413157735, "n": 28 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9212573587894439, "n": 5 } ], "ece": 0.14745988801491164, "n": 242 }, "first_request_s": 5.622470483183861, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 5.622470483183861, "latency_ok_excl_first_s": { "n": 241, "p50_s": 1.761275790631771, "p95_s": 3.291244864463806 }, "latency_ok_s": { "n": 242, "p50_s": 1.767673410475254, "p95_s": 3.349288306012749 }, "majority_class_accuracy": 0.3181818181818182, "mean_input_tokens": null, "mean_output_tokens": null, "n_attempted": 242, "n_cost_known": 0, "n_planned": 242, "n_renormalized": 0, "n_scorable": 242, "n_state_truncated": 80, "operational_success": 1.0, "ordinal_mae": 0.8161814389750361, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "paraphrase_consistency": { "both_correct_rate": 0.3611111111111111, "both_valid": 36, "pairs": 36, "same_answer_rate": 0.8333333333333334 }, "price_input_per_m": null, "price_output_per_m": null, "price_source": "open weights run on our own CPU; no provider tariff", "probability_source": [ "native" ], "ranked": true, "repo": "https://github.com/kotoba-lang/typed-decisions", "request_options": {}, "resolved_model_identities": [ "com-kotobalabs/open-jev-deberta-v3-large" ], "run_finished_utc": "2026-09-19T07:03:19.328909+00:00", "run_started_utc": "2026-09-19T06:55:35.715865+00:00", "stop_reason": null, "underlying": "microsoft/deberta-v3-large" }, { "adapter": "openai_compat", "author": "OpenAI", "by_cohort": { "heldout-private": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 24, "n_scenarios": 24, "point": 1.0 }, "brier_mean": 0.0002822500000000004, "calibration_n": 24, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.072425, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9918333333333331, "n": 24 } ], "ece": 0.008166666666666877, "n": 24 }, "first_request_s": 0.9213507100939751, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.6380441039800644, "latency_ok_excl_first_s": { "n": 23, "p50_s": 0.8548191711306572, "p95_s": 1.1270647779107095 }, "latency_ok_s": { "n": 24, "p50_s": 0.8596999533474445, "p95_s": 1.126737268269062 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 191.125, "mean_output_tokens": 28.5, "n_attempted": 24, "n_cost_known": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0050000000000000044, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.9657534246575342, "accuracy_ci": { "hi95": 0.9931506849315068, "lo95": 0.9315068493150684, "n_decisions": 146, "n_scenarios": 146, "point": 0.9657534246575342 }, "brier_mean": 0.06542171036373083, "calibration_n": 146, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.24012876712328765, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6965174129353234, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7154228855721393, "n": 3 }, { "accuracy": 0.6, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8568749954774051, "n": 5 }, { "accuracy": 0.9781021897810219, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9811288235824764, "n": 137 } ], "ece": 0.019563272181743512, "n": 146 }, "first_request_s": 1.1532617434859276, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 6.600082792341709, "latency_ok_excl_first_s": { "n": 145, "p50_s": 1.118903934955597, "p95_s": 2.4067078053951207 }, "latency_ok_s": { "n": 146, "p50_s": 1.1256222948431969, "p95_s": 2.3900160528719425 }, "majority_class_accuracy": 0.4178082191780822, "mean_input_tokens": 687.5205479452055, "mean_output_tokens": 85.52054794520548, "n_attempted": 146, "n_cost_known": 146, "n_planned": 146, "n_renormalized": 20, "n_scorable": 146, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 0.863013698630137 }, "original-public": { "accuracy": 0.9722222222222222, "accuracy_ci": { "hi95": 1.0, "lo95": 0.9305555555555556, "n_decisions": 72, "n_scenarios": 36, "point": 0.9722222222222222 }, "brier_mean": 0.05618039416666668, "calibration_n": 72, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.08053888888888894, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.9722222222222222, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9902638888888889, "n": 72 } ], "ece": 0.018041666666666734, "n": 72 }, "first_request_s": 0.785710409283638, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.4361780881881714, "latency_ok_excl_first_s": { "n": 71, "p50_s": 0.8413839414715767, "p95_s": 1.082804799079895 }, "latency_ok_s": { "n": 72, "p50_s": 0.8396322578191757, "p95_s": 1.0807839937508108 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 210.69444444444446, "mean_output_tokens": 32.0, "n_attempted": 72, "n_cost_known": 72, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.00833333333333334, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "by_family": { "adequacy": { "accuracy": 0.9285714285714286, "accuracy_ci": { "hi95": 0.9761904761904762, "lo95": 0.8705882352941177, "n_decisions": 84, "n_scenarios": 78, "point": 0.9285714285714286 }, "brier_mean": 0.1283291666666667, "calibration_n": 84, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.21170952380952385, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.85, "n": 2 }, { "accuracy": 0.9512195121951219, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9950609756097556, "n": 82 } ], "ece": 0.06303571428571383, "n": 84 }, "first_request_s": 0.7557826340198517, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 6.600082792341709, "latency_ok_excl_first_s": { "n": 83, "p50_s": 0.7356515973806381, "p95_s": 3.106605110317463 }, "latency_ok_s": { "n": 84, "p50_s": 0.737789686769247, "p95_s": 3.0714312050491532 }, "majority_class_accuracy": 0.8214285714285714, "mean_input_tokens": 536.7619047619048, "mean_output_tokens": 86.96428571428571, "n_attempted": 84, "n_cost_known": 84, "n_planned": 84, "n_renormalized": 0, "n_scorable": 84, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.0764375, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 1.0, "n": 16 } ], "ece": 0.0, "n": 16 }, "first_request_s": 0.914846234023571, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.9876766577363014, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.8178791999816895, "p95_s": 0.9786687798798084 }, "latency_ok_s": { "n": 16, "p50_s": 0.8296315707266331, "p95_s": 0.9780253600329161 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 202.1875, "mean_output_tokens": 30.0, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0011810962500000018, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.09068749999999999, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.978, "n": 16 } ], "ece": 0.02200000000000002, "n": 16 }, "first_request_s": 1.3119276240468025, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.6380441039800644, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.9985770285129547, "p95_s": 1.3086590252816672 }, "latency_ok_s": { "n": 16, "p50_s": 1.0102302059531212, "p95_s": 1.393456744030118 }, "majority_class_accuracy": 0.25, "mean_input_tokens": 214.1875, "mean_output_tokens": 39.875, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.000837500000000001, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.07865, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9868750000000001, "n": 16 } ], "ece": 0.013124999999999942, "n": 16 }, "first_request_s": 0.851229153573513, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.0066055953502655, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.8653890788555145, "p95_s": 0.989351575821638 }, "latency_ok_s": { "n": 16, "p50_s": 0.8583091162145138, "p95_s": 0.9881191458553076 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 217.0, "mean_output_tokens": 29.375, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.007500000000000007, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.9375, "accuracy_ci": { "hi95": 1.0, "lo95": 0.8125, "n_decisions": 16, "n_scenarios": 10, "point": 0.9375 }, "brier_mean": 0.1259875, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.06564999999999999, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.9375, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9893750000000001, "n": 16 } ], "ece": 0.051875000000000115, "n": 16 }, "first_request_s": 0.785710409283638, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.4361780881881714, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.867598831653595, "p95_s": 1.2029624231159683 }, "latency_ok_s": { "n": 16, "p50_s": 0.8364093191921711, "p95_s": 1.186304161325097 }, "majority_class_accuracy": 0.5, "mean_input_tokens": 184.625, "mean_output_tokens": 23.9375, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 94, "n_scenarios": 88, "point": 1.0 }, "brier_mean": 0.00825089950111382, "calibration_n": 94, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.2109510638297872, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6965174129353234, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7154228855721393, "n": 3 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8614583257956752, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9730419405839018, "n": 87 } ], "ece": 0.041682873640018894, "n": 94 }, "first_request_s": 0.9286513105034828, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 3.4389989599585533, "latency_ok_excl_first_s": { "n": 93, "p50_s": 1.176730364561081, "p95_s": 1.5357922360301017 }, "latency_ok_s": { "n": 94, "p50_s": 1.1649960540235043, "p95_s": 1.5355370614677666 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": 659.1382978723404, "mean_output_tokens": 65.93617021276596, "n_attempted": 94, "n_cost_known": 94, "n_planned": 94, "n_renormalized": 20, "n_scorable": 94, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 0.7872340425531915 } }, "class": "llm-baseline", "complete": true, "display": "GPT-5.6 Luna (low reasoning effort)", "key": "gpt-5.6-luna", "licence": "proprietary API", "open": "no", "overall": { "accuracy": 0.9710743801652892, "accuracy_ci": { "hi95": 0.9880952380952381, "lo95": 0.9471544715447154, "n_decisions": 242, "n_scenarios": 206, "point": 0.9710743801652892 }, "brier_mean": 0.05621211608720956, "calibration_n": 242, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.17601570247933887, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6965174129353234, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7154228855721393, "n": 3 }, { "accuracy": 0.6, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8568749954774051, "n": 5 }, { "accuracy": 0.9785407725321889, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9850542868274659, "n": 233 } ], "ece": 0.0163604865228712, "n": 242 }, "first_request_s": 0.785710409283638, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 6.600082792341709, "latency_ok_excl_first_s": { "n": 241, "p50_s": 0.9731973335146904, "p95_s": 1.8197348043322563 }, "latency_ok_s": { "n": 242, "p50_s": 0.9680032916367054, "p95_s": 1.8175220962613816 }, "majority_class_accuracy": 0.3181818181818182, "mean_input_tokens": 496.42561983471074, "mean_output_tokens": 63.94214876033058, "n_attempted": 242, "n_cost_known": 242, "n_planned": 242, "n_renormalized": 20, "n_scorable": 242, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.007500000000000007, "schema_validity": 1.0, "schema_validity_strict": 0.9173553719008265 }, "paraphrase_consistency": { "both_correct_rate": 0.9444444444444444, "both_valid": 36, "pairs": 36, "same_answer_rate": 0.9444444444444444 }, "price_input_per_m": 0.2, "price_output_per_m": 1.2, "price_source": "https://platform.openai.com/docs/pricing (standard tier, read 2026-09-19)", "probability_source": [ "verbalized" ], "ranked": true, "repo": null, "request_options": { "max_completion_tokens": 4096, "max_tokens": null, "reasoning_effort": "low", "temperature": null }, "resolved_model_identities": [ "gpt-5.6-luna" ], "run_finished_utc": "2026-09-19T06:11:47.226110+00:00", "run_started_utc": "2026-09-19T06:07:00.741002+00:00", "stop_reason": null, "underlying": "closed" }, { "adapter": "openai_compat", "author": "Google", "by_cohort": { "heldout-private": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 24, "n_scenarios": 24, "point": 1.0 }, "brier_mean": 0.0016666666666666676, "calibration_n": 24, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.08911458333333333, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.85, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9956521739130434, "n": 23 } ], "ece": 0.01041666666666675, "n": 24 }, "first_request_s": 0.8661342635750771, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.9694131091237068, "latency_ok_excl_first_s": { "n": 23, "p50_s": 0.7190650999546051, "p95_s": 0.8175770223140717 }, "latency_ok_s": { "n": 24, "p50_s": 0.72066705301404, "p95_s": 0.8589595567435026 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 133.45833333333334, "mean_output_tokens": 37.166666666666664, "n_attempted": 24, "n_cost_known": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.9315068493150684, "accuracy_ci": { "hi95": 0.9657534246575342, "lo95": 0.8835616438356164, "n_decisions": 146, "n_scenarios": 146, "point": 0.9315068493150684 }, "brier_mean": 0.14204109589041092, "calibration_n": 146, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.259794520547945, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.75, "n": 1 }, { "accuracy": 0.9047619047619048, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8142857142857145, "n": 21 }, { "accuracy": 0.9349593495934959, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9841463414634145, "n": 123 } ], "ece": 0.05890410958904098, "n": 146 }, "first_request_s": 0.8495721369981766, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.9942519664764404, "latency_ok_excl_first_s": { "n": 145, "p50_s": 0.7567648589611053, "p95_s": 0.8713414818048477 }, "latency_ok_s": { "n": 146, "p50_s": 0.7569496519863605, "p95_s": 0.8712460100650787 }, "majority_class_accuracy": 0.4178082191780822, "mean_input_tokens": 651.7671232876712, "mean_output_tokens": 64.56849315068493, "n_attempted": 146, "n_cost_known": 146, "n_planned": 146, "n_renormalized": 0, "n_scorable": 146, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "original-public": { "accuracy": 0.9861111111111112, "accuracy_ci": { "hi95": 1.0, "lo95": 0.9583333333333334, "n_decisions": 72, "n_scenarios": 36, "point": 0.9861111111111112 }, "brier_mean": 0.029554166666666666, "calibration_n": 72, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.09994791666666662, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8333333333333334, "n": 3 }, { "accuracy": 0.9855072463768116, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9960869565217392, "n": 69 } ], "ece": 0.017083333333333332, "n": 72 }, "first_request_s": 0.7321318835020065, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 2.4719615802168846, "latency_ok_excl_first_s": { "n": 71, "p50_s": 0.7617507502436638, "p95_s": 0.9683890901505947 }, "latency_ok_s": { "n": 72, "p50_s": 0.760966245085001, "p95_s": 0.9674323063343764 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 152.54166666666666, "mean_output_tokens": 41.208333333333336, "n_attempted": 72, "n_cost_known": 72, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "by_family": { "adequacy": { "accuracy": 0.8928571428571429, "accuracy_ci": { "hi95": 0.9529411764705882, "lo95": 0.8214285714285714, "n_decisions": 84, "n_scenarios": 78, "point": 0.8928571428571429 }, "brier_mean": 0.2143452380952381, "calibration_n": 84, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.15894345238095245, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.8928571428571429, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9994047619047619, "n": 84 } ], "ece": 0.106547619047619, "n": 84 }, "first_request_s": 0.639055073261261, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 2.4719615802168846, "latency_ok_excl_first_s": { "n": 83, "p50_s": 0.6781488656997681, "p95_s": 0.809466268867254 }, "latency_ok_s": { "n": 84, "p50_s": 0.677733700722456, "p95_s": 0.8084401469677686 }, "majority_class_accuracy": 0.8214285714285714, "mean_input_tokens": 513.3452380952381, "mean_output_tokens": 20.404761904761905, "n_attempted": 84, "n_cost_known": 84, "n_planned": 84, "n_renormalized": 0, "n_scorable": 84, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.00021250000000000029, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.097328125, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.996875, "n": 16 } ], "ece": 0.0031250000000000444, "n": 16 }, "first_request_s": 0.7899559885263443, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.8143109902739525, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.743505448102951, "p95_s": 0.8073169685900212 }, "latency_ok_s": { "n": 16, "p50_s": 0.746880941092968, "p95_s": 0.8068173956125975 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 138.8125, "mean_output_tokens": 41.75, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0074687500000000006, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.117140625, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8333333333333334, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9869230769230768, "n": 13 } ], "ece": 0.0418750000000001, "n": 16 }, "first_request_s": 0.7681590244174004, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.9694131091237068, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.7910791262984276, "p95_s": 0.8940448746085166 }, "latency_ok_s": { "n": 16, "p50_s": 0.7811347581446171, "p95_s": 0.8886614292860031 }, "majority_class_accuracy": 0.25, "mean_input_tokens": 148.6875, "mean_output_tokens": 53.3125, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.108, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 1.0, "n": 16 } ], "ece": 0.0, "n": 16 }, "first_request_s": 0.8179862424731255, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.9779569283127785, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.7680649310350418, "p95_s": 0.8832260541617869 }, "latency_ok_s": { "n": 16, "p50_s": 0.7695631682872772, "p95_s": 0.8764595631510019 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 156.0, "mean_output_tokens": 46.0, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0006250000000000006, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.068328125, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9937499999999999, "n": 16 } ], "ece": 0.006250000000000089, "n": 16 }, "first_request_s": 0.7321318835020065, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 0.8661342635750771, "latency_ok_excl_first_s": { "n": 15, "p50_s": 0.6904143393039703, "p95_s": 0.8018503479659556 }, "latency_ok_s": { "n": 16, "p50_s": 0.7050411999225616, "p95_s": 0.7972586397081614 }, "majority_class_accuracy": 0.5, "mean_input_tokens": 139.8125, "mean_output_tokens": 22.25, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 0.9787234042553191, "accuracy_ci": { "hi95": 1.0, "lo95": 0.946236559139785, "n_decisions": 94, "n_scenarios": 88, "point": 0.9787234042553191 }, "brier_mean": 0.05072340425531912, "calibration_n": 94, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.2942659574468085, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.75, "n": 1 }, { "accuracy": 0.9090909090909091, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8159090909090913, "n": 22 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9721428571428569, "n": 70 } ], "ece": 0.0494680851063831, "n": 94 }, "first_request_s": 0.750318743288517, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.1156677156686783, "latency_ok_excl_first_s": { "n": 93, "p50_s": 0.8110442608594894, "p95_s": 0.8834649562835694 }, "latency_ok_s": { "n": 94, "p50_s": 0.8109960854053497, "p95_s": 0.8832585126161574 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": 605.2127659574468, "mean_output_tokens": 95.30851063829788, "n_attempted": 94, "n_cost_known": 94, "n_planned": 94, "n_renormalized": 0, "n_scorable": 94, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "class": "llm-baseline", "complete": true, "display": "Gemini 3.1 Flash-Lite", "key": "gemini-3.1-flash-lite", "licence": "proprietary API", "open": "no", "overall": { "accuracy": 0.9545454545454546, "accuracy_ci": { "hi95": 0.9794238683127572, "lo95": 0.9259259259259259, "n_decisions": 242, "n_scenarios": 206, "point": 0.9545454545454546 }, "brier_mean": 0.09465247933884295, "calibration_n": 242, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.1953099173553718, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.75, "n": 1 }, { "accuracy": 0.92, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8180000000000003, "n": 25 }, { "accuracy": 0.958139534883721, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9892093023255819, "n": 215 } ], "ece": 0.040826446280992156, "n": 242 }, "first_request_s": 0.7321318835020065, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 2.4719615802168846, "latency_ok_excl_first_s": { "n": 241, "p50_s": 0.7567648589611053, "p95_s": 0.8763928338885307 }, "latency_ok_s": { "n": 242, "p50_s": 0.756230715662241, "p95_s": 0.8761593606323003 }, "majority_class_accuracy": 0.3181818181818182, "mean_input_tokens": 451.8347107438017, "mean_output_tokens": 54.900826446280995, "n_attempted": 242, "n_cost_known": 242, "n_planned": 242, "n_renormalized": 0, "n_scorable": 242, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "paraphrase_consistency": { "both_correct_rate": 0.9722222222222222, "both_valid": 36, "pairs": 36, "same_answer_rate": 0.9722222222222222 }, "price_input_per_m": 0.25, "price_output_per_m": 1.5, "price_source": "https://ai.google.dev/gemini-api/docs/pricing (paid tier, read 2026-09-19)", "probability_source": [ "verbalized" ], "ranked": true, "repo": null, "request_options": {}, "resolved_model_identities": [ "gemini-3.1-flash-lite" ], "run_finished_utc": "2026-09-19T06:15:12.046880+00:00", "run_started_utc": "2026-09-19T06:11:47.368208+00:00", "stop_reason": null, "underlying": "closed" }, { "adapter": "openai_compat", "author": "DeepSeek", "by_cohort": { "heldout-private": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 24, "n_scenarios": 24, "point": 1.0 }, "brier_mean": 5.8333333333333414e-05, "calibration_n": 24, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.11093124999999998, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9983333333333332, "n": 24 } ], "ece": 0.0016666666666668162, "n": 24 }, "first_request_s": 0.9475938603281975, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 3.6514964029192924, "latency_ok_excl_first_s": { "n": 23, "p50_s": 1.16769677400589, "p95_s": 1.6495045632123946 }, "latency_ok_s": { "n": 24, "p50_s": 1.165334526449442, "p95_s": 1.6411034591495988 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 182.70833333333334, "mean_output_tokens": 139.20833333333334, "n_attempted": 24, "n_cost_known": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.9315068493150684, "accuracy_ci": { "hi95": 0.9726027397260274, "lo95": 0.8904109589041096, "n_decisions": 146, "n_scenarios": 146, "point": 0.9315068493150684 }, "brier_mean": 0.04630228345720216, "calibration_n": 139, "cost_basis": [ "derived_usage_times_tariff", "unknown" ], "cost_per_1000_usd": 0.33125503597122297, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.73, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8577688442211056, "n": 5 }, { "accuracy": 0.9772727272727273, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9779228432787899, "n": 132 } ], "ece": 0.00961849706255203, "n": 139 }, "first_request_s": 2.4726640209555626, "latency_failed_s": { "n": 7, "p50_s": 17.70565514266491, "p95_s": 18.146939270943406 }, "latency_max_s": 11.306431531906128, "latency_ok_excl_first_s": { "n": 138, "p50_s": 1.768506970256567, "p95_s": 6.804122820869093 }, "latency_ok_s": { "n": 139, "p50_s": 1.7690915912389755, "p95_s": 6.766960745304818 }, "majority_class_accuracy": 0.4178082191780822, "mean_input_tokens": 657.9794520547945, "mean_output_tokens": 522.1506849315068, "n_attempted": 146, "n_cost_known": 139, "n_planned": 146, "n_renormalized": 4, "n_scorable": 146, "n_state_truncated": 0, "operational_success": 0.952054794520548, "ordinal_mae": null, "schema_validity": 0.952054794520548, "schema_validity_strict": 0.9246575342465754 }, "original-public": { "accuracy": 0.9861111111111112, "accuracy_ci": { "hi95": 1.0, "lo95": 0.9583333333333334, "n_decisions": 72, "n_scenarios": 36, "point": 0.9861111111111112 }, "brier_mean": 0.0009018591549295775, "calibration_n": 71, "cost_basis": [ "derived_usage_times_tariff", "unknown" ], "cost_per_1000_usd": 0.12080281690140843, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9885915492957746, "n": 71 } ], "ece": 0.011408450704225359, "n": 71 }, "first_request_s": 1.089051105082035, "latency_failed_s": { "n": 1, "p50_s": 3.215444341301918, "p95_s": 3.215444341301918 }, "latency_max_s": 4.353933893144131, "latency_ok_excl_first_s": { "n": 70, "p50_s": 1.1375588960945606, "p95_s": 2.042467338219285 }, "latency_ok_s": { "n": 71, "p50_s": 1.1318739727139473, "p95_s": 2.034134205430746 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 199.88888888888889, "mean_output_tokens": 156.93055555555554, "n_attempted": 72, "n_cost_known": 71, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 0.9861111111111112, "ordinal_mae": 0.0, "schema_validity": 0.9861111111111112, "schema_validity_strict": 0.9861111111111112 } }, "by_family": { "adequacy": { "accuracy": 0.8928571428571429, "accuracy_ci": { "hi95": 0.9529411764705882, "lo95": 0.8271604938271605, "n_decisions": 84, "n_scenarios": 78, "point": 0.8928571428571429 }, "brier_mean": 0.07491538461538462, "calibration_n": 78, "cost_basis": [ "derived_usage_times_tariff", "unknown" ], "cost_per_1000_usd": 0.3433961538461538, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.9615384615384616, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9988461538461538, "n": 78 } ], "ece": 0.037307692307692264, "n": 78 }, "first_request_s": 1.0556603074073792, "latency_failed_s": { "n": 6, "p50_s": 17.86946525424719, "p95_s": 18.14782533980906 }, "latency_max_s": 11.306431531906128, "latency_ok_excl_first_s": { "n": 77, "p50_s": 1.591658852994442, "p95_s": 8.601235774159433 }, "latency_ok_s": { "n": 78, "p50_s": 1.5622772462666035, "p95_s": 8.570920087024565 }, "majority_class_accuracy": 0.8214285714285714, "mean_input_tokens": 531.9880952380952, "mean_output_tokens": 672.6190476190476, "n_attempted": 84, "n_cost_known": 78, "n_planned": 84, "n_renormalized": 0, "n_scorable": 84, "n_state_truncated": 0, "operational_success": 0.9285714285714286, "ordinal_mae": null, "schema_validity": 0.9285714285714286, "schema_validity_strict": 0.9285714285714286 }, "extraction": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0005031249999999997, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.08274375, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.99375, "n": 16 } ], "ece": 0.006249999999999978, "n": 16 }, "first_request_s": 0.7942593917250633, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.3460919484496117, "latency_ok_excl_first_s": { "n": 15, "p50_s": 1.0565420165657997, "p95_s": 1.3052532456815242 }, "latency_ok_s": { "n": 16, "p50_s": 1.0527185574173927, "p95_s": 1.3023361954838037 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 190.375, "mean_output_tokens": 90.3125, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0022847499999999995, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.09523125, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.97125, "n": 16 } ], "ece": 0.028750000000000053, "n": 16 }, "first_request_s": 1.1555778309702873, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 1.5838969498872757, "latency_ok_excl_first_s": { "n": 15, "p50_s": 1.0747464746236801, "p95_s": 1.4426910467445848 }, "latency_ok_s": { "n": 16, "p50_s": 1.0866942666471004, "p95_s": 1.4326049108058214 }, "majority_class_accuracy": 0.25, "mean_input_tokens": 195.875, "mean_output_tokens": 109.75, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0, "calibration_n": 16, "cost_basis": [ "derived_usage_times_tariff" ], "cost_per_1000_usd": 0.152053125, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 1.0, "n": 16 } ], "ece": 0.0, "n": 16 }, "first_request_s": 1.4213277101516724, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 2.3060476183891296, "latency_ok_excl_first_s": { "n": 15, "p50_s": 1.4982846900820732, "p95_s": 2.174040158838033 }, "latency_ok_s": { "n": 16, "p50_s": 1.4601507298648357, "p95_s": 2.164611054584384 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 205.6875, "mean_output_tokens": 202.0, "n_attempted": 16, "n_cost_known": 16, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.9375, "accuracy_ci": { "hi95": 1.0, "lo95": 0.8125, "n_decisions": 16, "n_scenarios": 10, "point": 0.9375 }, "brier_mean": 0.000386666666666667, "calibration_n": 15, "cost_basis": [ "derived_usage_times_tariff", "unknown" ], "cost_per_1000_usd": 0.12819, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9953333333333333, "n": 15 } ], "ece": 0.004666666666666708, "n": 15 }, "first_request_s": 1.089051105082035, "latency_failed_s": { "n": 1, "p50_s": 3.215444341301918, "p95_s": 3.215444341301918 }, "latency_max_s": 3.5423786118626595, "latency_ok_excl_first_s": { "n": 14, "p50_s": 1.1639023423194885, "p95_s": 2.4824030928313725 }, "latency_ok_s": { "n": 15, "p50_s": 1.143243819475174, "p95_s": 2.400866514444349 }, "majority_class_accuracy": 0.5, "mean_input_tokens": 185.9375, "mean_output_tokens": 191.4375, "n_attempted": 16, "n_cost_known": 15, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 0.9375, "ordinal_mae": null, "schema_validity": 0.9375, "schema_validity_strict": 0.9375 }, "routing": { "accuracy": 0.9893617021276596, "accuracy_ci": { "hi95": 1.0, "lo95": 0.967032967032967, "n_decisions": 94, "n_scenarios": 88, "point": 0.9893617021276596 }, "brier_mean": 0.006533800005925806, "calibration_n": 93, "cost_basis": [ "derived_usage_times_tariff", "unknown" ], "cost_per_1000_usd": 0.25049032258064513, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.73, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8577688442211056, "n": 5 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9646025036372122, "n": 86 } ], "ece": 0.04618645662466904, "n": 93 }, "first_request_s": 1.0542996898293495, "latency_failed_s": { "n": 1, "p50_s": 2.312358818948269, "p95_s": 2.312358818948269 }, "latency_max_s": 4.582824312150478, "latency_ok_excl_first_s": { "n": 92, "p50_s": 1.6892830319702625, "p95_s": 2.748785972222686 }, "latency_ok_s": { "n": 93, "p50_s": 1.6839944496750832, "p95_s": 2.745423170924186 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": 613.9255319148937, "mean_output_tokens": 264.6595744680851, "n_attempted": 94, "n_cost_known": 93, "n_planned": 94, "n_renormalized": 4, "n_scorable": 94, "n_state_truncated": 0, "operational_success": 0.9893617021276596, "ordinal_mae": null, "schema_validity": 0.9893617021276596, "schema_validity_strict": 0.9468085106382979 } }, "class": "llm-baseline", "complete": true, "display": "DeepSeek V4.1 Flash (thinking default)", "key": "deepseek-flash", "licence": "open weights, proprietary API route", "open": "weights", "overall": { "accuracy": 0.9545454545454546, "accuracy_ci": { "hi95": 0.9794238683127572, "lo95": 0.9254385964912281, "n_decisions": 242, "n_scenarios": 206, "point": 0.9545454545454546 }, "brier_mean": 0.02778397179722692, "calibration_n": 234, "cost_basis": [ "derived_usage_times_tariff", "unknown" ], "cost_per_1000_usd": 0.2448025641025641, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.73, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8577688442211056, "n": 5 }, { "accuracy": 0.986784140969163, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9834176886026436, "n": 227 } ], "ece": 0.008612566094420454, "n": 234 }, "first_request_s": 1.089051105082035, "latency_failed_s": { "n": 8, "p50_s": 17.059729892760515, "p95_s": 18.146053202077745 }, "latency_max_s": 11.306431531906128, "latency_ok_excl_first_s": { "n": 233, "p50_s": 1.4213277101516724, "p95_s": 4.8912693843245485 }, "latency_ok_s": { "n": 234, "p50_s": 1.4163860343396664, "p95_s": 4.886470635980367 }, "majority_class_accuracy": 0.3181818181818182, "mean_input_tokens": 474.55371900826447, "mean_output_tokens": 375.5123966942149, "n_attempted": 242, "n_cost_known": 234, "n_planned": 242, "n_renormalized": 4, "n_scorable": 242, "n_state_truncated": 0, "operational_success": 0.9669421487603306, "ordinal_mae": 0.0, "schema_validity": 0.9669421487603306, "schema_validity_strict": 0.9504132231404959 }, "paraphrase_consistency": { "both_correct_rate": 0.9722222222222222, "both_valid": 35, "pairs": 36, "same_answer_rate": 1.0 }, "price_input_per_m": 0.15, "price_output_per_m": 0.6, "price_source": "https://api-docs.deepseek.com/quick_start/pricing (cache-miss off-peak; the run is on a Saturday, off-peak all day)", "probability_source": [ "verbalized" ], "ranked": true, "repo": null, "request_options": { "response_format": { "type": "json_object" } }, "resolved_model_identities": [ "deepseek-flash" ], "run_finished_utc": "2026-09-19T06:55:29.148375+00:00", "run_started_utc": "2026-09-19T06:45:53.481323+00:00", "stop_reason": null, "underlying": "DeepSeek-V4.1-Flash" }, { "adapter": "openai_compat", "author": "Qwen / Chutes", "by_cohort": { "heldout-private": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 24, "n_scenarios": 24, "point": 1.0 }, "brier_mean": 8.958333333333346e-05, "calibration_n": 24, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9970833333333333, "n": 24 } ], "ece": 0.0029166666666666785, "n": 24 }, "first_request_s": 4.208928138017654, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 13.482586711645126, "latency_ok_excl_first_s": { "n": 23, "p50_s": 3.9651807993650436, "p95_s": 12.068375802040098 }, "latency_ok_s": { "n": 24, "p50_s": 4.0362021923065186, "p95_s": 11.981396654248234 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 187.20833333333334, "mean_output_tokens": 211.75, "n_attempted": 24, "n_cost_known": 0, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "imported-public-source": { "accuracy": 0.952755905511811, "accuracy_ci": { "hi95": 0.984251968503937, "lo95": 0.9133858267716536, "n_decisions": 127, "n_scenarios": 127, "point": 0.952755905511811 }, "brier_mean": 0.03348981042626313, "calibration_n": 123, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.78, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8542857142857142, "n": 7 }, { "accuracy": 0.9826086956521739, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.971650417215635, "n": 115 } ], "ece": 0.02032684569269893, "n": 123 }, "first_request_s": 5.8964424058794975, "latency_failed_s": { "n": 4, "p50_s": 61.98592819273472, "p95_s": 65.15556229650974 }, "latency_max_s": 55.02765379101038, "latency_ok_excl_first_s": { "n": 122, "p50_s": 6.2192001566290855, "p95_s": 18.403021461889132 }, "latency_ok_s": { "n": 123, "p50_s": 6.20465961843729, "p95_s": 18.339937164634456 }, "majority_class_accuracy": 0.33858267716535434, "mean_input_tokens": 697.6141732283464, "mean_output_tokens": 562.1732283464567, "n_attempted": 127, "n_cost_known": 0, "n_planned": 146, "n_renormalized": 1, "n_scorable": 127, "n_state_truncated": 0, "operational_success": 0.968503937007874, "ordinal_mae": null, "schema_validity": 0.968503937007874, "schema_validity_strict": 0.9606299212598425 }, "original-public": { "accuracy": 0.9861111111111112, "accuracy_ci": { "hi95": 1.0, "lo95": 0.9583333333333334, "n_decisions": 72, "n_scenarios": 36, "point": 0.9861111111111112 }, "brier_mean": 0.000477802816901409, "calibration_n": 71, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9919718309859156, "n": 71 } ], "ece": 0.00802816901408443, "n": 71 }, "first_request_s": 5.253348663449287, "latency_failed_s": { "n": 1, "p50_s": 0.348337322473526, "p95_s": 0.348337322473526 }, "latency_max_s": 68.97106232494116, "latency_ok_excl_first_s": { "n": 70, "p50_s": 4.81880559027195, "p95_s": 9.0508129555732 }, "latency_ok_s": { "n": 71, "p50_s": 4.824444741010666, "p95_s": 9.01999581977725 }, "majority_class_accuracy": 0.16666666666666666, "mean_input_tokens": 203.8450704225352, "mean_output_tokens": 255.46478873239437, "n_attempted": 72, "n_cost_known": 0, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_state_truncated": 0, "operational_success": 0.9861111111111112, "ordinal_mae": 0.0, "schema_validity": 0.9861111111111112, "schema_validity_strict": 0.9861111111111112 } }, "by_family": { "adequacy": { "accuracy": 0.9076923076923077, "accuracy_ci": { "hi95": 0.9705882352941176, "lo95": 0.828125, "n_decisions": 65, "n_scenarios": 59, "point": 0.9076923076923077 }, "brier_mean": 0.05871803278688524, "calibration_n": 61, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.9672131147540983, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9922950819672132, "n": 61 } ], "ece": 0.025081967213114842, "n": 61 }, "first_request_s": 3.2228784412145615, "latency_failed_s": { "n": 4, "p50_s": 61.98592819273472, "p95_s": 65.15556229650974 }, "latency_max_s": 68.97106232494116, "latency_ok_excl_first_s": { "n": 60, "p50_s": 5.79454942420125, "p95_s": 32.14189588762817 }, "latency_ok_s": { "n": 61, "p50_s": 5.768647029995918, "p95_s": 31.674152694642544 }, "majority_class_accuracy": 0.7846153846153846, "mean_input_tokens": 535.1384615384616, "mean_output_tokens": 753.6153846153846, "n_attempted": 65, "n_cost_known": 0, "n_planned": 84, "n_renormalized": 0, "n_scorable": 65, "n_state_truncated": 0, "operational_success": 0.9384615384615385, "ordinal_mae": null, "schema_validity": 0.9384615384615385, "schema_validity_strict": 0.9384615384615385 }, "extraction": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0, "calibration_n": 16, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 1.0, "n": 16 } ], "ece": 0.0, "n": 16 }, "first_request_s": 3.697715863585472, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 9.43321217596531, "latency_ok_excl_first_s": { "n": 15, "p50_s": 3.774955816566944, "p95_s": 7.1005545020103416 }, "latency_ok_s": { "n": 16, "p50_s": 3.771383486688137, "p95_s": 6.933936096727848 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 192.75, "mean_output_tokens": 157.0, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0009687500000000011, "calibration_n": 16, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9818749999999999, "n": 16 } ], "ece": 0.018125000000000058, "n": 16 }, "first_request_s": 5.005506120622158, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 9.32816717773676, "latency_ok_excl_first_s": { "n": 15, "p50_s": 4.986872531473637, "p95_s": 8.634452467411755 }, "latency_ok_s": { "n": 16, "p50_s": 4.996189326047897, "p95_s": 8.584901416674256 }, "majority_class_accuracy": 0.25, "mean_input_tokens": 200.25, "mean_output_tokens": 268.3125, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 0.0, "calibration_n": 16, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 1.0, "n": 16 } ], "ece": 0.0, "n": 16 }, "first_request_s": 3.6943185925483704, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 12.242334097623825, "latency_ok_excl_first_s": { "n": 15, "p50_s": 4.448409840464592, "p95_s": 9.058673828095193 }, "latency_ok_s": { "n": 16, "p50_s": 4.422478258609772, "p95_s": 8.831269523128867 }, "majority_class_accuracy": 0.3125, "mean_input_tokens": 208.9375, "mean_output_tokens": 213.5, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 1.0, "accuracy_ci": { "hi95": 1.0, "lo95": 1.0, "n_decisions": 16, "n_scenarios": 10, "point": 1.0 }, "brier_mean": 1.2500000000000012e-05, "calibration_n": 16, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.999375, "n": 16 } ], "ece": 0.0006249999999999867, "n": 16 }, "first_request_s": 5.253348663449287, "latency_failed_s": { "n": 0, "p50_s": null, "p95_s": null }, "latency_max_s": 10.502751141786575, "latency_ok_excl_first_s": { "n": 15, "p50_s": 5.5458401665091515, "p95_s": 7.498319852352138 }, "latency_ok_s": { "n": 16, "p50_s": 5.399594414979219, "p95_s": 7.28371761739254 }, "majority_class_accuracy": 0.5, "mean_input_tokens": 192.9375, "mean_output_tokens": 266.3125, "n_attempted": 16, "n_cost_known": 0, "n_planned": 16, "n_renormalized": 0, "n_scorable": 16, "n_state_truncated": 0, "operational_success": 1.0, "ordinal_mae": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 0.9893617021276596, "accuracy_ci": { "hi95": 1.0, "lo95": 0.967741935483871, "n_decisions": 94, "n_scenarios": 88, "point": 0.9893617021276596 }, "brier_mean": 0.005998071854089941, "calibration_n": 93, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.78, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8542857142857142, "n": 7 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9631740938799772, "n": 85 } ], "ece": 0.04699141957206385, "n": 93 }, "first_request_s": 6.784624308347702, "latency_failed_s": { "n": 1, "p50_s": 0.348337322473526, "p95_s": 0.348337322473526 }, "latency_max_s": 35.24230071902275, "latency_ok_excl_first_s": { "n": 92, "p50_s": 6.01859887316823, "p95_s": 11.342151518911127 }, "latency_ok_s": { "n": 93, "p50_s": 6.021026194095612, "p95_s": 11.292485550045958 }, "majority_class_accuracy": 0.2127659574468085, "mean_input_tokens": 645.8172043010753, "mean_output_tokens": 334.93548387096774, "n_attempted": 94, "n_cost_known": 0, "n_planned": 94, "n_renormalized": 1, "n_scorable": 94, "n_state_truncated": 0, "operational_success": 0.9893617021276596, "ordinal_mae": null, "schema_validity": 0.9893617021276596, "schema_validity_strict": 0.9787234042553191 } }, "class": "llm-baseline", "complete": false, "display": "Qwen3.8 27B (Chutes TEE)", "key": "qwen3.8-27b", "licence": "open weights", "open": "weights", "overall": { "accuracy": 0.968609865470852, "accuracy_ci": { "hi95": 0.9908675799086758, "lo95": 0.9441860465116279, "n_decisions": 223, "n_scenarios": 187, "point": 0.968609865470852 }, "brier_mean": 0.01906110404784571, "calibration_n": 218, "cost_basis": [ "flat_rate_subscription_no_per_token_tariff" ], "cost_per_1000_usd": null, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.78, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8542857142857142, "n": 7 }, { "accuracy": 0.9904761904761905, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9814276094276083, "n": 210 } ], "ece": 0.014404596422946152, "n": 218 }, "first_request_s": 5.253348663449287, "latency_failed_s": { "n": 5, "p50_s": 61.726791486144066, "p95_s": 64.9843565672636 }, "latency_max_s": 68.97106232494116, "latency_ok_excl_first_s": { "n": 217, "p50_s": 5.760848723351955, "p95_s": 13.001508533954613 }, "latency_ok_s": { "n": 218, "p50_s": 5.753906108438969, "p95_s": 12.97144114784895 }, "majority_class_accuracy": 0.2645739910313901, "mean_input_tokens": 484.51801801801804, "mean_output_tokens": 426.1981981981982, "n_attempted": 223, "n_cost_known": 0, "n_planned": 242, "n_renormalized": 1, "n_scorable": 223, "n_state_truncated": 0, "operational_success": 0.9775784753363229, "ordinal_mae": 0.0, "schema_validity": 0.9775784753363229, "schema_validity_strict": 0.9730941704035875 }, "paraphrase_consistency": { "both_correct_rate": 0.9722222222222222, "both_valid": 35, "pairs": 36, "same_answer_rate": 1.0 }, "price_input_per_m": null, "price_output_per_m": null, "price_source": "flat-rate Chutes subscription; no per-token tariff on this route", "probability_source": [ "verbalized" ], "ranked": true, "repo": null, "request_options": {}, "resolved_model_identities": [ "Qwen/Qwen3.8-27B-TEE" ], "run_finished_utc": "2026-09-19T06:45:53.378636+00:00", "run_started_utc": "2026-09-19T06:15:12.182871+00:00", "stop_reason": "verbalized distribution parse failed: Expecting value: line 1 column 1 (char 0)", "underlying": "Qwen3.8-27B" } ] }