{
  "benchmark": "Simple Speech Eval",
  "cases": 10,
  "repetitions": 3,
  "interval": "case-clustered bootstrap 95%; 2000 resamples; seed 20260925; unknown criteria at midpoint",
  "caveat": "Historical runs, not a matched hosted control. Hosted latency uses caller recordings; on-prem latency uses target received-audio recordings. Both use the same CPU word alignment. On-prem qualification has not passed.",
  "sources": {
    "hosted": "eval/dialt-benchmarks/results/2026-10-21/live_comparison/live-comparison.json",
    "external_published_points": "web/blog/eleven-realtime/data.json",
    "external_oct21_intervals": "eval/dialt-benchmarks/results/2026-10-21/live_comparison/three-minute-followup.json",
    "gemini_sep26_interval": "eval/dialt-benchmarks/results/2026-09-26-live/live_comparison/live-comparison.json",
    "on_prem_scores": "eval/results/session_capacity/single-gpu-criterion-scores.json",
    "h100": "eval/results/session_capacity/single-gpu-h100-c4-r3-final-diagnosis-complete/assessments.json",
    "b200": "eval/results/session_capacity/single-gpu-b200-c4-r3-diagnosis/assessments.json"
  },
  "arms": [
    {"label": "Dialt Fast", "date": "2026-10-21", "score": 0.942222, "interval": [0.8956, 0.9867], "p50_ms": 1513, "p90_ms": 2748, "aligned_responses": 95, "clock": "caller recording", "criterion_coverage": 1},
    {"label": "Dialt Smart", "date": "2026-10-21", "score": 1, "interval": [1, 1], "p50_ms": 1922, "p90_ms": 3335, "aligned_responses": 94, "clock": "caller recording", "criterion_coverage": 1},
    {"label": "Dialt On-Prem H100", "date": "2026-10-28", "score": 0.8347685, "interval": [0.7338, 0.9248], "p50_ms": 1320.1, "p90_ms": 2471.9, "aligned_responses": 92, "clock": "target received-audio recording", "criterion_coverage": 0.954167, "concurrency": 4},
    {"label": "Dialt On-Prem B200", "date": "2026-10-28", "score": 0.870556, "interval": [0.775, 0.9583], "p50_ms": 1102, "p90_ms": 1826.9, "aligned_responses": 102, "clock": "target received-audio recording", "criterion_coverage": 0.95, "concurrency": 4},
    {"label": "GPT Live-1", "date": "2026-10-21", "score": 0.7967595, "interval": [0.6653, 0.9148], "p50_ms": 2273, "p90_ms": 3225, "aligned_responses": 94, "clock": "caller recording", "caller": "US Male", "limit_s": 180, "group": "external"},
    {"label": "Gemini 3.8 Live", "date": "2026-09-26", "score": 0.899, "interval": [0.8097, 0.9778], "p50_ms": 2018, "p90_ms": 2720, "aligned_responses": 84, "clock": "caller recording", "caller": "Irish Male", "limit_s": 120, "group": "external", "note": "Published point retained. The September 26 source cohort summary reports score 0.8986 and p50/p90 2016.5/2718 ms; its exact retained bootstrap interval is used here."},
    {"label": "Eleven Realtime Mini", "date": "2026-10-21", "score": 0.709074, "interval": [0.6014, 0.8208], "p50_ms": 2104.5, "p90_ms": 3544, "aligned_responses": 100, "clock": "caller recording", "caller": "US Male", "limit_s": 180, "group": "eleven"},
    {"label": "Eleven Realtime Max", "date": "2026-10-21", "score": 0.876852, "interval": [0.8125, 0.9338], "p50_ms": 2094, "p90_ms": 3395, "aligned_responses": 90, "clock": "caller recording", "caller": "US Male", "limit_s": 180, "group": "eleven"}
  ]
}