{
 "typesafe/jev-1.13|intent8": {
  "n": 160,
  "answered": 160,
  "failed": 0,
  "acc_of_all": 0.84375,
  "acc_ci": [
   0.7795106867956354,
   0.891869475507467
  ],
  "lat_p50": 0.3962399789597839,
  "lat_p95": 0.5212813750840724,
  "lat_mean": 0.41025615131802623,
  "cost_per_1k": 0.01511475,
  "in_tok_mean": 359.875,
  "gating": [
   {
    "min_conf": 0.0,
    "coverage": 1.0,
    "accuracy": 0.84375
   },
   {
    "min_conf": 0.5,
    "coverage": 0.9625,
    "accuracy": 0.8636363636363636
   },
   {
    "min_conf": 0.7,
    "coverage": 0.88125,
    "accuracy": 0.9078014184397163
   },
   {
    "min_conf": 0.8,
    "coverage": 0.80625,
    "accuracy": 0.9302325581395349
   },
   {
    "min_conf": 0.9,
    "coverage": 0.73125,
    "accuracy": 0.9487179487179487
   },
   {
    "min_conf": 0.95,
    "coverage": 0.6,
    "accuracy": 0.9583333333333334
   }
  ]
 },
 "typesafe/jev-1.13|intent77": {
  "n": 231,
  "answered": 231,
  "failed": 0,
  "acc_of_all": 0.8051948051948052,
  "acc_ci": [
   0.7493022703646343,
   0.8511024279511669
  ],
  "lat_p50": 0.36743720807135105,
  "lat_p95": 0.4686330419499427,
  "lat_mean": 0.36576986451355087,
  "cost_per_1k": 0.03996890909090909,
  "in_tok_mean": 951.6406926406927,
  "gating": [
   {
    "min_conf": 0.0,
    "coverage": 1.0,
    "accuracy": 0.8051948051948052
   },
   {
    "min_conf": 0.5,
    "coverage": 0.9264069264069265,
    "accuracy": 0.8084112149532711
   },
   {
    "min_conf": 0.7,
    "coverage": 0.8311688311688312,
    "accuracy": 0.84375
   },
   {
    "min_conf": 0.8,
    "coverage": 0.7748917748917749,
    "accuracy": 0.8770949720670391
   },
   {
    "min_conf": 0.9,
    "coverage": 0.696969696969697,
    "accuracy": 0.906832298136646
   },
   {
    "min_conf": 0.95,
    "coverage": 0.5974025974025974,
    "accuracy": 0.9492753623188406
   }
  ]
 },
 "typesafe/jev-1.13|inject": {
  "n": 400,
  "answered": 400,
  "failed": 0,
  "acc_of_all": 0.87,
  "acc_ci": [
   0.8334914191139058,
   0.8994692268423305
  ],
  "lat_p50": 0.35354879137594253,
  "lat_p95": 0.4633526250254363,
  "lat_mean": 0.357068630324793,
  "cost_per_1k": 0.013603065000000001,
  "in_tok_mean": 323.8825,
  "tp": 118,
  "fp": 0,
  "fn": 52,
  "tn": 230,
  "precision": 1.0,
  "recall": 0.6941176470588235,
  "f1": 0.8194444444444444,
  "auroc": 0.9908695652173913,
  "brier": 0.08951525
 },
 "anthropic/claude-haiku-4.5|intent8": {
  "n": 160,
  "answered": 160,
  "failed": 0,
  "acc_of_all": 0.7875,
  "acc_ci": [
   0.7177577354595728,
   0.8437602184666584
  ],
  "lat_p50": 1.5318094580434263,
  "lat_p95": 1.9890824579633772,
  "lat_mean": 1.5779562843614259,
  "cost_per_1k": 0.36006875000000005,
  "in_tok_mean": 303.00625
 },
 "anthropic/claude-haiku-4.5|intent77": {
  "n": 231,
  "answered": 231,
  "failed": 0,
  "acc_of_all": 0.7619047619047619,
  "acc_ci": [
   0.7029776141838151,
   0.8122632971368366
  ],
  "lat_p50": 1.4756249580532312,
  "lat_p95": 1.9745489999186248,
  "lat_mean": 1.5233449314547565,
  "cost_per_1k": 1.2677272727272728,
  "in_tok_mean": 1210.9307359307359
 },
 "anthropic/claude-haiku-4.5|inject": {
  "n": 400,
  "answered": 400,
  "failed": 0,
  "acc_of_all": 0.89,
  "acc_ci": [
   0.8555483868341889,
   0.9170317534980603
  ],
  "lat_p50": 1.5551671460270882,
  "lat_p95": 2.1209419169463217,
  "lat_mean": 1.604753694178071,
  "cost_per_1k": 0.347325,
  "in_tok_mean": 267.325,
  "tp": 126,
  "fp": 0,
  "fn": 44,
  "tn": 230,
  "precision": 1.0,
  "recall": 0.7411764705882353,
  "f1": 0.8513513513513514,
  "auroc": 0.9815601023017902,
  "brier": 0.09826625
 },
 "anthropic/claude-haiku-5.5|intent8": {
  "n": 160,
  "answered": 160,
  "failed": 0,
  "acc_of_all": 0.8375,
  "acc_ci": [
   0.7725461426123724,
   0.8866271076488556
  ],
  "lat_p50": 1.3814831250347197,
  "lat_p95": 1.9542491249740124,
  "lat_mean": 1.4165055054603726,
  "cost_per_1k": 0.05002125,
  "in_tok_mean": 407.65
 },
 "anthropic/claude-haiku-5.5|intent77": {
  "n": 231,
  "answered": 231,
  "failed": 0,
  "acc_of_all": 0.7532467532467533,
  "acc_ci": [
   0.6938084397084022,
   0.8043997142132199
  ],
  "lat_p50": 1.3426241669803858,
  "lat_p95": 1.706579125020653,
  "lat_mean": 1.384903418645061,
  "cost_per_1k": 0.17765454545454543,
  "in_tok_mean": 1686.047619047619
 },
 "anthropic/claude-haiku-5.5|inject": {
  "n": 400,
  "answered": 400,
  "failed": 0,
  "acc_of_all": 0.8475,
  "acc_ci": [
   0.8089753764784815,
   0.8794133383047404
  ],
  "lat_p50": 1.3639608540106565,
  "lat_p95": 1.7686840840615332,
  "lat_mean": 1.3874308500252663,
  "cost_per_1k": 0.045391249999999994,
  "in_tok_mean": 358.75,
  "tp": 109,
  "fp": 0,
  "fn": 61,
  "tn": 230,
  "precision": 1.0,
  "recall": 0.6411764705882353,
  "f1": 0.7813620071684588,
  "auroc": 0.9823145780051151,
  "brier": 0.1415655
 },
 "anthropic/claude-sonnet-5.5|intent8": {
  "n": 160,
  "answered": 160,
  "failed": 0,
  "acc_of_all": 0.9,
  "acc_ci": [
   0.843736268117816,
   0.9375061025621578
  ],
  "lat_p50": 1.7503248329740018,
  "lat_p95": 2.260366457980126,
  "lat_mean": 1.7073121666529913,
  "cost_per_1k": 0.9713625,
  "in_tok_mean": 407.65
 },
 "anthropic/claude-sonnet-5.5|intent77": {
  "n": 231,
  "answered": 231,
  "failed": 0,
  "acc_of_all": 0.8051948051948052,
  "acc_ci": [
   0.7493022703646343,
   0.8511024279511669
  ],
  "lat_p50": 1.4554199578706175,
  "lat_p95": 1.8184938749764115,
  "lat_mean": 1.5067868555263007,
  "cost_per_1k": 3.5231774891774887,
  "in_tok_mean": 1686.047619047619
 },
 "anthropic/claude-sonnet-5.5|inject": {
  "n": 400,
  "answered": 398,
  "failed": 2,
  "acc_of_all": 0.9075,
  "acc_ci": [
   0.8751007447962681,
   0.9321464531149922
  ],
  "lat_p50": 1.7776697920635343,
  "lat_p95": 2.5675389170646667,
  "lat_mean": 1.7716380614844671,
  "cost_per_1k": 0.8865628140703516,
  "in_tok_mean": 358.1683417085427,
  "tp": 133,
  "fp": 0,
  "fn": 35,
  "tn": 230,
  "precision": 1.0,
  "recall": 0.7916666666666666,
  "f1": 0.8837209302325582,
  "auroc": 0.9959756728778468,
  "brier": 0.08081608040201005
 }
}