import json, math, collections, statistics, os
HERE = os.path.dirname(os.path.abspath(__file__))
rows = [json.loads(l) for l in open(os.path.join(HERE, "results.jsonl"))]
JEV = "typesafe/jev-1.13"
LLMS = ["openai/gpt-5.4-nano", "google/gemini-3.5-flash-lite", "anthropic/claude-haiku-4.5", "openai/gpt-5.6-terra"]
by = collections.defaultdict(dict)
for r in rows:
    by[(r["system"], r["task"])][r["id"]] = r


def binom_two_sided(b, c):
    n = b + c
    if n == 0:
        return 1.0
    k = min(b, c)
    p = sum(math.comb(n, i) for i in range(0, k + 1)) / 2 ** n
    return min(1.0, 2 * p)


print("=== served model ids, retries, failures ===")
for s in [JEV] + LLMS:
    rs = [r for r in rows if r["system"] == s]
    served = collections.Counter(r.get("served_model") for r in rs if r["ok"])
    print(s, "| served:", dict(served), "| retries>0:", sum(1 for r in rs if r.get("retries")), "| failed:", [(r["task"], r["err"][:90]) for r in rs if not r["ok"]])

print("\n=== mean input tokens per call (billing-relevant) ===")
for s in [JEV] + LLMS:
    out = []
    for t in ("intent8", "intent77", "inject"):
        ok = [r for r in by[(s, t)].values() if r["ok"] and r.get("in_tok")]
        out.append("%s %.0f" % (t, statistics.mean(r["in_tok"] for r in ok)))
    print("%-30s" % s, " | ".join(out))

print("\n=== paired comparison vs Jev (correct/incorrect on identical items), exact McNemar p ===")
for t in ("intent8", "intent77", "inject"):
    for s in LLMS:
        common = [i for i in by[(JEV, t)] if i in by[(s, t)] and by[(JEV, t)][i]["ok"] and by[(s, t)][i]["ok"]]
        b = sum(1 for i in common if by[(JEV, t)][i]["pred"] == by[(JEV, t)][i]["gold"] and by[(s, t)][i]["pred"] != by[(s, t)][i]["gold"])
        c = sum(1 for i in common if by[(JEV, t)][i]["pred"] != by[(JEV, t)][i]["gold"] and by[(s, t)][i]["pred"] == by[(s, t)][i]["gold"])
        print("%-9s vs %-30s n=%3d  Jev-only-right %3d  LLM-only-right %3d  p=%.3f" % (t, s, len(common), b, c, binom_two_sided(b, c)))

print("\n=== Jev vs Terra AGREEMENT (TypeSafe's own metric) vs accuracy against ground truth ===")
for t in ("intent8", "intent77", "inject"):
    common = [i for i in by[(JEV, t)] if i in by[("openai/gpt-5.6-terra", t)] and by[(JEV, t)][i]["ok"] and by[("openai/gpt-5.6-terra", t)][i]["ok"]]
    agree = sum(1 for i in common if by[(JEV, t)][i]["pred"] == by[("openai/gpt-5.6-terra", t)][i]["pred"])
    acc = sum(1 for i in common if by[(JEV, t)][i]["pred"] == by[(JEV, t)][i]["gold"])
    tacc = sum(1 for i in common if by[("openai/gpt-5.6-terra", t)][i]["pred"] == by[("openai/gpt-5.6-terra", t)][i]["gold"])
    print("%-9s n=%3d  Jev-Terra agreement %.1f%% | Jev acc vs gold %.1f%% | Terra acc vs gold %.1f%%" % (t, len(common), 100 * agree / len(common), 100 * acc / len(common), 100 * tacc / len(common)))

print("\n=== Jev confidence: confident errors ===")
for t in ("intent8", "intent77"):
    ok = [r for r in by[(JEV, t)].values() if r["ok"]]
    hi = [r for r in ok if (r.get("conf") or 0) >= 0.9]
    wrong_hi = [r for r in hi if r["pred"] != r["gold"]]
    lo = [r for r in ok if (r.get("conf") or 0) < 0.5]
    print("%-9s conf>=0.9: %d items, %d wrong (%.1f%%) | conf<0.5: %d items, acc %.1f%%" % (t, len(hi), len(wrong_hi), 100 * len(wrong_hi) / max(1, len(hi)), len(lo), 100 * sum(1 for r in lo if r["pred"] == r["gold"]) / max(1, len(lo))))
    print("   wrong-with-conf>=0.9 examples (gold -> pred):", [(r["gold"], r["pred"], r["conf"]) for r in wrong_hi][:6])

print("\n=== intent8: most common Jev confusions ===")
conf = collections.Counter((r["gold"], r["pred"]) for r in by[(JEV, "intent8")].values() if r["ok"] and r["pred"] != r["gold"])
for (g, p), n in conf.most_common(6):
    print("  %2d x  %s  ->  %s" % (n, g, p))
print("  same confusions for Terra:")
conf = collections.Counter((r["gold"], r["pred"]) for r in by[("openai/gpt-5.6-terra", "intent8")].values() if r["ok"] and r["pred"] != r["gold"])
for (g, p), n in conf.most_common(4):
    print("  %2d x  %s  ->  %s" % (n, g, p))

print("\n=== injection: threshold sweep on the probability each system returns (in-sample) ===")
for s in [JEV, "anthropic/claude-haiku-4.5", "openai/gpt-5.6-terra"]:
    ok = [r for r in by[(s, "inject")].values() if r["ok"]]
    pos = sum(1 for r in ok if r["gold"])
    line = []
    for th in (0.5, 0.3, 0.2, 0.1, 0.05):
        tp = sum(1 for r in ok if r["prob"] >= th and r["gold"]); fp = sum(1 for r in ok if r["prob"] >= th and not r["gold"])
        line.append("t>=%.2f: recall %.2f prec %.2f (fp %d)" % (th, tp / pos, tp / (tp + fp) if tp + fp else float("nan"), fp))
    print("%-30s" % s, " | ".join(line))

print("\n=== latency detail (seconds) ===")
def pct(xs, q):
    xs = sorted(xs); return xs[max(0, min(len(xs) - 1, math.ceil(q * len(xs)) - 1))]
for s in [JEV] + LLMS:
    lat = [r["latency"] for r in rows if r["system"] == s and r["ok"]]
    print("%-30s all tasks: p50 %.2f  p90 %.2f  p95 %.2f  p99 %.2f  max %.2f" % (s, statistics.median(lat), pct(lat, .9), pct(lat, .95), pct(lat, .99), max(lat)))

print("\n=== total spend by system ($) ===")
for s in [JEV] + LLMS:
    print("%-30s %.4f" % (s, sum(r.get("cost") or 0 for r in rows if r["system"] == s)))
print("TOTAL %.4f over %d calls" % (sum(r.get("cost") or 0 for r in rows), len(rows)))


print("\n=== confidence-gated cascade: Jev answers if confidence >= threshold, else the fallback model answers ===")
def cascade(task, fb, th):
    ids = [i for i in by[(JEV, task)] if i in by[(fb, task)] and by[(JEV, task)][i]["ok"] and by[(fb, task)][i]["ok"]]
    n = len(ids); corr = 0; cost = 0.0; lat = 0.0; routed = 0
    for i in ids:
        j = by[(JEV, task)][i]; f = by[(fb, task)][i]
        cost += j["cost"]; lat += j["latency"]
        if (j.get("conf") or 0) >= th:
            corr += j["pred"] == j["gold"]
        else:
            routed += 1; corr += f["pred"] == f["gold"]; cost += f["cost"]; lat += f["latency"]
    fb_cost = sum(by[(fb, task)][i]["cost"] for i in ids)
    return dict(n=n, routed=routed / n, acc=corr / n, cost_per_1k=1000 * cost / n, cost_vs_fb=cost / fb_cost, lat=lat / n,
                jev_acc=sum(by[(JEV, task)][i]["pred"] == by[(JEV, task)][i]["gold"] for i in ids) / n,
                fb_acc=sum(by[(fb, task)][i]["pred"] == by[(fb, task)][i]["gold"] for i in ids) / n,
                fb_cost_per_1k=1000 * fb_cost / n, fb_lat=sum(by[(fb, task)][i]["latency"] for i in ids) / n)
for task in ("intent8", "intent77"):
    for fb in ("openai/gpt-5.6-terra", "anthropic/claude-haiku-4.5"):
        for th in (0.8, 0.9, 0.95):
            c = cascade(task, fb, th)
            print("%-9s fallback %-27s conf>=%.2f | routed %4.1f%% | cascade acc %5.1f%% (Jev %5.1f%%, fallback %5.1f%%) | $%.3f/1k = %4.1f%% of fallback-only ($%.3f/1k) | mean latency %.2fs vs %.2fs" % (
                task, fb, th, 100 * c["routed"], 100 * c["acc"], 100 * c["jev_acc"], 100 * c["fb_acc"], c["cost_per_1k"], 100 * c["cost_vs_fb"], c["fb_cost_per_1k"], c["lat"], c["fb_lat"]))