import json, math, collections, statistics, os
HERE = os.path.dirname(os.path.abspath(__file__))
rows = [json.loads(l) for l in open(os.path.join(HERE, "results.jsonl"))]
JEV = "typesafe/jev-1.13"
H45, H55, S55 = "anthropic/claude-haiku-4.5", "anthropic/claude-haiku-5.5", "anthropic/claude-sonnet-5.5"
LLMS = [H45, H55, S55]
REF = S55  # strongest model in this run: agreement and confusion reference
by = collections.defaultdict(dict)
for r in rows:
    by[(r["system"], r["task"])][r["id"]] = r


def binom_two_sided(b, c):
    n = b + c
    if n == 0:
        return 1.0
    k = min(b, c)
    p = sum(math.comb(n, i) for i in range(0, k + 1)) / 2 ** n
    return min(1.0, 2 * p)


print("=== served model ids, retries, failures ===")
for s in [JEV] + LLMS:
    rs = [r for r in rows if r["system"] == s]
    served = collections.Counter(r.get("served_model") for r in rs if r["ok"])
    print(s, "| served:", dict(served), "| retries>0:", sum(1 for r in rs if r.get("retries")), "| failed:", [(r["task"], r["err"][:90]) for r in rs if not r["ok"]])

print("\n=== mean input tokens per call (billing-relevant) ===")
for s in [JEV] + LLMS:
    out = []
    for t in ("intent8", "intent77", "inject"):
        ok = [r for r in by[(s, t)].values() if r["ok"] and r.get("in_tok")]
        out.append("%s %.0f" % (t, statistics.mean(r["in_tok"] for r in ok)))
    print("%-30s" % s, " | ".join(out))

print("\n=== paired comparison vs Jev (correct/incorrect on identical items), exact McNemar p ===")
for t in ("intent8", "intent77", "inject"):
    for s in LLMS:
        common = [i for i in by[(JEV, t)] if i in by[(s, t)] and by[(JEV, t)][i]["ok"] and by[(s, t)][i]["ok"]]
        b = sum(1 for i in common if by[(JEV, t)][i]["pred"] == by[(JEV, t)][i]["gold"] and by[(s, t)][i]["pred"] != by[(s, t)][i]["gold"])
        c = sum(1 for i in common if by[(JEV, t)][i]["pred"] != by[(JEV, t)][i]["gold"] and by[(s, t)][i]["pred"] == by[(s, t)][i]["gold"])
        print("%-9s vs %-30s n=%3d  Jev-only-right %3d  LLM-only-right %3d  p=%.3f" % (t, s, len(common), b, c, binom_two_sided(b, c)))

print("\n=== Jev vs Sonnet 5.5 AGREEMENT (TypeSafe's own metric) vs accuracy against ground truth ===")
for t in ("intent8", "intent77", "inject"):
    common = [i for i in by[(JEV, t)] if i in by[(REF, t)] and by[(JEV, t)][i]["ok"] and by[(REF, t)][i]["ok"]]
    agree = sum(1 for i in common if by[(JEV, t)][i]["pred"] == by[(REF, t)][i]["pred"])
    acc = sum(1 for i in common if by[(JEV, t)][i]["pred"] == by[(JEV, t)][i]["gold"])
    tacc = sum(1 for i in common if by[(REF, t)][i]["pred"] == by[(REF, t)][i]["gold"])
    print("%-9s n=%3d  Jev-Sonnet agreement %.1f%% | Jev acc vs gold %.1f%% | Sonnet acc vs gold %.1f%%" % (t, len(common), 100 * agree / len(common), 100 * acc / len(common), 100 * tacc / len(common)))

print("\n=== Jev confidence: confident errors ===")
for t in ("intent8", "intent77"):
    ok = [r for r in by[(JEV, t)].values() if r["ok"]]
    hi = [r for r in ok if (r.get("conf") or 0) >= 0.9]
    wrong_hi = [r for r in hi if r["pred"] != r["gold"]]
    lo = [r for r in ok if (r.get("conf") or 0) < 0.5]
    print("%-9s conf>=0.9: %d items, %d wrong (%.1f%%) | conf<0.5: %d items, acc %.1f%%" % (t, len(hi), len(wrong_hi), 100 * len(wrong_hi) / max(1, len(hi)), len(lo), 100 * sum(1 for r in lo if r["pred"] == r["gold"]) / max(1, len(lo))))
    print("   wrong-with-conf>=0.9 examples (gold -> pred):", [(r["gold"], r["pred"], r["conf"]) for r in wrong_hi][:6])

print("\n=== intent8: most common Jev confusions ===")
conf = collections.Counter((r["gold"], r["pred"]) for r in by[(JEV, "intent8")].values() if r["ok"] and r["pred"] != r["gold"])
for (g, p), n in conf.most_common(6):
    print("  %2d x  %s  ->  %s" % (n, g, p))
print("  same confusions for Sonnet 5.5:")
conf = collections.Counter((r["gold"], r["pred"]) for r in by[(REF, "intent8")].values() if r["ok"] and r["pred"] != r["gold"])
for (g, p), n in conf.most_common(4):
    print("  %2d x  %s  ->  %s" % (n, g, p))

print("\n=== injection: threshold sweep on the probability each system returns (in-sample) ===")
for s in [JEV] + LLMS:
    ok = [r for r in by[(s, "inject")].values() if r["ok"]]
    pos = sum(1 for r in ok if r["gold"])
    line = []
    for th in (0.5, 0.3, 0.2, 0.1, 0.05):
        tp = sum(1 for r in ok if r["prob"] >= th and r["gold"]); fp = sum(1 for r in ok if r["prob"] >= th and not r["gold"])
        line.append("t>=%.2f: recall %.2f prec %.2f (fp %d)" % (th, tp / pos, tp / (tp + fp) if tp + fp else float("nan"), fp))
    print("%-30s" % s, " | ".join(line))

print("\n=== latency detail (seconds) ===")
def pct(xs, q):
    xs = sorted(xs); return xs[max(0, min(len(xs) - 1, math.ceil(q * len(xs)) - 1))]
for s in [JEV] + LLMS:
    lat = [r["latency"] for r in rows if r["system"] == s and r["ok"]]
    print("%-30s all tasks: p50 %.2f  p90 %.2f  p95 %.2f  p99 %.2f  max %.2f" % (s, statistics.median(lat), pct(lat, .9), pct(lat, .95), pct(lat, .99), max(lat)))

print("\n=== total spend by system ($) ===")
for s in [JEV] + LLMS:
    print("%-30s %.4f" % (s, sum(r.get("cost") or 0 for r in rows if r["system"] == s)))
print("TOTAL %.4f over %d calls" % (sum(r.get("cost") or 0 for r in rows), len(rows)))


print("\n=== confidence-gated cascade: Jev answers if confidence >= threshold, else the fallback model answers ===")
def cascade(task, fb, th):
    ids = [i for i in by[(JEV, task)] if i in by[(fb, task)] and by[(JEV, task)][i]["ok"] and by[(fb, task)][i]["ok"]]
    n = len(ids); corr = 0; cost = 0.0; lat = 0.0; routed = 0
    for i in ids:
        j = by[(JEV, task)][i]; f = by[(fb, task)][i]
        cost += j["cost"]; lat += j["latency"]
        if (j.get("conf") or 0) >= th:
            corr += j["pred"] == j["gold"]
        else:
            routed += 1; corr += f["pred"] == f["gold"]; cost += f["cost"]; lat += f["latency"]
    fb_cost = sum(by[(fb, task)][i]["cost"] for i in ids)
    return dict(n=n, routed=routed / n, acc=corr / n, cost_per_1k=1000 * cost / n, cost_vs_fb=cost / fb_cost, lat=lat / n,
                jev_acc=sum(by[(JEV, task)][i]["pred"] == by[(JEV, task)][i]["gold"] for i in ids) / n,
                fb_acc=sum(by[(fb, task)][i]["pred"] == by[(fb, task)][i]["gold"] for i in ids) / n,
                fb_cost_per_1k=1000 * fb_cost / n, fb_lat=sum(by[(fb, task)][i]["latency"] for i in ids) / n)
for task in ("intent8", "intent77"):
    for fb in (S55, H55, H45):
        for th in (0.8, 0.9, 0.95):
            c = cascade(task, fb, th)
            print("%-9s fallback %-27s conf>=%.2f | routed %4.1f%% | cascade acc %5.1f%% (Jev %5.1f%%, fallback %5.1f%%) | $%.3f/1k = %4.1f%% of fallback-only ($%.3f/1k) | mean latency %.2fs vs %.2fs" % (
                task, fb, th, 100 * c["routed"], 100 * c["acc"], 100 * c["jev_acc"], 100 * c["fb_acc"], c["cost_per_1k"], 100 * c["cost_vs_fb"], c["fb_cost_per_1k"], c["lat"], c["fb_lat"]))


print("\n=== Haiku 5.5 vs Haiku 4.5, paired on identical items, exact McNemar p ===")
for t in ("intent8", "intent77", "inject"):
    common = [i for i in by[(H55, t)] if i in by[(H45, t)] and by[(H55, t)][i]["ok"] and by[(H45, t)][i]["ok"]]
    b = sum(1 for i in common if by[(H55, t)][i]["pred"] == by[(H55, t)][i]["gold"] and by[(H45, t)][i]["pred"] != by[(H45, t)][i]["gold"])
    c = sum(1 for i in common if by[(H55, t)][i]["pred"] != by[(H55, t)][i]["gold"] and by[(H45, t)][i]["pred"] == by[(H45, t)][i]["gold"])
    print("%-9s n=%3d  Haiku-5.5-only-right %3d  Haiku-4.5-only-right %3d  p=%.3f" % (t, len(common), b, c, binom_two_sided(b, c)))

print("\n=== billed tokens per call: Haiku 5.5 vs Haiku 4.5 (same prompts) ===")
for t in ("intent8", "intent77", "inject"):
    ids = [i for i in by[(H55, t)] if i in by[(H45, t)] and by[(H55, t)][i]["ok"] and by[(H45, t)][i]["ok"]]
    i55 = statistics.mean(by[(H55, t)][i]["in_tok"] for i in ids); i45 = statistics.mean(by[(H45, t)][i]["in_tok"] for i in ids)
    o55 = statistics.mean(by[(H55, t)][i]["out_tok"] for i in ids); o45 = statistics.mean(by[(H45, t)][i]["out_tok"] for i in ids)
    print("%-9s input %.0f vs %.0f (x%.2f) | output %.1f vs %.1f" % (t, i55, i45, i55 / i45, o55, o45))

print("\n=== run-to-run check: same system, same items, September 2026 run vs this run ===")
old = collections.defaultdict(dict)
for l in open(os.path.join(HERE, "..", "jev-vs-llm", "results.jsonl")):
    r = json.loads(l)
    old[(r["system"], r["task"])][r["id"]] = r
for s in (JEV, H45):
    for t in ("intent8", "intent77", "inject"):
        ids = [i for i in by[(s, t)] if i in old[(s, t)] and by[(s, t)][i]["ok"] and old[(s, t)][i]["ok"]]
        same = sum(1 for i in ids if by[(s, t)][i]["pred"] == old[(s, t)][i]["pred"])
        acc_new = sum(1 for i in ids if by[(s, t)][i]["pred"] == by[(s, t)][i]["gold"]) / len(ids)
        acc_old = sum(1 for i in ids if old[(s, t)][i]["pred"] == old[(s, t)][i]["gold"]) / len(ids)
        lat_new = statistics.median(by[(s, t)][i]["latency"] for i in ids); lat_old = statistics.median(old[(s, t)][i]["latency"] for i in ids)
        print("%-27s %-9s n=%3d  same answer %5.1f%% | acc Sep %5.1f%% -> Oct %5.1f%% | p50 latency Sep %.2fs -> Oct %.2fs" % (
            s, t, len(ids), 100 * same / len(ids), 100 * acc_old, 100 * acc_new, lat_old, lat_new))