research: add paired original vs proposed PROTEAN analysis

scripts/deep_comparison.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Deep paired comparison: original_protean vs support_weighted(selected beta).
+
+Reads only completed experimental artifacts under ``results/`` and
+``artifacts/analysis/`` and writes derived comparison tables under
+``artifacts/analysis/``. No training, no result mutation.
+
+For every (dataset, alpha, seed) the ``original_protean`` run and the
+``support_weighted`` run at the validation-selected beta are matched by
+dataset, alpha, seed, and manifest content hash, and compared on test
+metrics (global, per-client, per-class, confusion matrices).
+
+Run:
+    .venv/bin/python scripts/deep_comparison.py
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+RESULTS = ROOT / "results"
+ANALYSIS = ROOT / "artifacts" / "analysis"
+
+DATASETS = ["xiiotid", "5gnidd"]
+ALPHAS = [0.25, 0.50, 0.75]
+SEEDS = [0, 1, 2]
+# Validation-selected beta per condition (source: final__selection_strength.csv).
+SELECTED = {("xiiotid", 0.25): 0.0, ("xiiotid", 0.5): 0.75, ("xiiotid", 0.75): 0.5,
+            ("5gnidd", 0.25): 0.0, ("5gnidd", 0.5): 1.0, ("5gnidd", 0.75): 0.5}
+GLOBAL_METRICS = ["rare_class_macro_f1", "rare_class_recall", "macro_f1",
+                  "weighted_f1", "accuracy"]
+# "Effectively unchanged" band for paired per-client deltas. Chosen to reflect
+# the observed original_protean vs support_weighted(beta=0) residual divergence
+# (the beta=0 conditions are only semantically equivalent to the original under
+# the shared S_j rule, so their per-client residuals are numerical, not a
+# weighting effect). Reported alongside the raw deltas so a reader can apply a
+# different threshold.
+TOL = 0.02
+
+
+def run_dir(ds: str, alpha: float, seed: int, mode: str, beta: float | None = None) -> Path:
+    if mode == "original_protean":
+        return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__original_protean"
+    return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}"
+
+
+def read_json(p: Path):
+    return json.loads(p.read_text())
+
+
+def read_csv(p: Path) -> list[dict]:
+    with p.open() as fh:
+        return list(csv.DictReader(fh))
+
+
+def verify_pairing(ds, alpha, seed, beta) -> tuple[bool, dict]:
+    o = read_json(run_dir(ds, alpha, seed, "original_protean") / "metadata.json")
+    s = read_json(run_dir(ds, alpha, seed, "support_weighted", beta) / "metadata.json")
+    ok = (o["dataset"] == s["dataset"] == ds
+          and abs(float(o["alpha"]) - alpha) < 1e-9
+          and abs(float(s["alpha"]) - alpha) < 1e-9
+          and int(o["seed"]) == int(s["seed"]) == seed
+          and o["manifest_content_hash"] == s["manifest_content_hash"]
+          and abs(float(s["beta"]) - beta) < 1e-9)
+    return ok, o
+
+
+def main() -> None:
+    ANALYSIS.mkdir(parents=True, exist_ok=True)
+    global_rows, client_rows, class_rows, cm_rows = [], [], [], []
+    conditions = []
+
+    for ds in DATASETS:
+        for alpha in ALPHAS:
+            beta = SELECTED[(ds, alpha)]
+            cond = {"dataset": ds, "alpha": alpha, "selected_beta": beta}
+            conditions.append(cond)
+            for seed in SEEDS:
+                ok, meta = verify_pairing(ds, alpha, seed, beta)
+                if not ok:
+                    raise SystemExit(f"pairing failed for {ds} a{alpha} s{seed}")
+                odir = run_dir(ds, alpha, seed, "original_protean")
+                sdir = run_dir(ds, alpha, seed, "support_weighted", beta)
+
+                og = read_json(odir / "test_global_metrics.json")
+                sg = read_json(sdir / "test_global_metrics.json")
+                for m in GLOBAL_METRICS:
+                    global_rows.append({
+                        "dataset": ds, "alpha": alpha, "seed": seed,
+                        "selected_beta": beta, "metric": m,
+                        "original": og[m], "proposed": sg[m],
+                        "delta": sg[m] - og[m]})
+
+                opc = {int(r["client_id"]): r for r in read_csv(odir / "per_client.csv")}
+                spc = {int(r["client_id"]): r for r in read_csv(sdir / "per_client.csv")}
+                assert set(opc) == set(spc), "client id mismatch"
+                for cid in sorted(opc):
+                    o, s = opc[cid], spc[cid]
+                    dr = float(s["rare_class_macro_f1"]) - float(o["rare_class_macro_f1"])
+                    dm = float(s["macro_f1"]) - float(o["macro_f1"])
+                    da = float(s["accuracy"]) - float(o["accuracy"])
+                    dre = float(s["rare_class_recall"]) - float(o["rare_class_recall"])
+                    status = ("unchanged" if abs(dr) <= TOL
+                              else ("improved" if dr > 0 else "degraded"))
+                    client_rows.append({
+                        "dataset": ds, "alpha": alpha, "seed": seed,
+                        "selected_beta": beta, "client_id": cid,
+                        "n_samples": int(o["n_samples"]),
+                        "orig_rare_f1": o["rare_class_macro_f1"],
+                        "prop_rare_f1": s["rare_class_macro_f1"],
+                        "delta_rare_f1": dr,
+                        "orig_rare_recall": o["rare_class_recall"],
+                        "prop_rare_recall": s["rare_class_recall"],
+                        "delta_rare_recall": dre,
+                        "orig_macro_f1": o["macro_f1"], "prop_macro_f1": s["macro_f1"],
+                        "delta_macro_f1": dm,
+                        "orig_accuracy": o["accuracy"], "prop_accuracy": s["accuracy"],
+                        "delta_accuracy": da,
+                        "status": status})
+
+                opk = {int(r["class_id"]): r for r in read_csv(odir / "test_per_class.csv")}
+                spk = {int(r["class_id"]): r for r in read_csv(sdir / "test_per_class.csv")}
+                for cl in sorted(opk):
+                    o, s = opk[cl], spk[cl]
+                    class_rows.append({
+                        "dataset": ds, "alpha": alpha, "seed": seed,
+                        "class_id": cl, "class_name": o["class_name"],
+                        "support": int(o["support"]),
+                        "orig_precision": o["precision"], "prop_precision": s["precision"],
+                        "delta_precision": float(s["precision"]) - float(o["precision"]),
+                        "orig_recall": o["recall"], "prop_recall": s["recall"],
+                        "delta_recall": float(s["recall"]) - float(o["recall"]),
+                        "orig_f1": o["f1"], "prop_f1": s["f1"],
+                        "delta_f1": float(s["f1"]) - float(o["f1"])})
+
+                ocm = read_json(odir / "test_confusion_matrix.json")
+                scm = read_json(sdir / "test_confusion_matrix.json")
+                names = meta["class_names"]
+                for i, t in enumerate(names):
+                    row_o = sum(ocm[i])
+                    row_s = sum(scm[i])
+                    for j, p in enumerate(names):
+                        cm_rows.append({
+                            "dataset": ds, "alpha": alpha, "seed": seed,
+                            "true_class": t, "pred_class": p,
+                            "orig_count": ocm[i][j], "prop_count": scm[i][j],
+                            "delta": scm[i][j] - ocm[i][j],
+                            "orig_row_rate": (ocm[i][j] / row_o) if row_o else 0.0,
+                            "prop_row_rate": (scm[i][j] / row_s) if row_s else 0.0,
+                            "delta_row_rate": ((scm[i][j] / row_s) - (ocm[i][j] / row_o))
+                            if row_o and row_s else 0.0})
+
+    _write(ANALYSIS / "deepcmp__global_per_seed.csv", global_rows)
+    _write(ANALYSIS / "deepcmp__per_client.csv", client_rows)
+    _write(ANALYSIS / "deepcmp__per_class.csv", class_rows)
+    _write(ANALYSIS / "deepcmp__confusion_delta.csv", cm_rows)
+
+    # Summaries -------------------------------------------------------------
+    summary = []
+    for c in conditions:
+        ds, alpha, beta = c["dataset"], c["alpha"], c["selected_beta"]
+        for m in GLOBAL_METRICS:
+            rows = [r for r in global_rows
+                    if r["dataset"] == ds and r["alpha"] == alpha and r["metric"] == m]
+            orig = [float(r["original"]) for r in rows]
+            prop = [float(r["proposed"]) for r in rows]
+            dlt = [float(r["delta"]) for r in rows]
+            summary.append({
+                "dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m,
+                "orig_mean": _mean(orig), "orig_std": _std(orig),
+                "prop_mean": _mean(prop), "prop_std": _std(prop),
+                "delta_mean": _mean(dlt), "delta_std": _std(dlt),
+                "per_seed_deltas": "|".join(f"{d:.6f}" for d in dlt), "n": len(rows)})
+    _write(ANALYSIS / "deepcmp__global_summary.csv", summary)
+
+    for c in conditions:
+        ds, alpha = c["dataset"], c["alpha"]
+        rows = [r for r in client_rows if r["dataset"] == ds and r["alpha"] == alpha]
+        dlt = [float(r["delta_rare_f1"]) for r in rows]
+        st = [r["status"] for r in rows]
+        c["improved"] = st.count("improved")
+        c["degraded"] = st.count("degraded")
+        c["unchanged"] = st.count("unchanged")
+        c["total"] = len(rows)
+        c["mean_delta"] = _mean(dlt)
+        c["median_delta"] = _median(dlt)
+        c["min_delta"] = min(dlt)
+        c["max_delta"] = max(dlt)
+    _write(ANALYSIS / "deepcmp__per_client_by_condition.csv", [
+        {k: c[k] for k in ("dataset", "alpha", "selected_beta", "total",
+                           "improved", "degraded", "unchanged",
+                           "mean_delta", "median_delta", "min_delta", "max_delta")}
+        for c in conditions])
+
+    print("=== GLOBAL SUMMARY (delta = proposed - original) ===")
+    for r in summary:
+        print(f"  {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} "
+              f"{r['metric']:20} orig={float(r['orig_mean']):.4f} "
+              f"prop={float(r['prop_mean']):.4f} delta={float(r['delta_mean']):+.4f} "
+              f"per-seed=[{r['per_seed_deltas']}]")
+    print("=== PER-CLIENT COUNTS (rare-class F1) ===")
+    for c in conditions:
+        print(f"  {c['dataset']:8} a{c['alpha']:<4} beta={c['selected_beta']:<4} "
+              f"improved={c['improved']} degraded={c['degraded']} "
+              f"unchanged={c['unchanged']} (of {c['total']}) "
+              f"meanDelta={c['mean_delta']:+.4f} min={c['min_delta']:+.4f} "
+              f"max={c['max_delta']:+.4f}")
+    print("wrote deepcmp__* derived tables to", ANALYSIS)
+
+
+def _mean(v):
+    return sum(v) / len(v) if v else 0.0
+
+
+def _std(v):
+    if len(v) < 2:
+        return 0.0
+    m = _mean(v)
+    return (sum((x - m) ** 2 for x in v) / (len(v) - 1)) ** 0.5
+
+
+def _median(v):
+    s = sorted(v)
+    n = len(s)
+    return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
+
+
+def _write(path: Path, rows: list[dict]) -> None:
+    with path.open("w", newline="") as fh:
+        w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
+        w.writeheader()
+        w.writerows(rows)
+
+
+if __name__ == "__main__":
+    main()