#!/usr/bin/env python3 """Deep paired comparison: original_protean vs support_weighted(selected beta). Reads only completed experimental artifacts under ``results/`` and ``artifacts/analysis/`` and writes derived comparison tables under ``artifacts/analysis/``. No training, no result mutation. For every (dataset, alpha, seed) the ``original_protean`` run and the ``support_weighted`` run at the validation-selected beta are matched by dataset, alpha, seed, and manifest content hash, and compared on test metrics (global, per-client, per-class, confusion matrices). Run: .venv/bin/python scripts/deep_comparison.py """ from __future__ import annotations import csv import json from pathlib import Path ROOT = Path(__file__).resolve().parents[1] RESULTS = ROOT / "results" ANALYSIS = ROOT / "artifacts" / "analysis" DATASETS = ["xiiotid", "5gnidd"] ALPHAS = [0.25, 0.50, 0.75] SEEDS = [0, 1, 2] # Validation-selected beta per condition (source: final__selection_strength.csv). SELECTED = {("xiiotid", 0.25): 0.0, ("xiiotid", 0.5): 0.75, ("xiiotid", 0.75): 0.5, ("5gnidd", 0.25): 0.0, ("5gnidd", 0.5): 1.0, ("5gnidd", 0.75): 0.5} GLOBAL_METRICS = ["rare_class_macro_f1", "rare_class_recall", "macro_f1", "weighted_f1", "accuracy"] # "Effectively unchanged" band for paired per-client deltas. Chosen to reflect # the observed original_protean vs support_weighted(beta=0) residual divergence # (the beta=0 conditions are only semantically equivalent to the original under # the shared S_j rule, so their per-client residuals are numerical, not a # weighting effect). Reported alongside the raw deltas so a reader can apply a # different threshold. TOL = 0.02 def run_dir(ds: str, alpha: float, seed: int, mode: str, beta: float | None = None) -> Path: if mode == "original_protean": return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__original_protean" return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}" def read_json(p: Path): return json.loads(p.read_text()) def read_csv(p: Path) -> list[dict]: with p.open() as fh: return list(csv.DictReader(fh)) def verify_pairing(ds, alpha, seed, beta) -> tuple[bool, dict]: o = read_json(run_dir(ds, alpha, seed, "original_protean") / "metadata.json") s = read_json(run_dir(ds, alpha, seed, "support_weighted", beta) / "metadata.json") ok = (o["dataset"] == s["dataset"] == ds and abs(float(o["alpha"]) - alpha) < 1e-9 and abs(float(s["alpha"]) - alpha) < 1e-9 and int(o["seed"]) == int(s["seed"]) == seed and o["manifest_content_hash"] == s["manifest_content_hash"] and abs(float(s["beta"]) - beta) < 1e-9) return ok, o def main() -> None: ANALYSIS.mkdir(parents=True, exist_ok=True) global_rows, client_rows, class_rows, cm_rows = [], [], [], [] conditions = [] for ds in DATASETS: for alpha in ALPHAS: beta = SELECTED[(ds, alpha)] cond = {"dataset": ds, "alpha": alpha, "selected_beta": beta} conditions.append(cond) for seed in SEEDS: ok, meta = verify_pairing(ds, alpha, seed, beta) if not ok: raise SystemExit(f"pairing failed for {ds} a{alpha} s{seed}") odir = run_dir(ds, alpha, seed, "original_protean") sdir = run_dir(ds, alpha, seed, "support_weighted", beta) og = read_json(odir / "test_global_metrics.json") sg = read_json(sdir / "test_global_metrics.json") for m in GLOBAL_METRICS: global_rows.append({ "dataset": ds, "alpha": alpha, "seed": seed, "selected_beta": beta, "metric": m, "original": og[m], "proposed": sg[m], "delta": sg[m] - og[m]}) opc = {int(r["client_id"]): r for r in read_csv(odir / "per_client.csv")} spc = {int(r["client_id"]): r for r in read_csv(sdir / "per_client.csv")} assert set(opc) == set(spc), "client id mismatch" for cid in sorted(opc): o, s = opc[cid], spc[cid] dr = float(s["rare_class_macro_f1"]) - float(o["rare_class_macro_f1"]) dm = float(s["macro_f1"]) - float(o["macro_f1"]) da = float(s["accuracy"]) - float(o["accuracy"]) dre = float(s["rare_class_recall"]) - float(o["rare_class_recall"]) status = ("unchanged" if abs(dr) <= TOL else ("improved" if dr > 0 else "degraded")) client_rows.append({ "dataset": ds, "alpha": alpha, "seed": seed, "selected_beta": beta, "client_id": cid, "n_samples": int(o["n_samples"]), "orig_rare_f1": o["rare_class_macro_f1"], "prop_rare_f1": s["rare_class_macro_f1"], "delta_rare_f1": dr, "orig_rare_recall": o["rare_class_recall"], "prop_rare_recall": s["rare_class_recall"], "delta_rare_recall": dre, "orig_macro_f1": o["macro_f1"], "prop_macro_f1": s["macro_f1"], "delta_macro_f1": dm, "orig_accuracy": o["accuracy"], "prop_accuracy": s["accuracy"], "delta_accuracy": da, "status": status}) opk = {int(r["class_id"]): r for r in read_csv(odir / "test_per_class.csv")} spk = {int(r["class_id"]): r for r in read_csv(sdir / "test_per_class.csv")} for cl in sorted(opk): o, s = opk[cl], spk[cl] class_rows.append({ "dataset": ds, "alpha": alpha, "seed": seed, "class_id": cl, "class_name": o["class_name"], "support": int(o["support"]), "orig_precision": o["precision"], "prop_precision": s["precision"], "delta_precision": float(s["precision"]) - float(o["precision"]), "orig_recall": o["recall"], "prop_recall": s["recall"], "delta_recall": float(s["recall"]) - float(o["recall"]), "orig_f1": o["f1"], "prop_f1": s["f1"], "delta_f1": float(s["f1"]) - float(o["f1"])}) ocm = read_json(odir / "test_confusion_matrix.json") scm = read_json(sdir / "test_confusion_matrix.json") names = meta["class_names"] for i, t in enumerate(names): row_o = sum(ocm[i]) row_s = sum(scm[i]) for j, p in enumerate(names): cm_rows.append({ "dataset": ds, "alpha": alpha, "seed": seed, "true_class": t, "pred_class": p, "orig_count": ocm[i][j], "prop_count": scm[i][j], "delta": scm[i][j] - ocm[i][j], "orig_row_rate": (ocm[i][j] / row_o) if row_o else 0.0, "prop_row_rate": (scm[i][j] / row_s) if row_s else 0.0, "delta_row_rate": ((scm[i][j] / row_s) - (ocm[i][j] / row_o)) if row_o and row_s else 0.0}) _write(ANALYSIS / "deepcmp__global_per_seed.csv", global_rows) _write(ANALYSIS / "deepcmp__per_client.csv", client_rows) _write(ANALYSIS / "deepcmp__per_class.csv", class_rows) _write(ANALYSIS / "deepcmp__confusion_delta.csv", cm_rows) # Summaries ------------------------------------------------------------- summary = [] for c in conditions: ds, alpha, beta = c["dataset"], c["alpha"], c["selected_beta"] for m in GLOBAL_METRICS: rows = [r for r in global_rows if r["dataset"] == ds and r["alpha"] == alpha and r["metric"] == m] orig = [float(r["original"]) for r in rows] prop = [float(r["proposed"]) for r in rows] dlt = [float(r["delta"]) for r in rows] summary.append({ "dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m, "orig_mean": _mean(orig), "orig_std": _std(orig), "prop_mean": _mean(prop), "prop_std": _std(prop), "delta_mean": _mean(dlt), "delta_std": _std(dlt), "per_seed_deltas": "|".join(f"{d:.6f}" for d in dlt), "n": len(rows)}) _write(ANALYSIS / "deepcmp__global_summary.csv", summary) for c in conditions: ds, alpha = c["dataset"], c["alpha"] rows = [r for r in client_rows if r["dataset"] == ds and r["alpha"] == alpha] dlt = [float(r["delta_rare_f1"]) for r in rows] st = [r["status"] for r in rows] c["improved"] = st.count("improved") c["degraded"] = st.count("degraded") c["unchanged"] = st.count("unchanged") c["total"] = len(rows) c["mean_delta"] = _mean(dlt) c["median_delta"] = _median(dlt) c["min_delta"] = min(dlt) c["max_delta"] = max(dlt) _write(ANALYSIS / "deepcmp__per_client_by_condition.csv", [ {k: c[k] for k in ("dataset", "alpha", "selected_beta", "total", "improved", "degraded", "unchanged", "mean_delta", "median_delta", "min_delta", "max_delta")} for c in conditions]) print("=== GLOBAL SUMMARY (delta = proposed - original) ===") for r in summary: print(f" {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} " f"{r['metric']:20} orig={float(r['orig_mean']):.4f} " f"prop={float(r['prop_mean']):.4f} delta={float(r['delta_mean']):+.4f} " f"per-seed=[{r['per_seed_deltas']}]") print("=== PER-CLIENT COUNTS (rare-class F1) ===") for c in conditions: print(f" {c['dataset']:8} a{c['alpha']:<4} beta={c['selected_beta']:<4} " f"improved={c['improved']} degraded={c['degraded']} " f"unchanged={c['unchanged']} (of {c['total']}) " f"meanDelta={c['mean_delta']:+.4f} min={c['min_delta']:+.4f} " f"max={c['max_delta']:+.4f}") print("wrote deepcmp__* derived tables to", ANALYSIS) def _mean(v): return sum(v) / len(v) if v else 0.0 def _std(v): if len(v) < 2: return 0.0 m = _mean(v) return (sum((x - m) ** 2 for x in v) / (len(v) - 1)) ** 0.5 def _median(v): s = sorted(v) n = len(s) return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2 def _write(path: Path, rows: list[dict]) -> None: with path.open("w", newline="") as fh: w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) w.writeheader() w.writerows(rows) if __name__ == "__main__": main()