240 lines
11 KiB
Python
240 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Deep paired comparison: original_protean vs support_weighted(selected beta).
|
|
|
|
Reads only completed experimental artifacts under ``results/`` and
|
|
``artifacts/analysis/`` and writes derived comparison tables under
|
|
``artifacts/analysis/``. No training, no result mutation.
|
|
|
|
For every (dataset, alpha, seed) the ``original_protean`` run and the
|
|
``support_weighted`` run at the validation-selected beta are matched by
|
|
dataset, alpha, seed, and manifest content hash, and compared on test
|
|
metrics (global, per-client, per-class, confusion matrices).
|
|
|
|
Run:
|
|
.venv/bin/python scripts/deep_comparison.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
RESULTS = ROOT / "results"
|
|
ANALYSIS = ROOT / "artifacts" / "analysis"
|
|
|
|
DATASETS = ["xiiotid", "5gnidd"]
|
|
ALPHAS = [0.25, 0.50, 0.75]
|
|
SEEDS = [0, 1, 2]
|
|
# Validation-selected beta per condition (source: final__selection_strength.csv).
|
|
SELECTED = {("xiiotid", 0.25): 0.0, ("xiiotid", 0.5): 0.75, ("xiiotid", 0.75): 0.5,
|
|
("5gnidd", 0.25): 0.0, ("5gnidd", 0.5): 1.0, ("5gnidd", 0.75): 0.5}
|
|
GLOBAL_METRICS = ["rare_class_macro_f1", "rare_class_recall", "macro_f1",
|
|
"weighted_f1", "accuracy"]
|
|
# "Effectively unchanged" band for paired per-client deltas. Chosen to reflect
|
|
# the observed original_protean vs support_weighted(beta=0) residual divergence
|
|
# (the beta=0 conditions are only semantically equivalent to the original under
|
|
# the shared S_j rule, so their per-client residuals are numerical, not a
|
|
# weighting effect). Reported alongside the raw deltas so a reader can apply a
|
|
# different threshold.
|
|
TOL = 0.02
|
|
|
|
|
|
def run_dir(ds: str, alpha: float, seed: int, mode: str, beta: float | None = None) -> Path:
|
|
if mode == "original_protean":
|
|
return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__original_protean"
|
|
return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}"
|
|
|
|
|
|
def read_json(p: Path):
|
|
return json.loads(p.read_text())
|
|
|
|
|
|
def read_csv(p: Path) -> list[dict]:
|
|
with p.open() as fh:
|
|
return list(csv.DictReader(fh))
|
|
|
|
|
|
def verify_pairing(ds, alpha, seed, beta) -> tuple[bool, dict]:
|
|
o = read_json(run_dir(ds, alpha, seed, "original_protean") / "metadata.json")
|
|
s = read_json(run_dir(ds, alpha, seed, "support_weighted", beta) / "metadata.json")
|
|
ok = (o["dataset"] == s["dataset"] == ds
|
|
and abs(float(o["alpha"]) - alpha) < 1e-9
|
|
and abs(float(s["alpha"]) - alpha) < 1e-9
|
|
and int(o["seed"]) == int(s["seed"]) == seed
|
|
and o["manifest_content_hash"] == s["manifest_content_hash"]
|
|
and abs(float(s["beta"]) - beta) < 1e-9)
|
|
return ok, o
|
|
|
|
|
|
def main() -> None:
|
|
ANALYSIS.mkdir(parents=True, exist_ok=True)
|
|
global_rows, client_rows, class_rows, cm_rows = [], [], [], []
|
|
conditions = []
|
|
|
|
for ds in DATASETS:
|
|
for alpha in ALPHAS:
|
|
beta = SELECTED[(ds, alpha)]
|
|
cond = {"dataset": ds, "alpha": alpha, "selected_beta": beta}
|
|
conditions.append(cond)
|
|
for seed in SEEDS:
|
|
ok, meta = verify_pairing(ds, alpha, seed, beta)
|
|
if not ok:
|
|
raise SystemExit(f"pairing failed for {ds} a{alpha} s{seed}")
|
|
odir = run_dir(ds, alpha, seed, "original_protean")
|
|
sdir = run_dir(ds, alpha, seed, "support_weighted", beta)
|
|
|
|
og = read_json(odir / "test_global_metrics.json")
|
|
sg = read_json(sdir / "test_global_metrics.json")
|
|
for m in GLOBAL_METRICS:
|
|
global_rows.append({
|
|
"dataset": ds, "alpha": alpha, "seed": seed,
|
|
"selected_beta": beta, "metric": m,
|
|
"original": og[m], "proposed": sg[m],
|
|
"delta": sg[m] - og[m]})
|
|
|
|
opc = {int(r["client_id"]): r for r in read_csv(odir / "per_client.csv")}
|
|
spc = {int(r["client_id"]): r for r in read_csv(sdir / "per_client.csv")}
|
|
assert set(opc) == set(spc), "client id mismatch"
|
|
for cid in sorted(opc):
|
|
o, s = opc[cid], spc[cid]
|
|
dr = float(s["rare_class_macro_f1"]) - float(o["rare_class_macro_f1"])
|
|
dm = float(s["macro_f1"]) - float(o["macro_f1"])
|
|
da = float(s["accuracy"]) - float(o["accuracy"])
|
|
dre = float(s["rare_class_recall"]) - float(o["rare_class_recall"])
|
|
status = ("unchanged" if abs(dr) <= TOL
|
|
else ("improved" if dr > 0 else "degraded"))
|
|
client_rows.append({
|
|
"dataset": ds, "alpha": alpha, "seed": seed,
|
|
"selected_beta": beta, "client_id": cid,
|
|
"n_samples": int(o["n_samples"]),
|
|
"orig_rare_f1": o["rare_class_macro_f1"],
|
|
"prop_rare_f1": s["rare_class_macro_f1"],
|
|
"delta_rare_f1": dr,
|
|
"orig_rare_recall": o["rare_class_recall"],
|
|
"prop_rare_recall": s["rare_class_recall"],
|
|
"delta_rare_recall": dre,
|
|
"orig_macro_f1": o["macro_f1"], "prop_macro_f1": s["macro_f1"],
|
|
"delta_macro_f1": dm,
|
|
"orig_accuracy": o["accuracy"], "prop_accuracy": s["accuracy"],
|
|
"delta_accuracy": da,
|
|
"status": status})
|
|
|
|
opk = {int(r["class_id"]): r for r in read_csv(odir / "test_per_class.csv")}
|
|
spk = {int(r["class_id"]): r for r in read_csv(sdir / "test_per_class.csv")}
|
|
for cl in sorted(opk):
|
|
o, s = opk[cl], spk[cl]
|
|
class_rows.append({
|
|
"dataset": ds, "alpha": alpha, "seed": seed,
|
|
"class_id": cl, "class_name": o["class_name"],
|
|
"support": int(o["support"]),
|
|
"orig_precision": o["precision"], "prop_precision": s["precision"],
|
|
"delta_precision": float(s["precision"]) - float(o["precision"]),
|
|
"orig_recall": o["recall"], "prop_recall": s["recall"],
|
|
"delta_recall": float(s["recall"]) - float(o["recall"]),
|
|
"orig_f1": o["f1"], "prop_f1": s["f1"],
|
|
"delta_f1": float(s["f1"]) - float(o["f1"])})
|
|
|
|
ocm = read_json(odir / "test_confusion_matrix.json")
|
|
scm = read_json(sdir / "test_confusion_matrix.json")
|
|
names = meta["class_names"]
|
|
for i, t in enumerate(names):
|
|
row_o = sum(ocm[i])
|
|
row_s = sum(scm[i])
|
|
for j, p in enumerate(names):
|
|
cm_rows.append({
|
|
"dataset": ds, "alpha": alpha, "seed": seed,
|
|
"true_class": t, "pred_class": p,
|
|
"orig_count": ocm[i][j], "prop_count": scm[i][j],
|
|
"delta": scm[i][j] - ocm[i][j],
|
|
"orig_row_rate": (ocm[i][j] / row_o) if row_o else 0.0,
|
|
"prop_row_rate": (scm[i][j] / row_s) if row_s else 0.0,
|
|
"delta_row_rate": ((scm[i][j] / row_s) - (ocm[i][j] / row_o))
|
|
if row_o and row_s else 0.0})
|
|
|
|
_write(ANALYSIS / "deepcmp__global_per_seed.csv", global_rows)
|
|
_write(ANALYSIS / "deepcmp__per_client.csv", client_rows)
|
|
_write(ANALYSIS / "deepcmp__per_class.csv", class_rows)
|
|
_write(ANALYSIS / "deepcmp__confusion_delta.csv", cm_rows)
|
|
|
|
# Summaries -------------------------------------------------------------
|
|
summary = []
|
|
for c in conditions:
|
|
ds, alpha, beta = c["dataset"], c["alpha"], c["selected_beta"]
|
|
for m in GLOBAL_METRICS:
|
|
rows = [r for r in global_rows
|
|
if r["dataset"] == ds and r["alpha"] == alpha and r["metric"] == m]
|
|
orig = [float(r["original"]) for r in rows]
|
|
prop = [float(r["proposed"]) for r in rows]
|
|
dlt = [float(r["delta"]) for r in rows]
|
|
summary.append({
|
|
"dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m,
|
|
"orig_mean": _mean(orig), "orig_std": _std(orig),
|
|
"prop_mean": _mean(prop), "prop_std": _std(prop),
|
|
"delta_mean": _mean(dlt), "delta_std": _std(dlt),
|
|
"per_seed_deltas": "|".join(f"{d:.6f}" for d in dlt), "n": len(rows)})
|
|
_write(ANALYSIS / "deepcmp__global_summary.csv", summary)
|
|
|
|
for c in conditions:
|
|
ds, alpha = c["dataset"], c["alpha"]
|
|
rows = [r for r in client_rows if r["dataset"] == ds and r["alpha"] == alpha]
|
|
dlt = [float(r["delta_rare_f1"]) for r in rows]
|
|
st = [r["status"] for r in rows]
|
|
c["improved"] = st.count("improved")
|
|
c["degraded"] = st.count("degraded")
|
|
c["unchanged"] = st.count("unchanged")
|
|
c["total"] = len(rows)
|
|
c["mean_delta"] = _mean(dlt)
|
|
c["median_delta"] = _median(dlt)
|
|
c["min_delta"] = min(dlt)
|
|
c["max_delta"] = max(dlt)
|
|
_write(ANALYSIS / "deepcmp__per_client_by_condition.csv", [
|
|
{k: c[k] for k in ("dataset", "alpha", "selected_beta", "total",
|
|
"improved", "degraded", "unchanged",
|
|
"mean_delta", "median_delta", "min_delta", "max_delta")}
|
|
for c in conditions])
|
|
|
|
print("=== GLOBAL SUMMARY (delta = proposed - original) ===")
|
|
for r in summary:
|
|
print(f" {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} "
|
|
f"{r['metric']:20} orig={float(r['orig_mean']):.4f} "
|
|
f"prop={float(r['prop_mean']):.4f} delta={float(r['delta_mean']):+.4f} "
|
|
f"per-seed=[{r['per_seed_deltas']}]")
|
|
print("=== PER-CLIENT COUNTS (rare-class F1) ===")
|
|
for c in conditions:
|
|
print(f" {c['dataset']:8} a{c['alpha']:<4} beta={c['selected_beta']:<4} "
|
|
f"improved={c['improved']} degraded={c['degraded']} "
|
|
f"unchanged={c['unchanged']} (of {c['total']}) "
|
|
f"meanDelta={c['mean_delta']:+.4f} min={c['min_delta']:+.4f} "
|
|
f"max={c['max_delta']:+.4f}")
|
|
print("wrote deepcmp__* derived tables to", ANALYSIS)
|
|
|
|
|
|
def _mean(v):
|
|
return sum(v) / len(v) if v else 0.0
|
|
|
|
|
|
def _std(v):
|
|
if len(v) < 2:
|
|
return 0.0
|
|
m = _mean(v)
|
|
return (sum((x - m) ** 2 for x in v) / (len(v) - 1)) ** 0.5
|
|
|
|
|
|
def _median(v):
|
|
s = sorted(v)
|
|
n = len(s)
|
|
return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
|
|
|
|
|
|
def _write(path: Path, rows: list[dict]) -> None:
|
|
with path.open("w", newline="") as fh:
|
|
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|