willtanoe/thesis-prototype-agregation
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: add paired original vs proposed PROTEAN analysis

Browse Source
This commit is contained in:
willtanoe committed 2026-10-09 13:40:20 +07:00
1 parent 87f9b14ada
commit ad8b2d7671
17 files changed
+2960 -7
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/deep_comparison.py
+239
View File
@@ -0,0 +1,239 @@
#!/usr/bin/env python3
"""Deep paired comparison: original_protean vs support_weighted(selected beta).
Reads only completed experimental artifacts under ``results/`` and
``artifacts/analysis/`` and writes derived comparison tables under
``artifacts/analysis/``. No training, no result mutation.
For every (dataset, alpha, seed) the ``original_protean`` run and the
``support_weighted`` run at the validation-selected beta are matched by
dataset, alpha, seed, and manifest content hash, and compared on test
metrics (global, per-client, per-class, confusion matrices).
Run:
.venv/bin/python scripts/deep_comparison.py
"""
from __future__ import annotations
import csv
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
RESULTS = ROOT / "results"
ANALYSIS = ROOT / "artifacts" / "analysis"
DATASETS = ["xiiotid", "5gnidd"]
ALPHAS = [0.25, 0.50, 0.75]
SEEDS = [0, 1, 2]
# Validation-selected beta per condition (source: final__selection_strength.csv).
SELECTED = {("xiiotid", 0.25): 0.0, ("xiiotid", 0.5): 0.75, ("xiiotid", 0.75): 0.5,
("5gnidd", 0.25): 0.0, ("5gnidd", 0.5): 1.0, ("5gnidd", 0.75): 0.5}
GLOBAL_METRICS = ["rare_class_macro_f1", "rare_class_recall", "macro_f1",
"weighted_f1", "accuracy"]
# "Effectively unchanged" band for paired per-client deltas. Chosen to reflect
# the observed original_protean vs support_weighted(beta=0) residual divergence
# (the beta=0 conditions are only semantically equivalent to the original under
# the shared S_j rule, so their per-client residuals are numerical, not a
# weighting effect). Reported alongside the raw deltas so a reader can apply a
# different threshold.
TOL = 0.02
def run_dir(ds: str, alpha: float, seed: int, mode: str, beta: float | None = None) -> Path:
if mode == "original_protean":
return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__original_protean"
return RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}"
def read_json(p: Path):
return json.loads(p.read_text())
def read_csv(p: Path) -> list[dict]:
with p.open() as fh:
return list(csv.DictReader(fh))
def verify_pairing(ds, alpha, seed, beta) -> tuple[bool, dict]:
o = read_json(run_dir(ds, alpha, seed, "original_protean") / "metadata.json")
s = read_json(run_dir(ds, alpha, seed, "support_weighted", beta) / "metadata.json")
ok = (o["dataset"] == s["dataset"] == ds
and abs(float(o["alpha"]) - alpha) < 1e-9
and abs(float(s["alpha"]) - alpha) < 1e-9
and int(o["seed"]) == int(s["seed"]) == seed
and o["manifest_content_hash"] == s["manifest_content_hash"]
and abs(float(s["beta"]) - beta) < 1e-9)
return ok, o
def main() -> None:
ANALYSIS.mkdir(parents=True, exist_ok=True)
global_rows, client_rows, class_rows, cm_rows = [], [], [], []
conditions = []
for ds in DATASETS:
for alpha in ALPHAS:
beta = SELECTED[(ds, alpha)]
cond = {"dataset": ds, "alpha": alpha, "selected_beta": beta}
conditions.append(cond)
for seed in SEEDS:
ok, meta = verify_pairing(ds, alpha, seed, beta)
if not ok:
raise SystemExit(f"pairing failed for {ds} a{alpha} s{seed}")
odir = run_dir(ds, alpha, seed, "original_protean")
sdir = run_dir(ds, alpha, seed, "support_weighted", beta)
og = read_json(odir / "test_global_metrics.json")
sg = read_json(sdir / "test_global_metrics.json")
for m in GLOBAL_METRICS:
global_rows.append({
"dataset": ds, "alpha": alpha, "seed": seed,
"selected_beta": beta, "metric": m,
"original": og[m], "proposed": sg[m],
"delta": sg[m] - og[m]})
opc = {int(r["client_id"]): r for r in read_csv(odir / "per_client.csv")}
spc = {int(r["client_id"]): r for r in read_csv(sdir / "per_client.csv")}
assert set(opc) == set(spc), "client id mismatch"
for cid in sorted(opc):
o, s = opc[cid], spc[cid]
dr = float(s["rare_class_macro_f1"]) - float(o["rare_class_macro_f1"])
dm = float(s["macro_f1"]) - float(o["macro_f1"])
da = float(s["accuracy"]) - float(o["accuracy"])
dre = float(s["rare_class_recall"]) - float(o["rare_class_recall"])
status = ("unchanged" if abs(dr) <= TOL
else ("improved" if dr > 0 else "degraded"))
client_rows.append({
"dataset": ds, "alpha": alpha, "seed": seed,
"selected_beta": beta, "client_id": cid,
"n_samples": int(o["n_samples"]),
"orig_rare_f1": o["rare_class_macro_f1"],
"prop_rare_f1": s["rare_class_macro_f1"],
"delta_rare_f1": dr,
"orig_rare_recall": o["rare_class_recall"],
"prop_rare_recall": s["rare_class_recall"],
"delta_rare_recall": dre,
"orig_macro_f1": o["macro_f1"], "prop_macro_f1": s["macro_f1"],
"delta_macro_f1": dm,
"orig_accuracy": o["accuracy"], "prop_accuracy": s["accuracy"],
"delta_accuracy": da,
"status": status})
opk = {int(r["class_id"]): r for r in read_csv(odir / "test_per_class.csv")}
spk = {int(r["class_id"]): r for r in read_csv(sdir / "test_per_class.csv")}
for cl in sorted(opk):
o, s = opk[cl], spk[cl]
class_rows.append({
"dataset": ds, "alpha": alpha, "seed": seed,
"class_id": cl, "class_name": o["class_name"],
"support": int(o["support"]),
"orig_precision": o["precision"], "prop_precision": s["precision"],
"delta_precision": float(s["precision"]) - float(o["precision"]),
"orig_recall": o["recall"], "prop_recall": s["recall"],
"delta_recall": float(s["recall"]) - float(o["recall"]),
"orig_f1": o["f1"], "prop_f1": s["f1"],
"delta_f1": float(s["f1"]) - float(o["f1"])})
ocm = read_json(odir / "test_confusion_matrix.json")
scm = read_json(sdir / "test_confusion_matrix.json")
names = meta["class_names"]
for i, t in enumerate(names):
row_o = sum(ocm[i])
row_s = sum(scm[i])
for j, p in enumerate(names):
cm_rows.append({
"dataset": ds, "alpha": alpha, "seed": seed,
"true_class": t, "pred_class": p,
"orig_count": ocm[i][j], "prop_count": scm[i][j],
"delta": scm[i][j] - ocm[i][j],
"orig_row_rate": (ocm[i][j] / row_o) if row_o else 0.0,
"prop_row_rate": (scm[i][j] / row_s) if row_s else 0.0,
"delta_row_rate": ((scm[i][j] / row_s) - (ocm[i][j] / row_o))
if row_o and row_s else 0.0})
_write(ANALYSIS / "deepcmp__global_per_seed.csv", global_rows)
_write(ANALYSIS / "deepcmp__per_client.csv", client_rows)
_write(ANALYSIS / "deepcmp__per_class.csv", class_rows)
_write(ANALYSIS / "deepcmp__confusion_delta.csv", cm_rows)
# Summaries -------------------------------------------------------------
summary = []
for c in conditions:
ds, alpha, beta = c["dataset"], c["alpha"], c["selected_beta"]
for m in GLOBAL_METRICS:
rows = [r for r in global_rows
if r["dataset"] == ds and r["alpha"] == alpha and r["metric"] == m]
orig = [float(r["original"]) for r in rows]
prop = [float(r["proposed"]) for r in rows]
dlt = [float(r["delta"]) for r in rows]
summary.append({
"dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m,
"orig_mean": _mean(orig), "orig_std": _std(orig),
"prop_mean": _mean(prop), "prop_std": _std(prop),
"delta_mean": _mean(dlt), "delta_std": _std(dlt),
"per_seed_deltas": "|".join(f"{d:.6f}" for d in dlt), "n": len(rows)})
_write(ANALYSIS / "deepcmp__global_summary.csv", summary)
for c in conditions:
ds, alpha = c["dataset"], c["alpha"]
rows = [r for r in client_rows if r["dataset"] == ds and r["alpha"] == alpha]
dlt = [float(r["delta_rare_f1"]) for r in rows]
st = [r["status"] for r in rows]
c["improved"] = st.count("improved")
c["degraded"] = st.count("degraded")
c["unchanged"] = st.count("unchanged")
c["total"] = len(rows)
c["mean_delta"] = _mean(dlt)
c["median_delta"] = _median(dlt)
c["min_delta"] = min(dlt)
c["max_delta"] = max(dlt)
_write(ANALYSIS / "deepcmp__per_client_by_condition.csv", [
{k: c[k] for k in ("dataset", "alpha", "selected_beta", "total",
"improved", "degraded", "unchanged",
"mean_delta", "median_delta", "min_delta", "max_delta")}
for c in conditions])
print("=== GLOBAL SUMMARY (delta = proposed - original) ===")
for r in summary:
print(f" {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} "
f"{r['metric']:20} orig={float(r['orig_mean']):.4f} "
f"prop={float(r['prop_mean']):.4f} delta={float(r['delta_mean']):+.4f} "
f"per-seed=[{r['per_seed_deltas']}]")
print("=== PER-CLIENT COUNTS (rare-class F1) ===")
for c in conditions:
print(f" {c['dataset']:8} a{c['alpha']:<4} beta={c['selected_beta']:<4} "
f"improved={c['improved']} degraded={c['degraded']} "
f"unchanged={c['unchanged']} (of {c['total']}) "
f"meanDelta={c['mean_delta']:+.4f} min={c['min_delta']:+.4f} "
f"max={c['max_delta']:+.4f}")
print("wrote deepcmp__* derived tables to", ANALYSIS)
def _mean(v):
return sum(v) / len(v) if v else 0.0
def _std(v):
if len(v) < 2:
return 0.0
m = _mean(v)
return (sum((x - m) ** 2 for x in v) / (len(v) - 1)) ** 0.5
def _median(v):
s = sorted(v)
n = len(s)
return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
def _write(path: Path, rows: list[dict]) -> None:
with path.open("w", newline="") as fh:
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
if __name__ == "__main__":
main()
scripts/make_comparison_figures.py
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""Figures I, J, K: original_protean vs support_weighted(selected beta).
Figure I paired per-client rare-class F1 deltas (test), vs original_protean.
Figure J per-class test F1 change, beta>0 conditions.
Figure K test confusion-matrix difference (5G-NIDD, alpha=0.75, seed=0).
Reads derived tables written by ``scripts/deep_comparison.py`` and the raw
per-run confusion matrices. Writes PNG (220 dpi) and PDF to
``artifacts/figures/final``. No training, no result mutation.
Run:
.venv/bin/python scripts/make_comparison_figures.py
"""
from __future__ import annotations
import csv
import json
from collections import defaultdict
from pathlib import Path
import statistics
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.colors import TwoSlopeNorm
ROOT = Path(__file__).resolve().parents[1]
ANALYSIS = ROOT / "artifacts" / "analysis"
RESULTS = ROOT / "results"
OUT = ROOT / "artifacts" / "figures" / "final"
CHARCOAL = "#3A3A3A"
TEAL = "#0E7C7B"
RED = "#C0392B"
GREY = "#9A9A9A"
DATASETS = ["xiiotid", "5gnidd"]
DATASET_NAME = {"xiiotid": "X-IIoTID", "5gnidd": "5G-NIDD"}
ALPHAS = [0.25, 0.50, 0.75]
TOL = 0.02
SEED_MARKERS = {0: "o", 1: "s", 2: "^"}
def _style():
plt.rcParams.update({
"figure.facecolor": "white", "axes.facecolor": "white",
"axes.edgecolor": "#5A5A5A", "axes.linewidth": 0.9,
"axes.titlesize": 11.5, "axes.titleweight": "bold",
"axes.labelsize": 10.5, "xtick.labelsize": 9,
"ytick.labelsize": 9, "legend.fontsize": 9,
"font.size": 10, "axes.spines.top": False, "axes.spines.right": False,
"grid.color": "#B0B0B0", "grid.alpha": 0.3, "grid.linestyle": ":",
"savefig.dpi": 220, "savefig.bbox": "tight",
})
def _save(fig, name):
OUT.mkdir(parents=True, exist_ok=True)
for ext in ("png", "pdf"):
fig.savefig(OUT / f"{name}.{ext}")
plt.close(fig)
print("wrote", OUT / f"{name}.png")
def _read(path):
with path.open() as fh:
return list(csv.DictReader(fh))
# ---------------------------------------------------------------------------
def figure_i():
rows = _read(ANALYSIS / "deepcmp__per_client.csv")
fig, axes = plt.subplots(2, 3, figsize=(14.0, 7.4), constrained_layout=True,
sharey=True)
ylim = 0.47
for ri, ds in enumerate(DATASETS):
for ci, alpha in enumerate(ALPHAS):
ax = axes[ri][ci]
ax.set_axisbelow(True)
ax.grid(axis="y")
for seed in (0, 1, 2):
sub = [r for r in rows if r["dataset"] == ds
and abs(float(r["alpha"]) - alpha) < 1e-9
and int(r["seed"]) == seed]
xs = [int(r["client_id"]) for r in sub]
ys = [float(r["delta_rare_f1"]) for r in sub]
cols = [TEAL if y > TOL else (RED if y < -TOL else GREY) for y in ys]
ax.scatter(xs, ys, s=48, marker=SEED_MARKERS[seed], c=cols,
edgecolor="#333333", linewidth=0.5, alpha=0.9, zorder=4)
ax.axhspan(-TOL, TOL, color="#DDDDDD", alpha=0.35, zorder=1)
ax.axhline(0.0, color="#555555", lw=1.0, zorder=2)
beta = {"xiiotid": {0.25: 0.0, 0.5: 0.75, 0.75: 0.5},
"5gnidd": {0.25: 0.0, 0.5: 1.0, 0.75: 0.5}}[ds][alpha]
deltas = [float(r["delta_rare_f1"]) for r in rows
if r["dataset"] == ds and abs(float(r["alpha"]) - alpha) < 1e-9]
ax.set_ylim(-ylim, ylim)
ax.set_xlim(-0.7, 9.7)
ax.set_xticks(range(10))
ax.set_title(f"{DATASET_NAME[ds]} \u03b1={alpha:.2f} "
f"(beta={beta:g})")
ax.text(0.98, 0.04, f"mean \u0394={statistics.mean(deltas):+.3f}",
transform=ax.transAxes, ha="right", va="bottom",
fontsize=8.5, color="#333333")
if ri == 1:
ax.set_xlabel("client id (per-seed partition)")
if ci == 0:
ax.set_ylabel("test rare-class macro-F1\n\u0394 (proposed \u2212 original)")
handles = [plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc=TEAL, mec="#333", label="improved (> +0.02)"),
plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc=RED, mec="#333", label="degraded (< -0.02)"),
plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc=GREY, mec="#333", label="unchanged (|\u0394| \u2264 0.02)"),
plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc="white", mec="#333", label="seed 0"),
plt.Line2D([0], [0], marker="s", ls="", ms=7, mfc="white", mec="#333", label="seed 1"),
plt.Line2D([0], [0], marker="^", ls="", ms=7, mfc="white", mec="#333", label="seed 2")]
fig.legend(handles=handles, loc="upper center", bbox_to_anchor=(0.5, 1.05),
ncols=6, frameon=True, edgecolor="#CCCCCC")
fig.suptitle("Paired per-client test rare-class macro-F1: proposed (selected beta) vs ORIGINAL PROTEAN",
fontsize=12.5, weight="bold", y=1.11)
fig.text(0.5, -0.03,
"Each point is one (client, seed) test observation; x = client id, marker = seed. Points compare the "
"support_weighted run at the validation-selected beta against original_protean on the same partition. "
"At \u03b1=0.25 the selected beta is 0, so the residual spread is numerical (equal-weight equivalence), not a weighting effect. "
"Do not treat client id across seeds as the same real-world client.",
ha="center", va="top", fontsize=8.4, color="#444444", linespacing=1.5)
_save(fig, "figI_per_client_original_vs_proposed")
# ---------------------------------------------------------------------------
def figure_j():
rows = _read(ANALYSIS / "deepcmp__per_class.csv")
fig, axes = plt.subplots(2, 2, figsize=(13.4, 8.2), constrained_layout=True)
for ri, ds in enumerate(DATASETS):
for ci, alpha in enumerate([0.5, 0.75]):
ax = axes[ri][ci]
ax.set_axisbelow(True)
ax.grid(axis="x")
stats = defaultdict(lambda: {"f1": [], "sup": []})
for r in rows:
if r["dataset"] == ds and abs(float(r["alpha"]) - alpha) < 1e-9:
stats[r["class_name"]]["f1"].append(float(r["delta_f1"]))
stats[r["class_name"]]["sup"].append(int(r["support"]))
names = sorted(stats, key=lambda n: statistics.mean(stats[n]["f1"]))
ys = np.arange(len(names))
vals = [statistics.mean(stats[n]["f1"]) for n in names]
sup = [int(statistics.mean(stats[n]["sup"])) for n in names]
cols = [TEAL if v >= 0 else RED for v in vals]
ax.barh(ys, vals, color=cols, edgecolor="#333333", linewidth=0.6, zorder=3)
ax.axvline(0.0, color="#555555", lw=1.0, zorder=2)
ax.set_yticks(ys)
ax.set_yticklabels([f"{n} (n\u2248{s:,})" for n, s in zip(names, sup)],
fontsize=8.2)
beta = {"xiiotid": {0.5: 0.75, 0.75: 0.5},
"5gnidd": {0.5: 1.0, 0.75: 0.5}}[ds][alpha]
ax.set_title(f"{DATASET_NAME[ds]} \u03b1={alpha:.2f} beta={beta:g}")
ax.set_xlabel("\u0394 F1 (proposed \u2212 original), test")
fig.suptitle("Per-class test F1 change for beta>0 conditions (mean over seeds {0,1,2})",
fontsize=12.5, weight="bold", y=1.04)
fig.text(0.5, -0.015,
"Only beta>0 conditions are shown; \u03b1=0.25 selected beta=0 so it carries no weighting effect. "
"Class support n\u2248 is the mean test support over seeds. Teal = F1 increased, red = F1 decreased. "
"X-IIoTID and 5G-NIDD class labels are never mixed. No significance testing (3 seeds).",
ha="center", va="top", fontsize=8.4, color="#444444", linespacing=1.5)
_save(fig, "figJ_per_class_original_vs_proposed")
# ---------------------------------------------------------------------------
def figure_k(dataset="5gnidd", alpha=0.75, seed=0, beta=0.5):
odir = RESULTS / f"{dataset}__alpha{alpha:g}__seed{seed}__original_protean"
sdir = RESULTS / f"{dataset}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}"
names = json.loads((odir / "metadata.json").read_text())["class_names"]
o = np.array(json.loads((odir / "test_confusion_matrix.json").read_text()), float)
s = np.array(json.loads((sdir / "test_confusion_matrix.json").read_text()), float)
diff = s - o
orow = np.where(o.sum(1, keepdims=True) == 0, 1, o.sum(1, keepdims=True))
srow = np.where(s.sum(1, keepdims=True) == 0, 1, s.sum(1, keepdims=True))
diff_rate = s / srow - o / orow
fig, axes = plt.subplots(1, 2, figsize=(14.0, 6.4), constrained_layout=True)
vmax_c = np.abs(diff).max()
vmax_r = np.abs(diff_rate).max() * 100
im0 = axes[0].imshow(diff, cmap="RdBu_r",
norm=TwoSlopeNorm(vcenter=0, vmin=-vmax_c, vmax=vmax_c))
im1 = axes[1].imshow(diff_rate * 100, cmap="RdBu_r",
norm=TwoSlopeNorm(vcenter=0, vmin=-vmax_r, vmax=vmax_r))
for ax in axes:
ax.set_xticks(range(len(names)))
ax.set_yticks(range(len(names)))
ax.set_xticklabels(names, rotation=45, ha="right", fontsize=8)
ax.set_yticklabels(names, fontsize=8)
ax.set_xlabel("predicted class")
axes[0].set_ylabel("true class")
axes[0].set_title("A. Difference in raw counts\n(proposed \u2212 original)")
axes[1].set_title("B. Difference in row-normalized rate\n(percentage points, proposed \u2212 original)")
# annotate the strongest cells on each panel
for ax, M, is_rate in ((axes[0], diff, False), (axes[1], diff_rate * 100, True)):
flat = [(abs(M[i, j]), i, j) for i in range(9) for j in range(9) if i != j]
for _, i, j in sorted(flat, reverse=True)[:6]:
v = M[i, j]
ax.text(j, i, f"{v:+.0f}" if not is_rate else f"{v:+.1f}",
ha="center", va="center", fontsize=6.4, fontweight="bold",
color="white" if abs(v) > (vmax_c if not is_rate else vmax_r) * 0.5
else "#222222")
cb0 = fig.colorbar(im0, ax=axes[0], fraction=0.046, pad=0.02)
cb0.set_label("count delta")
cb1 = fig.colorbar(im1, ax=axes[1], fraction=0.046, pad=0.02)
cb1.set_label("row-rate delta (pp)")
dname = DATASET_NAME[dataset]
fig.suptitle(f"Confusion-matrix change: proposed (beta={beta:g}) minus original PROTEAN - "
f"{dname}, alpha={alpha:g}, seed={seed}, test",
fontsize=12, weight="bold", y=1.05)
fig.text(0.5, -0.02,
"Rows = true class, columns = predicted class; identical class order and sample counts (243,177) for both runs. "
"Diagonal: positive = more correct predictions; off-diagonal: positive = more errors of that (true\u2192predicted) type. "
"Red = positive delta (proposed has MORE samples in the cell); blue = negative delta (fewer). Values are descriptive only; no significance is inferred.",
ha="center", va="top", fontsize=8.4, color="#444444", linespacing=1.5)
_save(fig, f"figK_confusion_delta_{dataset}_a{alpha:g}_seed{seed}")
def main():
_style()
figure_i()
figure_j()
figure_k()
if __name__ == "__main__":
main()