willtanoe/thesis-prototype-agregation
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: improve readability of PROTEAN comparison figures

Browse Source
This commit is contained in:
willtanoe committed 2026-10-09 14:19:33 +07:00
1 parent ad8b2d7671
commit 34bab45865
22 files changed
+398 -212
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/make_comparison_figures.py
+199 -123
View File
@@ -1,13 +1,16 @@
#!/usr/bin/env python3
"""Figures I, J, K: original_protean vs support_weighted(selected beta).
Figure I paired per-client rare-class F1 deltas (test), vs original_protean.
Figure J per-class test F1 change, beta>0 conditions.
Figure K test confusion-matrix difference (5G-NIDD, alpha=0.75, seed=0).
Readability overhaul. Produces:
Reads derived tables written by ``scripts/deep_comparison.py`` and the raw
per-run confusion matrices. Writes PNG (220 dpi) and PDF to
``artifacts/figures/final``. No training, no result mutation.
figI1 / figI2 paired per-client test rare-class F1 deltas, one figure per dataset
figJ1 / figJ2 per-class test F1 change, one figure per dataset (two alpha panels)
figK_confusion_error_changes_* ranked reduced/increased misclassifications (primary)
figK_confusion_delta_* full difference matrix (supplemental, preserved)
Reads derived tables from ``scripts/deep_comparison.py`` and raw confusion
matrices. Writes PNG (220 dpi) and PDF to ``artifacts/figures/final``.
No training, no result mutation.
Run:
.venv/bin/python scripts/make_comparison_figures.py
@@ -17,9 +20,9 @@ from __future__ import annotations
import csv
import json
import statistics
from collections import defaultdict
from pathlib import Path
import statistics
import matplotlib
@@ -35,11 +38,13 @@ OUT = ROOT / "artifacts" / "figures" / "final"
CHARCOAL = "#3A3A3A"
TEAL = "#0E7C7B"
GREEN = "#1B7F3B"
RED = "#C0392B"
GREY = "#9A9A9A"
DATASETS = ["xiiotid", "5gnidd"]
DATASET_NAME = {"xiiotid": "X-IIoTID", "5gnidd": "5G-NIDD"}
ALPHAS = [0.25, 0.50, 0.75]
SELECTED = {"xiiotid": {0.25: 0.0, 0.5: 0.75, 0.75: 0.5},
"5gnidd": {0.25: 0.0, 0.5: 1.0, 0.75: 0.5}}
TOL = 0.02
SEED_MARKERS = {0: "o", 1: "s", 2: "^"}
@@ -48,10 +53,10 @@ def _style():
plt.rcParams.update({
"figure.facecolor": "white", "axes.facecolor": "white",
"axes.edgecolor": "#5A5A5A", "axes.linewidth": 0.9,
"axes.titlesize": 11.5, "axes.titleweight": "bold",
"axes.labelsize": 10.5, "xtick.labelsize": 9,
"ytick.labelsize": 9, "legend.fontsize": 9,
"font.size": 10, "axes.spines.top": False, "axes.spines.right": False,
"axes.titlesize": 12, "axes.titleweight": "bold",
"axes.labelsize": 11, "xtick.labelsize": 10, "ytick.labelsize": 10,
"legend.fontsize": 9.5, "font.size": 10.5,
"axes.spines.top": False, "axes.spines.right": False,
"grid.color": "#B0B0B0", "grid.alpha": 0.3, "grid.linestyle": ":",
"savefig.dpi": 220, "savefig.bbox": "tight",
})
@@ -71,102 +76,175 @@ def _read(path):
# ---------------------------------------------------------------------------
def figure_i():
rows = _read(ANALYSIS / "deepcmp__per_client.csv")
fig, axes = plt.subplots(2, 3, figsize=(14.0, 7.4), constrained_layout=True,
sharey=True)
ylim = 0.47
for ri, ds in enumerate(DATASETS):
for ci, alpha in enumerate(ALPHAS):
ax = axes[ri][ci]
ax.set_axisbelow(True)
ax.grid(axis="y")
for seed in (0, 1, 2):
sub = [r for r in rows if r["dataset"] == ds
and abs(float(r["alpha"]) - alpha) < 1e-9
and int(r["seed"]) == seed]
xs = [int(r["client_id"]) for r in sub]
ys = [float(r["delta_rare_f1"]) for r in sub]
cols = [TEAL if y > TOL else (RED if y < -TOL else GREY) for y in ys]
ax.scatter(xs, ys, s=48, marker=SEED_MARKERS[seed], c=cols,
edgecolor="#333333", linewidth=0.5, alpha=0.9, zorder=4)
ax.axhspan(-TOL, TOL, color="#DDDDDD", alpha=0.35, zorder=1)
ax.axhline(0.0, color="#555555", lw=1.0, zorder=2)
beta = {"xiiotid": {0.25: 0.0, 0.5: 0.75, 0.75: 0.5},
"5gnidd": {0.25: 0.0, 0.5: 1.0, 0.75: 0.5}}[ds][alpha]
deltas = [float(r["delta_rare_f1"]) for r in rows
if r["dataset"] == ds and abs(float(r["alpha"]) - alpha) < 1e-9]
ax.set_ylim(-ylim, ylim)
ax.set_xlim(-0.7, 9.7)
ax.set_xticks(range(10))
ax.set_title(f"{DATASET_NAME[ds]} \u03b1={alpha:.2f} "
f"(beta={beta:g})")
ax.text(0.98, 0.04, f"mean \u0394={statistics.mean(deltas):+.3f}",
transform=ax.transAxes, ha="right", va="bottom",
fontsize=8.5, color="#333333")
if ri == 1:
ax.set_xlabel("client id (per-seed partition)")
if ci == 0:
ax.set_ylabel("test rare-class macro-F1\n\u0394 (proposed \u2212 original)")
handles = [plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc=TEAL, mec="#333", label="improved (> +0.02)"),
plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc=RED, mec="#333", label="degraded (< -0.02)"),
plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc=GREY, mec="#333", label="unchanged (|\u0394| \u2264 0.02)"),
plt.Line2D([0], [0], marker="o", ls="", ms=7, mfc="white", mec="#333", label="seed 0"),
plt.Line2D([0], [0], marker="s", ls="", ms=7, mfc="white", mec="#333", label="seed 1"),
plt.Line2D([0], [0], marker="^", ls="", ms=7, mfc="white", mec="#333", label="seed 2")]
# Figure I: paired per-client rare-class F1 vs original_protean (one per dataset)
# ---------------------------------------------------------------------------
def figure_i(dataset: str) -> None:
rows = [r for r in _read(ANALYSIS / "deepcmp__per_client.csv")
if r["dataset"] == dataset]
all_d = [float(r["delta_rare_f1"]) for r in rows]
ymax = max(abs(min(all_d)), abs(max(all_d))) * 1.12
fig, axes = plt.subplots(1, 3, figsize=(16.0, 5.6), sharey=True,
constrained_layout=True)
for ax, alpha in zip(axes, ALPHAS):
ax.set_axisbelow(True)
ax.grid(axis="y")
ax.axhspan(-TOL, TOL, color="#DDDDDD", alpha=0.4, zorder=1)
ax.axhline(0.0, color="#444444", lw=1.1, zorder=2)
sub = [r for r in rows if abs(float(r["alpha"]) - alpha) < 1e-9]
for seed in (0, 1, 2):
pts = [r for r in sub if int(r["seed"]) == seed]
xs = [int(r["client_id"]) + (seed - 1) * 0.18 for r in pts]
ys = [float(r["delta_rare_f1"]) for r in pts]
cols = [TEAL if y > TOL else (RED if y < -TOL else GREY) for y in ys]
ax.scatter(xs, ys, s=62, marker=SEED_MARKERS[seed], c=cols,
edgecolor="#333333", linewidth=0.6, zorder=4)
# annotate the strongest positive and negative observation
for r in (max(sub, key=lambda r: float(r["delta_rare_f1"])),
min(sub, key=lambda r: float(r["delta_rare_f1"]))):
y = float(r["delta_rare_f1"])
ax.annotate(f"c{r['client_id']} s{r['seed']}", (int(r["client_id"]), y),
textcoords="offset points", xytext=(6, 6 if y >= 0 else -12),
fontsize=8.2, color="#333333")
imp = sum(1 for r in sub if float(r["delta_rare_f1"]) > TOL)
deg = sum(1 for r in sub if float(r["delta_rare_f1"]) < -TOL)
unc = len(sub) - imp - deg
ax.text(0.5, 0.985, f"improved {imp} | degraded {deg} | unchanged {unc}",
transform=ax.transAxes, ha="center", va="top", fontsize=9,
bbox=dict(boxstyle="round", fc="white", ec="#CCCCCC"))
ax.set_ylim(-ymax, ymax)
ax.set_xlim(-0.75, 9.75)
ax.set_xticks(range(10))
ax.set_xlabel("client id (per-seed partition)")
beta = SELECTED[dataset][alpha]
ax.set_title(f"{DATASET_NAME[dataset]} \u03b1={alpha:.2f} (beta={beta:g})")
axes[0].set_ylabel("test rare-class macro-F1\n\u0394 (proposed \u2212 ORIGINAL PROTEAN)")
handles = [plt.Line2D([0], [0], marker="o", ls="", ms=8, mfc=TEAL, mec="#333", label=f"improved (> +{TOL})"),
plt.Line2D([0], [0], marker="o", ls="", ms=8, mfc=RED, mec="#333", label=f"degraded (< \u2212{TOL})"),
plt.Line2D([0], [0], marker="o", ls="", ms=8, mfc=GREY, mec="#333", label=f"unchanged (|\u0394| \u2264 {TOL})"),
plt.Line2D([0], [0], marker="o", ls="", ms=8, mfc="white", mec="#333", label="seed 0"),
plt.Line2D([0], [0], marker="s", ls="", ms=8, mfc="white", mec="#333", label="seed 1"),
plt.Line2D([0], [0], marker="^", ls="", ms=8, mfc="white", mec="#333", label="seed 2")]
fig.legend(handles=handles, loc="upper center", bbox_to_anchor=(0.5, 1.05),
ncols=6, frameon=True, edgecolor="#CCCCCC")
fig.suptitle("Paired per-client test rare-class macro-F1: proposed (selected beta) vs ORIGINAL PROTEAN",
fontsize=12.5, weight="bold", y=1.11)
fig.text(0.5, -0.03,
"Each point is one (client, seed) test observation; x = client id, marker = seed. Points compare the "
"support_weighted run at the validation-selected beta against original_protean on the same partition. "
"At \u03b1=0.25 the selected beta is 0, so the residual spread is numerical (equal-weight equivalence), not a weighting effect. "
"Do not treat client id across seeds as the same real-world client.",
ha="center", va="top", fontsize=8.4, color="#444444", linespacing=1.5)
_save(fig, "figI_per_client_original_vs_proposed")
fig.suptitle(f"Per-client test rare-class macro-F1: proposed vs ORIGINAL PROTEAN \u2014 {DATASET_NAME[dataset]}",
fontsize=13, weight="bold", y=1.11)
fig.text(0.5, -0.04,
"Each point is one (client, seed) test observation on the client's own evaluation partition; same partition for both runs. "
"The shaded band is the \u00b10.02 effectively-unchanged threshold. Client ids are per-seed realizations (id k is not the same entity across seeds). "
"At \u03b1=0.25 the selected beta is 0, so the residual spread is numerical, not a weighting effect.",
ha="center", va="top", fontsize=8.6, color="#444444", linespacing=1.5)
_save(fig, f"figI{'1' if dataset == 'xiiotid' else '2'}_{dataset}_per_client_original_vs_proposed")
# ---------------------------------------------------------------------------
def figure_j():
rows = _read(ANALYSIS / "deepcmp__per_class.csv")
fig, axes = plt.subplots(2, 2, figsize=(13.4, 8.2), constrained_layout=True)
for ri, ds in enumerate(DATASETS):
for ci, alpha in enumerate([0.5, 0.75]):
ax = axes[ri][ci]
ax.set_axisbelow(True)
ax.grid(axis="x")
stats = defaultdict(lambda: {"f1": [], "sup": []})
for r in rows:
if r["dataset"] == ds and abs(float(r["alpha"]) - alpha) < 1e-9:
stats[r["class_name"]]["f1"].append(float(r["delta_f1"]))
stats[r["class_name"]]["sup"].append(int(r["support"]))
names = sorted(stats, key=lambda n: statistics.mean(stats[n]["f1"]))
ys = np.arange(len(names))
vals = [statistics.mean(stats[n]["f1"]) for n in names]
sup = [int(statistics.mean(stats[n]["sup"])) for n in names]
cols = [TEAL if v >= 0 else RED for v in vals]
ax.barh(ys, vals, color=cols, edgecolor="#333333", linewidth=0.6, zorder=3)
ax.axvline(0.0, color="#555555", lw=1.0, zorder=2)
ax.set_yticks(ys)
ax.set_yticklabels([f"{n} (n\u2248{s:,})" for n, s in zip(names, sup)],
fontsize=8.2)
beta = {"xiiotid": {0.5: 0.75, 0.75: 0.5},
"5gnidd": {0.5: 1.0, 0.75: 0.5}}[ds][alpha]
ax.set_title(f"{DATASET_NAME[ds]} \u03b1={alpha:.2f} beta={beta:g}")
ax.set_xlabel("\u0394 F1 (proposed \u2212 original), test")
fig.suptitle("Per-class test F1 change for beta>0 conditions (mean over seeds {0,1,2})",
# Figure J: per-class test F1 change (one per dataset; two alpha panels)
# ---------------------------------------------------------------------------
def figure_j(dataset: str) -> None:
rows = [r for r in _read(ANALYSIS / "deepcmp__per_class.csv")
if r["dataset"] == dataset]
alphas = [0.5, 0.75]
stats = {}
for alpha in alphas:
agg = defaultdict(lambda: {"f1": [], "sup": []})
for r in rows:
if abs(float(r["alpha"]) - alpha) < 1e-9:
agg[r["class_name"]]["f1"].append(float(r["delta_f1"]))
agg[r["class_name"]]["sup"].append(int(r["support"]))
stats[alpha] = {n: (statistics.mean(v["f1"]), int(statistics.mean(v["sup"])))
for n, v in agg.items()}
amax = max(abs(v[0]) for a in alphas for v in stats[a].values())
xlim = amax * 1.35
fig, axes = plt.subplots(1, 2, figsize=(15.5, 6.4), constrained_layout=True)
for ax, alpha in zip(axes, alphas):
ax.set_axisbelow(True)
ax.grid(axis="x")
items = sorted(stats[alpha].items(), key=lambda kv: kv[1][0])
names = [n for n, _ in items]
vals = [v[0] for _, v in items]
sups = [v[1] for _, v in items]
ys = np.arange(len(names))
cols = [TEAL if v >= 0 else RED for v in vals]
bars = ax.barh(ys, vals, color=cols, edgecolor="#333333", linewidth=0.6, zorder=3)
ax.axvline(0.0, color="#444444", lw=1.1, zorder=2)
for y, v in zip(ys, vals):
off = amax * 0.03
ax.annotate(f"{v:+.4f}", (v + (off if v >= 0 else -off), y),
ha="left" if v >= 0 else "right", va="center",
fontsize=9.2, weight="bold",
color="#1B7F3B" if v >= 0 else "#8E1F14")
ax.set_yticks(ys)
ax.set_yticklabels([f"{n}\ntest n\u2248{s:,}" for n, s in zip(names, sups)],
fontsize=10)
ax.set_xlim(-xlim, xlim)
ax.set_xlabel("\u0394 F1 (proposed \u2212 ORIGINAL PROTEAN), test")
beta = SELECTED[dataset][alpha]
ax.set_title(f"{DATASET_NAME[dataset]} \u03b1={alpha:.2f} (beta={beta:g})")
fig.suptitle(f"Per-class test F1 change vs ORIGINAL PROTEAN \u2014 {DATASET_NAME[dataset]}",
fontsize=13, weight="bold", y=1.04)
fig.text(0.5, -0.02,
"Bars are mean \u0394F1 over seeds {0,1,2}; both panels use the same x-scale so bar lengths are comparable. "
"Teal = F1 increased, red = F1 decreased and the exact signed \u0394F1 is printed at each bar. Delta is F1, NOT precision or recall. "
"Only beta>0 conditions are shown (\u03b1=0.25 selected beta=0). Test support is the mean over seeds. No significance test (3 seeds).",
ha="center", va="top", fontsize=8.6, color="#444444", linespacing=1.5)
_save(fig, f"figJ{'1' if dataset == 'xiiotid' else '2'}_{dataset}_per_class_original_vs_proposed")
# ---------------------------------------------------------------------------
# Figure K (primary): ranked error changes for 5G-NIDD alpha=0.75 seed=0
# ---------------------------------------------------------------------------
def figure_k_ranked(dataset="5gnidd", alpha=0.75, seed=0):
rows = [r for r in _read(ANALYSIS / "deepcmp__confusion_delta.csv")
if r["dataset"] == dataset and abs(float(r["alpha"]) - alpha) < 1e-9
and int(r["seed"]) == seed]
off = [r for r in rows if r["true_class"] != r["pred_class"]]
reduced = sorted(off, key=lambda r: int(r["delta"]))[:5]
increased = sorted(off, key=lambda r: -int(r["delta"]))[:5]
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(11.5, 7.0),
constrained_layout=True, sharex=True)
for ax, grp, colour, sign_label in (
(ax1, increased, RED, "more errors after weighting"),
(ax2, reduced, TEAL, "fewer errors after weighting")):
ax.set_axisbelow(True)
ax.grid(axis="x")
order = sorted(grp, key=lambda r: -abs(int(r["delta"])))
labels = [f"{r['true_class']} \u2192 {r['pred_class']}" for r in order]
deltas = [int(r["delta"]) for r in order]
ys = np.arange(len(order))
ax.barh(ys, deltas, color=colour, edgecolor="#333333", linewidth=0.6, zorder=3)
ax.axvline(0.0, color="#444444", lw=1.1, zorder=2)
for y, r in zip(ys, order):
d = int(r["delta"])
ax.annotate(f"{d:+d} ({r['orig_count']} \u2192 {r['prop_count']})",
(d, y), textcoords="offset points",
xytext=(6 if d >= 0 else -6, 0), va="center",
ha="left" if d >= 0 else "right", fontsize=9,
weight="bold", color="#8E1F14" if d > 0 else "#0E5C5A")
ax.set_yticks(ys)
ax.set_yticklabels(labels, fontsize=10)
ax.set_title(("\u25b2 More errors after weighting" if colour == RED
else "\u25bc Fewer errors after weighting"), fontsize=10.5)
all_show = [int(r["delta"]) for r in increased] + [int(r["delta"]) for r in reduced]
ax1.set_xlim(min(all_show) * 1.5, max(all_show) * 1.7)
ax2.set_xlabel("change in test sample count (proposed \u2212 original; "
"negative = fewer, positive = more)")
dname = DATASET_NAME[dataset]
fig.suptitle(f"Ranked change in misclassifications vs ORIGINAL PROTEAN \u2014 "
f"{dname}, \u03b1={alpha:g}, seed={seed}, test",
fontsize=12.5, weight="bold", y=1.04)
fig.text(0.5, -0.015,
"Only beta>0 conditions are shown; \u03b1=0.25 selected beta=0 so it carries no weighting effect. "
"Class support n\u2248 is the mean test support over seeds. Teal = F1 increased, red = F1 decreased. "
"X-IIoTID and 5G-NIDD class labels are never mixed. No significance testing (3 seeds).",
ha="center", va="top", fontsize=8.4, color="#444444", linespacing=1.5)
_save(fig, "figJ_per_class_original_vs_proposed")
fig.text(0.5, -0.02,
"Each bar is one off-diagonal (true \u2192 predicted) confusion cell; raw test counts over the same 243,177 samples. "
"Below zero = fewer such errors under the proposed selected-beta run; above zero = more. "
"The dominant error Benign \u2192 UDPFlood is ~unchanged (56,899 \u2192 57,054, +155 more) and remains the largest error. "
"Full difference matrices are in the supplemental figure. No statistical significance is inferred.",
ha="center", va="top", fontsize=8.6, color="#444444", linespacing=1.5)
_save(fig, f"figK_confusion_error_changes_{dataset}_a{alpha:g}_seed{seed}")
# ---------------------------------------------------------------------------
def figure_k(dataset="5gnidd", alpha=0.75, seed=0, beta=0.5):
def figure_k_full(dataset="5gnidd", alpha=0.75, seed=0, beta=0.5):
"""Supplemental full difference matrices (raw counts and row-rate)."""
odir = RESULTS / f"{dataset}__alpha{alpha:g}__seed{seed}__original_protean"
sdir = RESULTS / f"{dataset}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}"
names = json.loads((odir / "metadata.json").read_text())["class_names"]
@@ -187,42 +265,40 @@ def figure_k(dataset="5gnidd", alpha=0.75, seed=0, beta=0.5):
for ax in axes:
ax.set_xticks(range(len(names)))
ax.set_yticks(range(len(names)))
ax.set_xticklabels(names, rotation=45, ha="right", fontsize=8)
ax.set_yticklabels(names, fontsize=8)
ax.set_xticklabels(names, rotation=45, ha="right", fontsize=9)
ax.set_yticklabels(names, fontsize=9)
ax.set_xlabel("predicted class")
axes[0].set_ylabel("true class")
axes[0].set_title("A. Difference in raw counts\n(proposed \u2212 original)")
axes[1].set_title("B. Difference in row-normalized rate\n(percentage points, proposed \u2212 original)")
# annotate the strongest cells on each panel
axes[0].set_title("A. Raw count difference\n(proposed \u2212 original)")
axes[1].set_title("B. Row-normalized rate difference\n(percentage points)")
for ax, M, is_rate in ((axes[0], diff, False), (axes[1], diff_rate * 100, True)):
flat = [(abs(M[i, j]), i, j) for i in range(9) for j in range(9) if i != j]
flat = [(abs(M[i, j]), i, j) for i in range(len(names)) for j in range(len(names)) if i != j]
for _, i, j in sorted(flat, reverse=True)[:6]:
v = M[i, j]
ax.text(j, i, f"{v:+.0f}" if not is_rate else f"{v:+.1f}",
ha="center", va="center", fontsize=6.4, fontweight="bold",
color="white" if abs(v) > (vmax_c if not is_rate else vmax_r) * 0.5
else "#222222")
cb0 = fig.colorbar(im0, ax=axes[0], fraction=0.046, pad=0.02)
cb0.set_label("count delta")
cb1 = fig.colorbar(im1, ax=axes[1], fraction=0.046, pad=0.02)
cb1.set_label("row-rate delta (pp)")
dname = DATASET_NAME[dataset]
fig.suptitle(f"Confusion-matrix change: proposed (beta={beta:g}) minus original PROTEAN - "
f"{dname}, alpha={alpha:g}, seed={seed}, test",
ha="center", va="center", fontsize=6.6, fontweight="bold",
color="white" if abs(v) > (vmax_c if not is_rate else vmax_r) * 0.5 else "#222222")
fig.colorbar(im0, ax=axes[0], fraction=0.046, pad=0.02).set_label("count delta")
fig.colorbar(im1, ax=axes[1], fraction=0.046, pad=0.02).set_label("row-rate delta (pp)")
fig.suptitle(f"Supplemental: full confusion-matrix change (proposed \u2212 original) \u2014 "
f"{DATASET_NAME[dataset]}, \u03b1={alpha:g}, seed={seed}, test",
fontsize=12, weight="bold", y=1.05)
fig.text(0.5, -0.02,
"Rows = true class, columns = predicted class; identical class order and sample counts (243,177) for both runs. "
"Diagonal: positive = more correct predictions; off-diagonal: positive = more errors of that (true\u2192predicted) type. "
"Red = positive delta (proposed has MORE samples in the cell); blue = negative delta (fewer). Values are descriptive only; no significance is inferred.",
ha="center", va="top", fontsize=8.4, color="#444444", linespacing=1.5)
"Rows = true class, columns = predicted class; identical class order and 243,177 samples. "
"Diagonal: positive = more correct predictions; off-diagonal: positive = more errors of that type. "
"Red = positive delta, blue = negative. See the ranked figure for the practical reading.",
ha="center", va="top", fontsize=8.6, color="#444444", linespacing=1.5)
_save(fig, f"figK_confusion_delta_{dataset}_a{alpha:g}_seed{seed}")
def main():
_style()
figure_i()
figure_j()
figure_k()
figure_i("xiiotid")
figure_i("5gnidd")
figure_j("xiiotid")
figure_j("5gnidd")
figure_k_ranked()
figure_k_full()
if __name__ == "__main__":
scripts/make_final_figures.py
+91 -44
View File
@@ -327,43 +327,69 @@ def figure_f() -> None:
sel_e = np.array([float(master[c]["selected_rare_macro_f1_std"]) for c in conditions])
betas = [float(master[c]["selected_beta"]) for c in conditions]
delta = sel - orig
fig, ax = plt.subplots(figsize=(11.4, 6.0), constrained_layout=True)
ax.set_axisbelow(True)
ax.grid(axis="y")
x = np.arange(len(conditions))
w = 0.36
ax.bar(x - w / 2, orig, w, yerr=orig_e, capsize=3.5, color=CHARCOAL,
edgecolor="#1A1A1A", linewidth=0.7, label="original_protean", zorder=3)
ax.bar(x + w / 2, sel, w, yerr=sel_e, capsize=3.5, color=TEAL,
edgecolor="#0A4F4E", linewidth=0.7, hatch="///",
label="support_weighted (validation-selected beta)", zorder=3)
w = 0.38
for xi, db, sb in zip(x, delta, betas):
if sb == 0.0:
colour = GREY
elif db > 0:
colour = GREEN
else:
colour = RED
sign = "+" if db >= 0 else "\u2212"
fig, (axA, axB) = plt.subplots(2, 1, figsize=(11.2, 8.6),
constrained_layout=True,
gridspec_kw={"height_ratios": [1.15, 1.0]})
# --- Panel A: absolute levels ---------------------------------------
axA.set_axisbelow(True)
axA.grid(axis="y")
axA.bar(x - w / 2, orig, w, yerr=orig_e, capsize=3.5, color=CHARCOAL,
edgecolor="#1A1A1A", linewidth=0.7, label="original_protean", zorder=3)
axA.bar(x + w / 2, sel, w, yerr=sel_e, capsize=3.5, color=TEAL,
edgecolor="#0A4F4E", linewidth=0.7, hatch="///",
label="proposed: support_weighted (selected beta)", zorder=3)
for xi, sb in zip(x, betas):
top = max(orig[xi] + orig_e[xi], sel[xi] + sel_e[xi])
ax.annotate(f"{sign}{abs(db):.4f}", (xi, top + 0.012), ha="center",
fontsize=9.5, weight="bold", color=colour)
ax.annotate(f"beta={sb:g}", (xi, top + 0.030), ha="center", fontsize=8.5,
color="#444444")
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.set_ylabel("test rare-class macro-F1\n(mean over seeds {0,1,2})")
ax.set_ylim(0.5, max(max(orig + orig_e), max(sel + sel_e)) + 0.10)
ax.set_title("Test rare-class macro-F1: original PROTEAN vs validation-selected beta")
ax.legend(loc="upper left", frameon=True, edgecolor="#CCCCCC")
axA.annotate(f"beta={sb:g}", (xi, top + 0.006), ha="center",
fontsize=9, color="#444444")
axA.set_xticks(x)
axA.set_xticklabels([])
axA.set_ylabel("test rare-class macro-F1\n(mean over seeds {0,1,2})")
axA.set_ylim(0.5, max(max(orig + orig_e), max(sel + sel_e)) + 0.075)
axA.set_title("A. Absolute test rare-class macro-F1")
axA.legend(loc="upper left", frameon=True, edgecolor="#CCCCCC")
# --- Panel B: signed difference -------------------------------------
axB.set_axisbelow(True)
axB.grid(axis="y")
cols = [GREY if sb == 0 else (GREEN if d > 0 else RED)
for d, sb in zip(delta, betas)]
axB.bar(x, delta, 0.55, color=cols, edgecolor="#333333", linewidth=0.6, zorder=3)
axB.axhline(0.0, color="#444444", lw=1.1, zorder=2)
for xi, d, sb in zip(x, delta, betas):
sign = "\u2212" if d < 0 else "+"
colr = "#1B7F3B" if (sb > 0 and d > 0) else ("#8E1F14" if (sb > 0 and d < 0) else "#555555")
axB.annotate(f"{sign}{abs(d):.4f}", (xi, d), textcoords="offset points",
xytext=(0, 6 if d >= 0 else -14), ha="center", fontsize=10.5,
weight="bold", color=colr)
axB.set_xticks(x)
axB.set_xticklabels(labels)
axB.set_ylabel("\u0394 rare-class macro-F1\n(proposed \u2212 original)")
m = max(abs(delta)) * 1.65
axB.set_ylim(-m, m)
axB.set_title("B. Signed difference (proposed \u2212 original)")
handles = [plt.Rectangle((0, 0), 1, 1, facecolor=GREEN, edgecolor="#333"),
plt.Rectangle((0, 0), 1, 1, facecolor=RED, edgecolor="#333"),
plt.Rectangle((0, 0), 1, 1, facecolor=GREY, edgecolor="#333")]
axB.legend(handles, ["positive (3 conditions)", "negative (1 condition)",
"selected beta = 0 / ~0 (2 conditions)"],
title="signed delta", loc="upper right", frameon=True,
edgecolor="#CCCCCC", ncols=1)
fig.suptitle("Original PROTEAN vs proposed selected-beta: test rare-class macro-F1",
fontsize=13, weight="bold", y=1.02)
_footnote(fig,
"Bar height = mean test rare-class macro-F1 over seeds {0,1,2}; error bars = \u00b11 sample std across seeds.\n"
"Delta (top) = selected-beta minus original_protean; green = positive, red = negative, grey = selected beta 0\n"
"(floating-point ~0). No statistical significance test is applied (3 seeds). Gains are condition-dependent;\n"
"Panel A: bar height = mean test rare-class macro-F1 over seeds {0,1,2}; error bars = \u00b11 sample std across seeds "
"(the levels are close by design, so panel B is the informative view).\n"
"Panel B: delta = proposed \u2212 original; the exact value is printed at each bar. Green = positive (X-IIoTID a0.50, "
"X-IIoTID a0.75, 5G-NIDD a0.75); red = negative (5G-NIDD a0.50); grey = selected beta 0 (both a0.25, floating-point ~0).\n"
"No statistical significance test is applied (3 seeds). A positive mean delta does not establish significance; "
"the proposed weighting does NOT consistently outperform original PROTEAN.",
y=-0.13)
y=-0.10)
_save(fig, "figF_selected_vs_original_test_rare_macro_f1")
@@ -458,15 +484,34 @@ def figure_h(dataset="5gnidd", alpha=0.75, seed=0, selected_beta=0.5) -> bool:
class_names = meta["class_names"]
mats = {k: np.array(json.loads(p.read_text())) for k, p in paths.items()}
o, s = mats["original_protean"], mats["selected_beta"]
diff = s - o
idx = {n: i for i, n in enumerate(class_names)}
# Computed summary of the most important changes (never hard-coded).
off = [(int(diff[i, j]), i, j) for i in range(len(class_names))
for j in range(len(class_names)) if i != j]
reduced = sorted(off, key=lambda t: t[0])[:2]
increased = sorted(off, key=lambda t: -t[0])[:2]
diag = sorted(range(len(class_names)), key=lambda i: -abs(int(diff[i, i])))[:3]
lines = ["Most important ranked changes (proposed \u2212 original):"]
for d, i, j in reduced:
lines.append(f" \u2022 {class_names[i]} \u2192 {class_names[j]}: "
f"{int(o[i, j])} \u2192 {int(s[i, j])} ({d} fewer)")
for d, i, j in increased:
note = " \u2190 dominant residual error; NOT an improvement" \
if (class_names[i], class_names[j]) == ("Benign", "UDPFlood") else ""
lines.append(f" \u2022 {class_names[i]} \u2192 {class_names[j]}: "
f"{int(o[i, j])} \u2192 {int(s[i, j])} (+{d} more){note}")
lines.append(" \u2022 diagonal (correct): " + ", ".join(
f"{class_names[i]} {int(diff[i, i]):+d}" for i in diag))
from matplotlib.colors import LogNorm
vmax = max(m.max() for m in mats.values())
vmin = 1
fig, axes = plt.subplots(1, 2, figsize=(13.2, 6.2), constrained_layout=True)
titles = {
"original_protean": "ORIGINAL PROTEAN",
"selected_beta": f"SELECTED-BETA PROTEAN (beta={selected_beta:g})",
}
fig, axes = plt.subplots(1, 2, figsize=(13.6, 6.0), constrained_layout=True)
titles = {"original_protean": "LEFT: ORIGINAL PROTEAN",
"selected_beta": f"RIGHT: BETA-TUNED PROTEAN (beta={selected_beta:g})"}
im = None
for ax, key in zip(axes, ["original_protean", "selected_beta"]):
m = mats[key]
@@ -474,8 +519,8 @@ def figure_h(dataset="5gnidd", alpha=0.75, seed=0, selected_beta=0.5) -> bool:
aspect="equal")
ax.set_xticks(range(len(class_names)))
ax.set_yticks(range(len(class_names)))
ax.set_xticklabels(class_names, rotation=45, ha="right", fontsize=8.2)
ax.set_yticklabels(class_names, fontsize=8.2)
ax.set_xticklabels(class_names, rotation=45, ha="right", fontsize=9)
ax.set_yticklabels(class_names, fontsize=9)
ax.set_xlabel("predicted class")
ax.set_title(titles[key], fontsize=11)
thr = m.max() * 0.06
@@ -485,7 +530,7 @@ def figure_h(dataset="5gnidd", alpha=0.75, seed=0, selected_beta=0.5) -> bool:
if v == 0:
continue
txt = f"{v/1000:.1f}k" if v >= 10000 else str(v)
ax.text(j, i, txt, ha="center", va="center", fontsize=6.4,
ax.text(j, i, txt, ha="center", va="center", fontsize=6.8,
color="white" if m[i, j] > thr else "#333333")
axes[0].set_ylabel("true class")
cbar = fig.colorbar(im, ax=axes, fraction=0.025, pad=0.02)
@@ -494,11 +539,13 @@ def figure_h(dataset="5gnidd", alpha=0.75, seed=0, selected_beta=0.5) -> bool:
fig.suptitle(
f"Paired test confusion matrices - {dname}, alpha={alpha:g}, seed={seed} "
f"(same manifest, split, seed, class order)",
fontsize=11.5, weight="bold", y=1.05)
fontsize=12, weight="bold", y=1.05)
_footnote(fig,
"Left = original_protean; right = support_weighted at the validation-selected beta. Raw counts; identical class order and colour scale.\n"
"Dominant off-diagonal error: Benign predicted as UDPFlood, essentially unchanged by beta. No statistical significance is inferred.",
y=-0.20)
"Rows = actual class, columns = predicted class; diagonal = correct, off-diagonal = misclassified. Raw counts, "
"identical class order and colour scale; both matrices contain the same 243,177 test samples.\n"
+ "\n".join(lines) +
"\nNo statistical significance is inferred from the visual difference.",
y=-0.13)
_save(fig, f"figH_confusion_matrix_{dataset}_a{alpha:g}_seed{seed}")
return True