experiment: validate beta selection across three seeds
This commit is contained in:
willtanoe committed
1 parent
50f2da7216
commit
6b97a0d38b
14 files changed
+1963
-150
No files matched your search
@@ -0,0 +1,450 @@
|
||||
#!/usr/bin/env python
|
||||
"""Pooled multi-seed validation analysis for the beta aggregation study.
|
||||
|
||||
Reads the per-run artifacts already written by ``scripts/run_baseline.py`` for
|
||||
one ``(dataset, alpha)`` condition across several seeds and beta candidates and
|
||||
produces:
|
||||
|
||||
- a per-(seed, beta) validation table (the candidate-evaluation record);
|
||||
- a pooled-across-seed validation table (the PRIMARY selection rule);
|
||||
- per-client rare-class deltas relative to ``beta = 0`` for each seed;
|
||||
- a machine-readable selection summary.
|
||||
|
||||
This script only reads existing artifacts. It never trains and never uses test
|
||||
metrics for selection; test columns, where emitted, are clearly labelled as
|
||||
descriptive only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT / "src"))
|
||||
|
||||
from protean_research.aggregation import BETA_CANDIDATES # noqa: E402
|
||||
|
||||
|
||||
def beta_tag(beta: float) -> str:
|
||||
return f"{beta:g}"
|
||||
|
||||
|
||||
def run_name(dataset: str, alpha: float, seed: int, beta: float) -> str:
|
||||
return f"{dataset}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta_tag(beta)}"
|
||||
|
||||
|
||||
def original_run_name(dataset: str, alpha: float, seed: int) -> str:
|
||||
return f"{dataset}__alpha{alpha:g}__seed{seed}__original_protean"
|
||||
|
||||
|
||||
def read_json(path: Path):
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def read_csv(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
def parse_args(argv=None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--dataset", default="xiiotid")
|
||||
parser.add_argument("--alpha", type=float, default=0.5)
|
||||
parser.add_argument("--seeds", type=int, nargs="+", default=[0, 1, 2])
|
||||
parser.add_argument(
|
||||
"--betas", type=float, nargs="+", default=list(BETA_CANDIDATES)
|
||||
)
|
||||
parser.add_argument("--results-dir", default="results")
|
||||
parser.add_argument("--analysis-dir", default="artifacts/analysis")
|
||||
parser.add_argument(
|
||||
"--tag",
|
||||
default=None,
|
||||
help="output stem suffix (default: <dataset>__alpha<alpha>__multiseed)",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def mean_or_none(values):
|
||||
values = [float(v) for v in values if v is not None]
|
||||
return statistics.fmean(values) if values else None
|
||||
|
||||
|
||||
def std_or_none(values, ddof: int = 1):
|
||||
values = [float(v) for v in values if v is not None]
|
||||
if len(values) < 2:
|
||||
return None
|
||||
return statistics.stdev(values) if ddof == 1 else statistics.pstdev(values)
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
args = parse_args(argv)
|
||||
results_root = Path(args.results_dir).expanduser()
|
||||
if not results_root.is_absolute():
|
||||
results_root = REPO_ROOT / results_root
|
||||
analysis_dir = Path(args.analysis_dir).expanduser()
|
||||
if not analysis_dir.is_absolute():
|
||||
analysis_dir = REPO_ROOT / analysis_dir
|
||||
analysis_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tag = args.tag or f"{args.dataset}__alpha{args.alpha:g}__multiseed"
|
||||
|
||||
per_seed_rows: list[dict] = []
|
||||
client_rows: list[dict] = []
|
||||
missing: list[str] = []
|
||||
|
||||
for seed in args.seeds:
|
||||
for beta in args.betas:
|
||||
name = run_name(args.dataset, args.alpha, seed, beta)
|
||||
run_dir = results_root / name
|
||||
selection_path = run_dir / "val_selection.json"
|
||||
if not selection_path.exists():
|
||||
missing.append(str(selection_path))
|
||||
continue
|
||||
selection = read_json(selection_path)
|
||||
metadata = read_json(run_dir / "metadata.json")
|
||||
row = {
|
||||
"seed": seed,
|
||||
"beta": beta,
|
||||
"eligible_clients": selection["val_n_eval_clients"],
|
||||
"val_rare_macro_f1_per_client_mean": selection[
|
||||
"val_rare_class_macro_f1_per_client_mean"
|
||||
],
|
||||
"val_rare_macro_f1_global": selection["val_rare_class_macro_f1_global"],
|
||||
"val_rare_recall_per_client_mean": selection[
|
||||
"val_rare_class_recall_per_client_mean"
|
||||
],
|
||||
"val_rare_recall_global": selection["val_rare_class_recall_global"],
|
||||
"val_macro_f1_global": selection["val_macro_f1_global"],
|
||||
"val_accuracy_global": selection["val_accuracy_global"],
|
||||
"val_weighted_f1_global": selection["val_weighted_f1_global"],
|
||||
"manifest_content_hash": metadata.get("manifest_content_hash"),
|
||||
"manifest_reload_ok": metadata.get("manifest_reload_ok"),
|
||||
}
|
||||
per_seed_rows.append(row)
|
||||
|
||||
# per-client validation metrics relative to beta=0
|
||||
per_client_path = run_dir / "val_per_client.csv"
|
||||
if per_client_path.exists():
|
||||
for entry in read_csv(per_client_path):
|
||||
client_rows.append({
|
||||
"seed": seed,
|
||||
"beta": beta,
|
||||
"client_id": int(entry["client_id"]),
|
||||
"n_samples": int(entry["n_samples"]),
|
||||
"val_rare_macro_f1": (
|
||||
float(entry["rare_class_macro_f1"])
|
||||
if entry["rare_class_macro_f1"] not in ("", "None")
|
||||
else None
|
||||
),
|
||||
"val_rare_recall": (
|
||||
float(entry["rare_class_recall"])
|
||||
if entry["rare_class_recall"] not in ("", "None")
|
||||
else None
|
||||
),
|
||||
})
|
||||
|
||||
if missing:
|
||||
print("MISSING ARTIFACTS (analysis incomplete):", file=sys.stderr)
|
||||
for path in missing:
|
||||
print(" " + path, file=sys.stderr)
|
||||
|
||||
# ---- pooled table -----------------------------------------------------
|
||||
pooled_rows: list[dict] = []
|
||||
for beta in args.betas:
|
||||
rows = [r for r in per_seed_rows if r["beta"] == beta]
|
||||
scores = [r["val_rare_macro_f1_per_client_mean"] for r in rows]
|
||||
recalls_pc = [r["val_rare_recall_per_client_mean"] for r in rows]
|
||||
recalls_g = [r["val_rare_recall_global"] for r in rows]
|
||||
pooled_rows.append({
|
||||
"beta": beta,
|
||||
"n_seeds": len(rows),
|
||||
"pooled_val_rare_macro_f1_per_client_mean": mean_or_none(scores),
|
||||
"std_sample_val_rare_macro_f1": std_or_none(scores, ddof=1),
|
||||
"std_pop_val_rare_macro_f1": std_or_none(scores, ddof=0),
|
||||
"min_val_rare_macro_f1": min(scores) if scores else None,
|
||||
"max_val_rare_macro_f1": max(scores) if scores else None,
|
||||
"pooled_val_rare_recall_per_client_mean": mean_or_none(recalls_pc),
|
||||
"pooled_val_rare_recall_global": mean_or_none(recalls_g),
|
||||
})
|
||||
|
||||
# ---- PRIMARY pooled selection ----------------------------------------
|
||||
selectable = [r for r in pooled_rows if r["n_seeds"] == len(args.seeds)]
|
||||
selected_beta = None
|
||||
selection_criterion = None
|
||||
tie_break_used = False
|
||||
if selectable:
|
||||
best_score = max(
|
||||
r["pooled_val_rare_macro_f1_per_client_mean"] for r in selectable
|
||||
)
|
||||
finalists = [
|
||||
r for r in selectable
|
||||
if abs(r["pooled_val_rare_macro_f1_per_client_mean"] - best_score) <= 1e-12
|
||||
]
|
||||
if len(finalists) > 1:
|
||||
tie_break_used = True
|
||||
best_recall = max(
|
||||
r["pooled_val_rare_recall_per_client_mean"] for r in finalists
|
||||
)
|
||||
finalists = [
|
||||
r for r in finalists
|
||||
if abs(r["pooled_val_rare_recall_per_client_mean"] - best_recall) <= 1e-12
|
||||
]
|
||||
if len(finalists) > 1:
|
||||
min_std = min(r["std_sample_val_rare_macro_f1"] for r in finalists)
|
||||
finalists = [
|
||||
r for r in finalists
|
||||
if abs(r["std_sample_val_rare_macro_f1"] - min_std) <= 1e-12
|
||||
]
|
||||
selected_beta = min(r["beta"] for r in finalists)
|
||||
selection_criterion = (
|
||||
"max pooled validation per-client-mean rare-class macro-F1 "
|
||||
"(tie-break: recall, then lower std, then smaller beta)"
|
||||
)
|
||||
|
||||
# ---- per-seed diagnostic selection -----------------------------------
|
||||
per_seed_selection = []
|
||||
for seed in args.seeds:
|
||||
rows = [r for r in per_seed_rows if r["seed"] == seed]
|
||||
if not rows:
|
||||
continue
|
||||
best = max(r["val_rare_macro_f1_per_client_mean"] for r in rows)
|
||||
finalists = [
|
||||
r for r in rows
|
||||
if abs(r["val_rare_macro_f1_per_client_mean"] - best) <= 1e-12
|
||||
]
|
||||
if len(finalists) > 1:
|
||||
best_recall = max(r["val_rare_recall_per_client_mean"] for r in finalists)
|
||||
finalists = [
|
||||
r for r in finalists
|
||||
if abs(r["val_rare_recall_per_client_mean"] - best_recall) <= 1e-12
|
||||
]
|
||||
selected = min(r["beta"] for r in finalists)
|
||||
per_seed_selection.append({
|
||||
"seed": seed,
|
||||
"selected_beta": selected,
|
||||
"val_rare_macro_f1_per_client_mean": best,
|
||||
"test_used_for_selection": False,
|
||||
})
|
||||
|
||||
# ---- support lookup + client-level summary (validation, vs beta = 0) --
|
||||
def support_lookup(seed: int):
|
||||
meta_path = (
|
||||
results_root / run_name(args.dataset, args.alpha, seed, args.betas[0])
|
||||
/ "metadata.json"
|
||||
)
|
||||
benign = read_json(meta_path).get("benign_class") if meta_path.exists() else None
|
||||
support_path = (
|
||||
results_root / run_name(args.dataset, args.alpha, seed, args.betas[0])
|
||||
/ "train_support_Nij.csv"
|
||||
)
|
||||
support: dict[int, dict[str, int]] = {}
|
||||
if support_path.exists():
|
||||
for entry in read_csv(support_path):
|
||||
cid = int(entry["client_id"])
|
||||
support[cid] = {k: int(v) for k, v in entry.items() if k != "client_id"}
|
||||
return support, benign
|
||||
|
||||
client_summary_rows: list[dict] = []
|
||||
client_detail_rows: list[dict] = []
|
||||
for seed in args.seeds:
|
||||
support, benign = support_lookup(seed)
|
||||
min_pos = {}
|
||||
for cid, vals in support.items():
|
||||
attacks = [v for k, v in vals.items() if k != benign and v > 0]
|
||||
min_pos[cid] = min(attacks) if attacks else None
|
||||
base = {
|
||||
r["client_id"]: r["val_rare_macro_f1"]
|
||||
for r in client_rows
|
||||
if r["seed"] == seed and r["beta"] == args.betas[0]
|
||||
}
|
||||
for beta in args.betas:
|
||||
deltas = []
|
||||
for r in client_rows:
|
||||
if r["seed"] != seed or r["beta"] != beta:
|
||||
continue
|
||||
if r["val_rare_macro_f1"] is None or base.get(r["client_id"]) is None:
|
||||
continue
|
||||
delta = r["val_rare_macro_f1"] - base[r["client_id"]]
|
||||
deltas.append(delta)
|
||||
client_detail_rows.append({
|
||||
"seed": seed,
|
||||
"beta": beta,
|
||||
"client_id": r["client_id"],
|
||||
"min_positive_attack_Nij": min_pos.get(r["client_id"]),
|
||||
"val_rare_macro_f1": r["val_rare_macro_f1"],
|
||||
"val_rare_recall": r["val_rare_recall"],
|
||||
"delta_vs_beta0": delta,
|
||||
})
|
||||
n_improve = sum(1 for d in deltas if d > 1e-9)
|
||||
n_degrade = sum(1 for d in deltas if d < -1e-9)
|
||||
client_summary_rows.append({
|
||||
"seed": seed,
|
||||
"beta": beta,
|
||||
"n_eligible": len(deltas),
|
||||
"n_improve_vs_beta0": n_improve,
|
||||
"n_degrade_vs_beta0": n_degrade,
|
||||
"n_unchanged": len(deltas) - n_improve - n_degrade,
|
||||
"mean_delta_vs_beta0": mean_or_none(deltas),
|
||||
"median_delta_vs_beta0": statistics.median(deltas) if deltas else None,
|
||||
"min_delta_vs_beta0": min(deltas) if deltas else None,
|
||||
"max_delta_vs_beta0": max(deltas) if deltas else None,
|
||||
})
|
||||
|
||||
# ---- direct prototype-effect cross-seed summary ----------------------
|
||||
effect_raw: list[dict] = []
|
||||
for seed in args.seeds:
|
||||
path = analysis_dir / (
|
||||
f"{args.dataset}__alpha{args.alpha:g}__seed{seed}__support_weighted__beta0"
|
||||
"__prototype_snapshot__prototype_effect.csv"
|
||||
)
|
||||
if not path.exists():
|
||||
continue
|
||||
for r in read_csv(path):
|
||||
effect_raw.append({
|
||||
"seed": seed,
|
||||
"class_id": int(r["class_id"]),
|
||||
"class_name": r["class_name"],
|
||||
"beta": float(r["beta"]),
|
||||
"n_contributors": int(r["n_contributors"]),
|
||||
"support_min": int(r["support_min"]),
|
||||
"support_max": int(r["support_max"]),
|
||||
"prototype_norm": float(r["prototype_norm"]),
|
||||
"displacement_from_beta0": float(r["displacement_from_beta0"]),
|
||||
"relative_displacement": float(r["relative_displacement"]),
|
||||
})
|
||||
effect_pooled_rows: list[dict] = []
|
||||
for class_id in sorted({r["class_id"] for r in effect_raw}):
|
||||
class_name = next(
|
||||
r["class_name"] for r in effect_raw if r["class_id"] == class_id
|
||||
)
|
||||
for beta in args.betas:
|
||||
subset = [
|
||||
r for r in effect_raw
|
||||
if r["class_id"] == class_id and abs(r["beta"] - beta) <= 1e-9
|
||||
]
|
||||
if not subset:
|
||||
continue
|
||||
disps = [r["displacement_from_beta0"] for r in subset]
|
||||
rel = [r["relative_displacement"] for r in subset]
|
||||
effect_pooled_rows.append({
|
||||
"class_id": class_id,
|
||||
"class_name": class_name,
|
||||
"beta": beta,
|
||||
"n_seeds": len(subset),
|
||||
"n_contributors_seed0": subset[0]["n_contributors"],
|
||||
"mean_displacement": mean_or_none(disps),
|
||||
"std_sample_displacement": std_or_none(disps, ddof=1),
|
||||
"min_displacement": min(disps),
|
||||
"max_displacement": max(disps),
|
||||
"mean_relative_displacement": mean_or_none(rel),
|
||||
})
|
||||
|
||||
# ---- test metrics for the selected beta + original baseline ----------
|
||||
test_rows: list[dict] = []
|
||||
for seed in args.seeds:
|
||||
for beta in args.betas:
|
||||
path = results_root / run_name(args.dataset, args.alpha, seed, beta) / "test_global_metrics.json"
|
||||
if not path.exists():
|
||||
continue
|
||||
metrics = read_json(path)
|
||||
test_rows.append({
|
||||
"seed": seed,
|
||||
"aggregation": f"support_weighted_beta{beta_tag(beta)}",
|
||||
"beta": beta,
|
||||
"test_rare_macro_f1": metrics["rare_class_macro_f1"],
|
||||
"test_rare_recall": metrics["rare_class_recall"],
|
||||
"test_macro_f1": metrics["macro_f1"],
|
||||
"test_accuracy": metrics["accuracy"],
|
||||
"test_weighted_f1": metrics["weighted_f1"],
|
||||
"test_rare_support": metrics.get("rare_support"),
|
||||
})
|
||||
original_path = results_root / original_run_name(args.dataset, args.alpha, seed) / "test_global_metrics.json"
|
||||
if original_path.exists():
|
||||
metrics = read_json(original_path)
|
||||
test_rows.append({
|
||||
"seed": seed,
|
||||
"aggregation": "original_protean",
|
||||
"beta": None,
|
||||
"test_rare_macro_f1": metrics["rare_class_macro_f1"],
|
||||
"test_rare_recall": metrics["rare_class_recall"],
|
||||
"test_macro_f1": metrics["macro_f1"],
|
||||
"test_accuracy": metrics["accuracy"],
|
||||
"test_weighted_f1": metrics["weighted_f1"],
|
||||
"test_rare_support": metrics.get("rare_support"),
|
||||
})
|
||||
|
||||
# ---- write artifacts --------------------------------------------------
|
||||
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||
if not rows:
|
||||
path.write_text("", encoding="utf-8")
|
||||
return
|
||||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
per_seed_path = analysis_dir / f"{tag}__validation_per_seed.csv"
|
||||
pooled_path = analysis_dir / f"{tag}__validation_pooled.csv"
|
||||
client_path = analysis_dir / f"{tag}__client_validation_deltas.csv"
|
||||
test_path = analysis_dir / f"{tag}__test_descriptive.csv"
|
||||
client_summary_path = analysis_dir / f"{tag}__client_validation_summary.csv"
|
||||
client_detail_path = analysis_dir / f"{tag}__client_validation_detail.csv"
|
||||
effect_path = analysis_dir / f"{tag}__prototype_effect_pooled.csv"
|
||||
summary_path = analysis_dir / f"{tag}__selection_summary.json"
|
||||
|
||||
write_csv(per_seed_path, per_seed_rows)
|
||||
write_csv(pooled_path, pooled_rows)
|
||||
write_csv(client_path, client_rows)
|
||||
write_csv(test_path, test_rows)
|
||||
write_csv(client_summary_path, client_summary_rows)
|
||||
write_csv(client_detail_path, client_detail_rows)
|
||||
write_csv(effect_path, effect_pooled_rows)
|
||||
|
||||
summary = {
|
||||
"dataset": args.dataset,
|
||||
"alpha": args.alpha,
|
||||
"seeds": args.seeds,
|
||||
"betas": args.betas,
|
||||
"primary_selection": {
|
||||
"rule": selection_criterion,
|
||||
"selected_beta": selected_beta,
|
||||
"tie_break_used": tie_break_used,
|
||||
"selection_split": "validation",
|
||||
"test_used_for_selection": False,
|
||||
},
|
||||
"per_seed_diagnostic": per_seed_selection,
|
||||
"pooled_table": pooled_rows,
|
||||
"n_missing_artifacts": len(missing),
|
||||
"missing_artifacts": missing,
|
||||
"outputs": {
|
||||
"per_seed": str(per_seed_path),
|
||||
"pooled": str(pooled_path),
|
||||
"client_deltas": str(client_path),
|
||||
"client_validation_summary": str(client_summary_path),
|
||||
"client_validation_detail": str(client_detail_path),
|
||||
"test_descriptive": str(test_path),
|
||||
"prototype_effect_pooled": str(effect_path),
|
||||
},
|
||||
}
|
||||
with summary_path.open("w", encoding="utf-8") as handle:
|
||||
json.dump(summary, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
|
||||
print(json.dumps({
|
||||
"pooled_selected_beta": selected_beta,
|
||||
"tie_break_used": tie_break_used,
|
||||
"per_seed_diagnostic": per_seed_selection,
|
||||
"n_missing_artifacts": len(missing),
|
||||
"summary": str(summary_path),
|
||||
}, indent=2, sort_keys=True))
|
||||
return 0 if not missing else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user