diff --git a/scripts/sweep/README.md b/scripts/sweep/README.md new file mode 100644 index 0000000..f35b6f3 --- /dev/null +++ b/scripts/sweep/README.md @@ -0,0 +1,99 @@ +# Sweep analysis scripts + +Read-only analysis helpers for the completed seed-0 sweeps. They were originally +written inside `results/sweep/` and `results/sweep_repro/` (both git-ignored); +these copies are the version-controlled source. + +All four scripts accept `--repo-root` (default: the repository that contains +`scripts/sweep/`, i.e. two levels up from the script). Data is read from the +sweep run directories and, where noted, from the earlier grid runs and +`artifacts/analysis/`. Nothing is retrained. + +## `analyze_sweep.py` + +Inputs +- `results/sweep/runs/__alpha__seed0__support_weighted__beta/COMPLETE` +- `.../results//val_selection.json` +- `.../results//test_global_metrics.json` +- `results//val_selection.json` and `test_global_metrics.json` for the earlier + grid runs (stage 3) + +Outputs +- `results/sweep/analysis/selection.csv` (validation only) +- `results/sweep/analysis/test_at_selected_beta.csv` +- `results/sweep/analysis/repro_vs_previous.csv` + +Command +```bash +python3 scripts/sweep/analyze_sweep.py +# or, against a different tree: +python3 scripts/sweep/analyze_sweep.py --repo-root /path/to/repo +``` + +## `check_noise.py` + +Inputs +- `results/sweep/runs//results//{history.json,val_selection.json,test_global_metrics.json}` +- `results//{history.json,val_selection.json,test_global_metrics.json}` (earlier grid) + +Outputs +- `results/sweep/analysis/noise_checks.json` + +Command +```bash +python3 scripts/sweep/check_noise.py +# or: +python3 scripts/sweep/check_noise.py --repo-root /path/to/repo +``` + +## `compare_with_grid.py` + +Inputs +- `results/sweep/analysis/selection.csv` (run `analyze_sweep.py` first) +- `artifacts/analysis/final__six_condition_master.csv` +- `artifacts/analysis/__alpha__multiseed__validation_per_seed.csv` + +Outputs +- `results/sweep/analysis/compare_with_grid.csv` + +Command +```bash +python3 scripts/sweep/compare_with_grid.py +# or: +python3 scripts/sweep/compare_with_grid.py --repo-root /path/to/repo +``` + +## `compare_repro.py` + +Inputs +- `results/sweep_repro/a_1thread_repeat/results//` +- `results/sweep_repro/b_8thread/results//` +- `results/sweep/runs//results//` +- `results//` (earlier grid) +- prototype snapshots under `results/sweep_repro/*/analysis/`, + `results/sweep/runs//analysis/` and `artifacts/analysis/` + +where `` is `xiiotid__alpha0.25__seed0__support_weighted__beta0`. + +Outputs +- `results/sweep_repro/compare_repro.json` +- a compact comparison table on stdout + +Command +```bash +python3 scripts/sweep/compare_repro.py +# or: +python3 scripts/sweep/compare_repro.py --repo-root /path/to/repo +``` + +## Notes + +- `results/` is git-ignored. These scripts write only into + `results/sweep/analysis/` and `results/sweep_repro/`; they never write into a + tracked file. +- `analyze_sweep.py` opens validation data in stage 1 and only opens test data + afterwards, by construction. The scripts do not select or recommend a beta for + the thesis; they print descriptive tables. +- The computation is unchanged from the originals. The only change is that the + repository root is a `--repo-root` argument with a relative default instead of + being derived from the script's old location inside `results/`. diff --git a/scripts/sweep/analyze_sweep.py b/scripts/sweep/analyze_sweep.py new file mode 100644 index 0000000..696ef4d --- /dev/null +++ b/scripts/sweep/analyze_sweep.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Read-only analysis of the 30-run 1-thread sweep (seed 0). + +Stage 1 (validation only) -> selection.csv + For each (dataset, alpha) choose the beta with the highest + val_rare_class_macro_f1_per_client_mean. Ties go to the smaller beta. + No test file is opened in this stage. + +Stage 2 (test, read only AFTER stage 1 is written) -> test_at_selected_beta.csv + Test rare-class macro-F1 at the beta fixed in stage 1, next to every other + candidate, for description only. + +Stage 3 (reproducibility vs. the earlier 108-run grid, seed 0) -> repro_vs_previous.csv + Same (dataset, alpha, seed 0, beta) re-run with 1 thread vs. the earlier + run in results/. Reports absolute differences; it does not select + anything. + +Nothing under planningawal/ or results/ is written. + +Inputs and outputs are rooted at --repo-root, which defaults to the repository +that contains this script (scripts/sweep/ -> repo root). No computation +depends on the root. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_REPO = SCRIPT_DIR.parents[1] + +REPO = DEFAULT_REPO +SWEEP = REPO / "results" / "sweep" +RUNS = SWEEP / "runs" +PREV = REPO / "results" +OUT = SWEEP / "analysis" + + +def configure(repo_root: Path) -> None: + """Rebind the path globals for an explicit repository root.""" + global REPO, SWEEP, RUNS, PREV, OUT + REPO = repo_root.resolve() + SWEEP = REPO / "results" / "sweep" + RUNS = SWEEP / "runs" + PREV = REPO / "results" + OUT = SWEEP / "analysis" + +DATASETS = ("xiiotid", "5gnidd") +ALPHAS = ("0.25", "0.5", "0.75") +BETAS = ("0", "0.25", "0.5", "0.75", "1") +SEL_KEY = "val_rare_class_macro_f1_per_client_mean" + + +def name(ds: str, a: str, b: str) -> str: + return f"{ds}__alpha{a}__seed0__support_weighted__beta{b}" + + +def rdir(root: Path, ds: str, a: str, b: str) -> Path: + n = name(ds, a, b) + return root / n / "results" / n if root == RUNS else root / n + + +def load(path: Path) -> dict: + with path.open(encoding="utf-8") as h: + return json.load(h) + + +def write_csv(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as h: + w = csv.DictWriter(h, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + + +def stage1() -> list[dict]: + rows = [] + for ds in DATASETS: + for a in ALPHAS: + cand = [] + for b in BETAS: + d = rdir(RUNS, ds, a, b) + if not (RUNS / name(ds, a, b) / "COMPLETE").is_file(): + sys.exit(f"missing COMPLETE: {name(ds, a, b)}") + sel = load(d / "val_selection.json") + assert sel["selection_split"] == "validation" + assert sel["test_used_for_selection"] is False + cand.append((float(sel[SEL_KEY]), -float(b), b, sel)) + best = max(cand, key=lambda t: (t[0], t[1])) + ordered = sorted(cand, key=lambda t: -t[0]) + gap = ordered[0][0] - ordered[1][0] + for v, _, b, sel in cand: + rows.append( + { + "dataset": ds, + "alpha": a, + "beta": b, + "val_rare_f1_per_client_mean": f"{v:.9f}", + "val_rare_f1_global": f"{sel['val_rare_class_macro_f1_global']:.9f}", + "val_rare_recall_per_client_mean": f"{sel['val_rare_class_recall_per_client_mean']:.9f}", + "val_macro_f1_global": f"{sel['val_macro_f1_global']:.9f}", + "selected": int(b == best[2]), + "margin_to_runner_up": f"{gap:.9f}" if b == best[2] else "", + } + ) + return rows + + +def stage2(sel_rows: list[dict]) -> list[dict]: + chosen = {(r["dataset"], r["alpha"]): r["beta"] for r in sel_rows if r["selected"]} + rows = [] + for ds in DATASETS: + for a in ALPHAS: + for b in BETAS: + t = load(rdir(RUNS, ds, a, b) / "test_global_metrics.json") + rows.append( + { + "dataset": ds, + "alpha": a, + "beta": b, + "selected_on_validation": int(chosen[(ds, a)] == b), + "test_rare_f1_global": f"{t['rare_class_macro_f1']:.9f}", + "test_rare_recall_global": f"{t['rare_class_recall']:.9f}", + "test_macro_f1": f"{t['macro_f1']:.9f}", + "test_accuracy": f"{t['accuracy']:.9f}", + "test_rare_support": t["rare_support"], + } + ) + return rows + + +def stage3() -> list[dict]: + rows = [] + for ds in DATASETS: + for a in ALPHAS: + for b in BETAS: + new = rdir(RUNS, ds, a, b) + old = rdir(PREV, ds, a, b) + if not old.is_dir(): + rows.append({"dataset": ds, "alpha": a, "beta": b, "note": "no previous run"}) + continue + nt = load(new / "test_global_metrics.json") + ot = load(old / "test_global_metrics.json") + row = { + "dataset": ds, + "alpha": a, + "beta": b, + "test_rare_f1_new": f"{nt['rare_class_macro_f1']:.9f}", + "test_rare_f1_prev": f"{ot['rare_class_macro_f1']:.9f}", + "abs_diff_test_rare_f1": f"{abs(nt['rare_class_macro_f1'] - ot['rare_class_macro_f1']):.3e}", + "abs_diff_test_macro_f1": f"{abs(nt['macro_f1'] - ot['macro_f1']):.3e}", + "abs_diff_test_accuracy": f"{abs(nt['accuracy'] - ot['accuracy']):.3e}", + "note": "", + } + nv, ov = new / "val_selection.json", old / "val_selection.json" + if nv.is_file() and ov.is_file(): + row["abs_diff_val_sel_metric"] = ( + f"{abs(load(nv)[SEL_KEY] - load(ov)[SEL_KEY]):.3e}" + ) + else: + row["abs_diff_val_sel_metric"] = "" + row["note"] = "previous run has no val_selection.json" + rows.append(row) + return rows + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repo-root", + type=Path, + default=DEFAULT_REPO, + help="repository root containing results/ (default: %(default)s)", + ) + args = parser.parse_args(argv) + configure(args.repo_root) + + s1 = stage1() + write_csv(OUT / "selection.csv", s1) + print(f"stage 1 written: {OUT / 'selection.csv'} ({len(s1)} rows); test not opened so far") + s2 = stage2(s1) + write_csv(OUT / "test_at_selected_beta.csv", s2) + s3 = stage3() + write_csv(OUT / "repro_vs_previous.csv", s3) + print(f"stage 2/3 written under {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sweep/check_noise.py b/scripts/sweep/check_noise.py new file mode 100644 index 0000000..8dff2ca --- /dev/null +++ b/scripts/sweep/check_noise.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Read-only checks on the sweep (1 thread) vs. the earlier grid (seed 0). + +Three questions, no training: + A. Where do paired runs first diverge (round-0 vs later rounds)? + B. Is the sweep-minus-grid difference systematic (sign) or scattered? + C. How stable is the beta ranking, per condition, between the two runs? + +Writes results/sweep/analysis/noise_checks.json. Exploratory only: the 30 +pairs share partitions across betas, so they are NOT independent; the sign +statistics are descriptive, not significance claims. + +Paths are rooted at --repo-root, which defaults to the repository that contains +this script (scripts/sweep/ -> repo root). No computation depends on the root. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_REPO = SCRIPT_DIR.parents[1] + +REPO = DEFAULT_REPO +SWEEP = REPO / "results" / "sweep" +RUNS = SWEEP / "runs" +PREV = REPO / "results" +OUT = SWEEP / "analysis" / "noise_checks.json" + + +def configure(repo_root: Path) -> None: + """Rebind the path globals for an explicit repository root.""" + global REPO, SWEEP, RUNS, PREV, OUT + REPO = repo_root.resolve() + SWEEP = REPO / "results" / "sweep" + RUNS = SWEEP / "runs" + PREV = REPO / "results" + OUT = SWEEP / "analysis" / "noise_checks.json" + +DATASETS = ("xiiotid", "5gnidd") +ALPHAS = ("0.25", "0.5", "0.75") +BETAS = ("0", "0.25", "0.5", "0.75", "1") +SEL = "val_rare_class_macro_f1_per_client_mean" + + +def name(ds, a, b): + return f"{ds}__alpha{a}__seed0__support_weighted__beta{b}" + + +def new_dir(ds, a, b): + n = name(ds, a, b) + return RUNS / n / "results" / n + + +def old_dir(ds, a, b): + return PREV / name(ds, a, b) + + +def jl(p): + with p.open(encoding="utf-8") as h: + return json.load(h) + + +def spearman(x, y): + def ranks(v): + order = sorted(range(len(v)), key=lambda i: v[i]) + r = [0.0] * len(v) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and v[order[j + 1]] == v[order[i]]: + j += 1 + for k in range(i, j + 1): + r[order[k]] = (i + j) / 2 + 1 + i = j + 1 + return r + + rx, ry = ranks(x), ranks(y) + mx, my = sum(rx) / len(rx), sum(ry) / len(ry) + num = sum((a - mx) * (b - my) for a, b in zip(rx, ry)) + den = math.sqrt(sum((a - mx) ** 2 for a in rx) * sum((b - my) ** 2 for b in ry)) + return num / den if den else float("nan") + + +def binom_two_sided(k, n): + """Exact two-sided sign-test p, p0 = 0.5.""" + tail = sum(math.comb(n, i) for i in range(0, min(k, n - k) + 1)) / 2**n + return min(1.0, 2 * tail) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repo-root", + type=Path, + default=DEFAULT_REPO, + help="repository root containing results/ (default: %(default)s)", + ) + args = parser.parse_args(argv) + configure(args.repo_root) + + res: dict = {"A_divergence": [], "B_sign": {}, "C_rank": []} + + # A. per-round divergence (history.json: one dict per round) + first_round_counts: dict = {} + for ds in DATASETS: + for a in ALPHAS: + for b in BETAS: + hn, ho = jl(new_dir(ds, a, b) / "history.json"), jl(old_dir(ds, a, b) / "history.json") + diffs = [ + abs(x["rare_class_macro_f1"] - y["rare_class_macro_f1"]) + for x, y in zip(hn, ho) + ] + first = next((i for i, d in enumerate(diffs) if d > 0), None) + first_round_counts[str(first)] = first_round_counts.get(str(first), 0) + 1 + res["A_divergence"].append( + {"dataset": ds, "alpha": a, "beta": b, "first_round_with_difference": first, + "round0_abs_diff_rare_f1": diffs[0], "last_round_abs_diff_rare_f1": diffs[-1], + "n_samples_equal_all_rounds": all(x["n_samples"] == y["n_samples"] for x, y in zip(hn, ho))} + ) + res["A_summary"] = {"first_round_with_difference_counts": first_round_counts} + + # B. signed differences (sweep - grid) + for key, fname, field in ( + ("test_rare_f1", "test_global_metrics.json", "rare_class_macro_f1"), + ("val_selection_metric", "val_selection.json", SEL), + ): + d = [] + for ds in DATASETS: + for a in ALPHAS: + for b in BETAS: + d.append(jl(new_dir(ds, a, b) / fname)[field] - jl(old_dir(ds, a, b) / fname)[field]) + pos = sum(1 for v in d if v > 0) + neg = sum(1 for v in d if v < 0) + res["B_sign"][key] = { + "n": len(d), "n_sweep_higher": pos, "n_sweep_lower": neg, + "mean_signed_diff": sum(d) / len(d), + "mean_abs_diff": sum(abs(v) for v in d) / len(d), + "sign_test_two_sided_p_NOT_independent_pairs": binom_two_sided(min(pos, neg), pos + neg), + } + by_ds = {} + for ds in DATASETS: + sub = [v for i, v in enumerate(d) if i // 15 == DATASETS.index(ds)] + by_ds[ds] = {"mean_signed": sum(sub) / len(sub), "mean_abs": sum(abs(v) for v in sub) / len(sub)} + res["B_sign"][key]["by_dataset"] = by_ds + + # C. ranking stability + for ds in DATASETS: + for a in ALPHAS: + s = [jl(new_dir(ds, a, b) / "val_selection.json")[SEL] for b in BETAS] + o = [jl(old_dir(ds, a, b) / "val_selection.json")[SEL] for b in BETAS] + mad = max(abs(x - y) for x, y in zip(s, o)) + best_s = max(s) + close = [BETAS[i] for i, v in enumerate(s) if best_s - v <= mad] + res["C_rank"].append( + {"dataset": ds, "alpha": a, "spearman_sweep_vs_grid_over_5_betas": spearman(s, o), + "max_abs_val_diff": mad, + "betas_within_max_diff_of_sweep_best": close} + ) + + OUT.parent.mkdir(parents=True, exist_ok=True) + with OUT.open("w", encoding="utf-8") as h: + json.dump(res, h, indent=2) + print(f"written {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sweep/compare_repro.py b/scripts/sweep/compare_repro.py new file mode 100644 index 0000000..afe78f1 --- /dev/null +++ b/scripts/sweep/compare_repro.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Compare the two repro runs (xiiotid alpha0.25 seed0 beta0) with the sweep run and the old grid run. + +Read-only. Writes results/sweep_repro/compare_repro.json and prints a compact table. + +Paths are rooted at --repo-root, which defaults to the repository that contains +this script (scripts/sweep/ -> repo root). No computation depends on the root. + +Pairs: + A_vs_sweep : a_1thread_repeat vs results/sweep run (same 1-thread setup) + B_vs_grid : b_8thread vs results/ + A_vs_grid, B_vs_sweep, sweep_vs_grid : context +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_REPO = SCRIPT_DIR.parents[1] +N = "xiiotid__alpha0.25__seed0__support_weighted__beta0" +JSON_FILES = ("history.json", "test_global_metrics.json", "val_global_metrics.json", "val_selection.json") +SEL = "val_rare_class_macro_f1_per_client_mean" + +REPO = DEFAULT_REPO +DIRS: dict[str, Path] = {} +SNAP: dict[str, Path] = {} +OUT = REPO / "results" / "sweep_repro" / "compare_repro.json" + + +def configure(repo_root: Path) -> None: + """Rebind the path globals for an explicit repository root.""" + global REPO, DIRS, SNAP, OUT + REPO = repo_root.resolve() + DIRS = { + "a_1thread_repeat": REPO / "results/sweep_repro/a_1thread_repeat/results" / N, + "b_8thread": REPO / "results/sweep_repro/b_8thread/results" / N, + "sweep": REPO / "results/sweep/runs" / N / "results" / N, + "grid": REPO / "results" / N, + } + SNAP = { + "a_1thread_repeat": REPO / "results/sweep_repro/a_1thread_repeat/analysis" / f"{N}__prototype_snapshot.json", + "b_8thread": REPO / "results/sweep_repro/b_8thread/analysis" / f"{N}__prototype_snapshot.json", + "sweep": REPO / "results/sweep/runs" / N / "analysis" / f"{N}__prototype_snapshot.json", + "grid": REPO / "artifacts/analysis" / f"{N}__prototype_snapshot.json", + } + OUT = REPO / "results" / "sweep_repro" / "compare_repro.json" + + +configure(DEFAULT_REPO) + + +def jl(p: Path): + with p.open(encoding="utf-8") as h: + return json.load(h) + + +def sha(p: Path) -> str: + return hashlib.sha256(p.read_bytes()).hexdigest() + + +def compare(x: str, y: str) -> dict: + out: dict = {} + for f in JSON_FILES: + px, py = DIRS[x] / f, DIRS[y] / f + if not (px.is_file() and py.is_file()): + out[f] = "missing in " + ("x" if not px.is_file() else "y") + continue + out[f] = {"byte_identical": sha(px) == sha(py), "json_equal": jl(px) == jl(py)} + hx, hy = jl(DIRS[x] / "history.json"), jl(DIRS[y] / "history.json") + d = [abs(a["rare_class_macro_f1"] - b["rare_class_macro_f1"]) for a, b in zip(hx, hy)] + out["history_rare_f1_abs_diff_by_round"] = d + tx, ty = jl(DIRS[x] / "test_global_metrics.json"), jl(DIRS[y] / "test_global_metrics.json") + out["test_rare_f1"] = {x: tx["rare_class_macro_f1"], y: ty["rare_class_macro_f1"], + "abs_diff": abs(tx["rare_class_macro_f1"] - ty["rare_class_macro_f1"])} + vx, vy = jl(DIRS[x] / "val_selection.json"), jl(DIRS[y] / "val_selection.json") + out["val_selection_metric"] = {x: vx[SEL], y: vy[SEL], "abs_diff": abs(vx[SEL] - vy[SEL])} + sx, sy = SNAP[x], SNAP[y] + if sx.is_file() and sy.is_file(): + out["prototype_snapshot"] = {"byte_identical": sha(sx) == sha(sy), "json_equal": jl(sx) == jl(sy)} + else: + out["prototype_snapshot"] = "missing" + return out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repo-root", + type=Path, + default=DEFAULT_REPO, + help="repository root containing results/ and artifacts/ (default: %(default)s)", + ) + args = parser.parse_args(argv) + configure(args.repo_root) + + pairs = [("a_1thread_repeat", "sweep"), ("b_8thread", "grid"), ("a_1thread_repeat", "grid"), + ("b_8thread", "sweep"), ("sweep", "grid"), ("a_1thread_repeat", "b_8thread")] + res = {f"{x}__vs__{y}": compare(x, y) for x, y in pairs} + out = OUT + out.write_text(json.dumps(res, indent=2), encoding="utf-8") + for k, v in res.items(): + t = v["test_rare_f1"]["abs_diff"] + s = v["val_selection_metric"]["abs_diff"] + h = v["history_rare_f1_abs_diff_by_round"] + same = {f: v[f]["json_equal"] for f in JSON_FILES if isinstance(v[f], dict)} + snap = v["prototype_snapshot"] + snap_s = snap["byte_identical"] if isinstance(snap, dict) else snap + print(f"{k:34s} test_rare={t:.3e} val_sel={s:.3e} round0={h[0]:.3e} last={h[-1]:.3e} " + f"all_json_equal={all(same.values())} snapshot_identical={snap_s}") + print(f"written {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sweep/compare_with_grid.py b/scripts/sweep/compare_with_grid.py new file mode 100644 index 0000000..240daac --- /dev/null +++ b/scripts/sweep/compare_with_grid.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Compare the 1-thread seed-0 validation selection with the earlier 108-run grid. + +Read-only. Uses VALIDATION metrics and the already-fixed pooled beta only; +no test file is opened. + +Columns written to analysis/compare_with_grid.csv, per (dataset, alpha): + beta_sweep_seed0 beta chosen from results/sweep (1 thread, seed 0) + beta_grid_seed0 beta chosen from the earlier grid, seed 0 only (per-seed diagnostic) + beta_grid_pooled PRIMARY pooled-validation beta of the earlier grid (3 seeds) + sweep_equals_grid_s0 whether the two seed-0 choices coincide + grid_s0_margin top-1 minus top-2 validation score of the earlier seed-0 run + sweep_margin same for the sweep + max_abs_val_diff_s0 largest |sweep - grid| validation difference over the 5 betas + sweep_pick_rank_in_grid_s0 rank (1 = best) the sweep's beta has in the grid's seed-0 ranking + +Paths are rooted at --repo-root, which defaults to the repository that contains +this script (scripts/sweep/ -> repo root). No computation depends on the root. +""" + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_REPO = SCRIPT_DIR.parents[1] + +REPO = DEFAULT_REPO +SWEEP = REPO / "results" / "sweep" +AN = REPO / "artifacts" / "analysis" +OUT = SWEEP / "analysis" / "compare_with_grid.csv" + + +def configure(repo_root: Path) -> None: + """Rebind the path globals for an explicit repository root.""" + global REPO, SWEEP, AN, OUT + REPO = repo_root.resolve() + SWEEP = REPO / "results" / "sweep" + AN = REPO / "artifacts" / "analysis" + OUT = SWEEP / "analysis" / "compare_with_grid.csv" + +DATASETS = ("xiiotid", "5gnidd") +ALPHAS = ("0.25", "0.5", "0.75") + + +def rows(path: Path) -> list[dict]: + with path.open(newline="", encoding="utf-8") as h: + return list(csv.DictReader(h)) + + +def f(x: str) -> float: + return float(x) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repo-root", + type=Path, + default=DEFAULT_REPO, + help="repository root containing results/ and artifacts/ (default: %(default)s)", + ) + args = parser.parse_args(argv) + configure(args.repo_root) + + sweep_sel = rows(SWEEP / "analysis" / "selection.csv") + master = rows(AN / "final__six_condition_master.csv") + out = [] + for ds in DATASETS: + for a in ALPHAS: + sw = {f(r["beta"]): f(r["val_rare_f1_per_client_mean"]) + for r in sweep_sel if r["dataset"] == ds and r["alpha"] == a} + path = AN / f"{ds}__alpha{a}__multiseed__validation_per_seed.csv" + gd = {f(r["beta"]): f(r["val_rare_macro_f1_per_client_mean"]) + for r in rows(path) if r["seed"] == "0"} + assert set(sw) == set(gd) == {0.0, 0.25, 0.5, 0.75, 1.0}, (ds, a, set(sw), set(gd)) + pooled = [f(r["selected_beta"]) for r in master + if r["dataset"] == ds and f(r["alpha"]) == f(a)] + assert len(pooled) == 1, (ds, a, pooled) + + def pick(d): + best = max(d.items(), key=lambda kv: (kv[1], -kv[0])) + srt = sorted(d.values(), reverse=True) + return best[0], srt[0] - srt[1] + + b_sw, m_sw = pick(sw) + b_gd, m_gd = pick(gd) + rank = 1 + sum(1 for v in gd.values() if v > gd[b_sw]) + out.append({ + "dataset": ds, "alpha": a, + "beta_sweep_seed0": b_sw, "beta_grid_seed0": b_gd, + "beta_grid_pooled": pooled[0], + "sweep_equals_grid_s0": int(b_sw == b_gd), + "sweep_equals_pooled": int(b_sw == pooled[0]), + "grid_s0_equals_pooled": int(b_gd == pooled[0]), + "sweep_margin": f"{m_sw:.6f}", "grid_s0_margin": f"{m_gd:.6f}", + "max_abs_val_diff_s0": f"{max(abs(sw[b] - gd[b]) for b in sw):.6f}", + "sweep_pick_rank_in_grid_s0": rank, + }) + OUT.parent.mkdir(parents=True, exist_ok=True) + with OUT.open("w", newline="", encoding="utf-8") as h: + w = csv.DictWriter(h, fieldnames=list(out[0])) + w.writeheader() + w.writerows(out) + print(f"written {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())