chore: version-control sweep analysis scripts
Copy analyze_sweep.py, check_noise.py, compare_with_grid.py and compare_repro.py out of the git-ignored results/ tree into scripts/sweep/. Replace the implicit script-location repository root with a --repo-root argument (relative default) without changing any computation. Outputs were verified byte-identical to the originals on a scratch copy of the data.
This commit is contained in:
willtanoe committed
1 parent
e66ecc4b59
commit
9fe584d08e
5 files changed
+697
No files matched your search
@@ -0,0 +1,99 @@
|
||||
# Sweep analysis scripts
|
||||
|
||||
Read-only analysis helpers for the completed seed-0 sweeps. They were originally
|
||||
written inside `results/sweep/` and `results/sweep_repro/` (both git-ignored);
|
||||
these copies are the version-controlled source.
|
||||
|
||||
All four scripts accept `--repo-root` (default: the repository that contains
|
||||
`scripts/sweep/`, i.e. two levels up from the script). Data is read from the
|
||||
sweep run directories and, where noted, from the earlier grid runs and
|
||||
`artifacts/analysis/`. Nothing is retrained.
|
||||
|
||||
## `analyze_sweep.py`
|
||||
|
||||
Inputs
|
||||
- `results/sweep/runs/<dataset>__alpha<alpha>__seed0__support_weighted__beta<beta>/COMPLETE`
|
||||
- `.../results/<run>/val_selection.json`
|
||||
- `.../results/<run>/test_global_metrics.json`
|
||||
- `results/<run>/val_selection.json` and `test_global_metrics.json` for the earlier
|
||||
grid runs (stage 3)
|
||||
|
||||
Outputs
|
||||
- `results/sweep/analysis/selection.csv` (validation only)
|
||||
- `results/sweep/analysis/test_at_selected_beta.csv`
|
||||
- `results/sweep/analysis/repro_vs_previous.csv`
|
||||
|
||||
Command
|
||||
```bash
|
||||
python3 scripts/sweep/analyze_sweep.py
|
||||
# or, against a different tree:
|
||||
python3 scripts/sweep/analyze_sweep.py --repo-root /path/to/repo
|
||||
```
|
||||
|
||||
## `check_noise.py`
|
||||
|
||||
Inputs
|
||||
- `results/sweep/runs/<run>/results/<run>/{history.json,val_selection.json,test_global_metrics.json}`
|
||||
- `results/<run>/{history.json,val_selection.json,test_global_metrics.json}` (earlier grid)
|
||||
|
||||
Outputs
|
||||
- `results/sweep/analysis/noise_checks.json`
|
||||
|
||||
Command
|
||||
```bash
|
||||
python3 scripts/sweep/check_noise.py
|
||||
# or:
|
||||
python3 scripts/sweep/check_noise.py --repo-root /path/to/repo
|
||||
```
|
||||
|
||||
## `compare_with_grid.py`
|
||||
|
||||
Inputs
|
||||
- `results/sweep/analysis/selection.csv` (run `analyze_sweep.py` first)
|
||||
- `artifacts/analysis/final__six_condition_master.csv`
|
||||
- `artifacts/analysis/<dataset>__alpha<alpha>__multiseed__validation_per_seed.csv`
|
||||
|
||||
Outputs
|
||||
- `results/sweep/analysis/compare_with_grid.csv`
|
||||
|
||||
Command
|
||||
```bash
|
||||
python3 scripts/sweep/compare_with_grid.py
|
||||
# or:
|
||||
python3 scripts/sweep/compare_with_grid.py --repo-root /path/to/repo
|
||||
```
|
||||
|
||||
## `compare_repro.py`
|
||||
|
||||
Inputs
|
||||
- `results/sweep_repro/a_1thread_repeat/results/<run>/`
|
||||
- `results/sweep_repro/b_8thread/results/<run>/`
|
||||
- `results/sweep/runs/<run>/results/<run>/`
|
||||
- `results/<run>/` (earlier grid)
|
||||
- prototype snapshots under `results/sweep_repro/*/analysis/`,
|
||||
`results/sweep/runs/<run>/analysis/` and `artifacts/analysis/`
|
||||
|
||||
where `<run>` is `xiiotid__alpha0.25__seed0__support_weighted__beta0`.
|
||||
|
||||
Outputs
|
||||
- `results/sweep_repro/compare_repro.json`
|
||||
- a compact comparison table on stdout
|
||||
|
||||
Command
|
||||
```bash
|
||||
python3 scripts/sweep/compare_repro.py
|
||||
# or:
|
||||
python3 scripts/sweep/compare_repro.py --repo-root /path/to/repo
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- `results/` is git-ignored. These scripts write only into
|
||||
`results/sweep/analysis/` and `results/sweep_repro/`; they never write into a
|
||||
tracked file.
|
||||
- `analyze_sweep.py` opens validation data in stage 1 and only opens test data
|
||||
afterwards, by construction. The scripts do not select or recommend a beta for
|
||||
the thesis; they print descriptive tables.
|
||||
- The computation is unchanged from the originals. The only change is that the
|
||||
repository root is a `--repo-root` argument with a relative default instead of
|
||||
being derived from the script's old location inside `results/`.
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only analysis of the 30-run 1-thread sweep (seed 0).
|
||||
|
||||
Stage 1 (validation only) -> selection.csv
|
||||
For each (dataset, alpha) choose the beta with the highest
|
||||
val_rare_class_macro_f1_per_client_mean. Ties go to the smaller beta.
|
||||
No test file is opened in this stage.
|
||||
|
||||
Stage 2 (test, read only AFTER stage 1 is written) -> test_at_selected_beta.csv
|
||||
Test rare-class macro-F1 at the beta fixed in stage 1, next to every other
|
||||
candidate, for description only.
|
||||
|
||||
Stage 3 (reproducibility vs. the earlier 108-run grid, seed 0) -> repro_vs_previous.csv
|
||||
Same (dataset, alpha, seed 0, beta) re-run with 1 thread vs. the earlier
|
||||
run in results/<name>. Reports absolute differences; it does not select
|
||||
anything.
|
||||
|
||||
Nothing under planningawal/ or results/<old runs> is written.
|
||||
|
||||
Inputs and outputs are rooted at --repo-root, which defaults to the repository
|
||||
that contains this script (scripts/sweep/ -> repo root). No computation
|
||||
depends on the root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_REPO = SCRIPT_DIR.parents[1]
|
||||
|
||||
REPO = DEFAULT_REPO
|
||||
SWEEP = REPO / "results" / "sweep"
|
||||
RUNS = SWEEP / "runs"
|
||||
PREV = REPO / "results"
|
||||
OUT = SWEEP / "analysis"
|
||||
|
||||
|
||||
def configure(repo_root: Path) -> None:
|
||||
"""Rebind the path globals for an explicit repository root."""
|
||||
global REPO, SWEEP, RUNS, PREV, OUT
|
||||
REPO = repo_root.resolve()
|
||||
SWEEP = REPO / "results" / "sweep"
|
||||
RUNS = SWEEP / "runs"
|
||||
PREV = REPO / "results"
|
||||
OUT = SWEEP / "analysis"
|
||||
|
||||
DATASETS = ("xiiotid", "5gnidd")
|
||||
ALPHAS = ("0.25", "0.5", "0.75")
|
||||
BETAS = ("0", "0.25", "0.5", "0.75", "1")
|
||||
SEL_KEY = "val_rare_class_macro_f1_per_client_mean"
|
||||
|
||||
|
||||
def name(ds: str, a: str, b: str) -> str:
|
||||
return f"{ds}__alpha{a}__seed0__support_weighted__beta{b}"
|
||||
|
||||
|
||||
def rdir(root: Path, ds: str, a: str, b: str) -> Path:
|
||||
n = name(ds, a, b)
|
||||
return root / n / "results" / n if root == RUNS else root / n
|
||||
|
||||
|
||||
def load(path: Path) -> dict:
|
||||
with path.open(encoding="utf-8") as h:
|
||||
return json.load(h)
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="", encoding="utf-8") as h:
|
||||
w = csv.DictWriter(h, fieldnames=list(rows[0]))
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
|
||||
|
||||
def stage1() -> list[dict]:
|
||||
rows = []
|
||||
for ds in DATASETS:
|
||||
for a in ALPHAS:
|
||||
cand = []
|
||||
for b in BETAS:
|
||||
d = rdir(RUNS, ds, a, b)
|
||||
if not (RUNS / name(ds, a, b) / "COMPLETE").is_file():
|
||||
sys.exit(f"missing COMPLETE: {name(ds, a, b)}")
|
||||
sel = load(d / "val_selection.json")
|
||||
assert sel["selection_split"] == "validation"
|
||||
assert sel["test_used_for_selection"] is False
|
||||
cand.append((float(sel[SEL_KEY]), -float(b), b, sel))
|
||||
best = max(cand, key=lambda t: (t[0], t[1]))
|
||||
ordered = sorted(cand, key=lambda t: -t[0])
|
||||
gap = ordered[0][0] - ordered[1][0]
|
||||
for v, _, b, sel in cand:
|
||||
rows.append(
|
||||
{
|
||||
"dataset": ds,
|
||||
"alpha": a,
|
||||
"beta": b,
|
||||
"val_rare_f1_per_client_mean": f"{v:.9f}",
|
||||
"val_rare_f1_global": f"{sel['val_rare_class_macro_f1_global']:.9f}",
|
||||
"val_rare_recall_per_client_mean": f"{sel['val_rare_class_recall_per_client_mean']:.9f}",
|
||||
"val_macro_f1_global": f"{sel['val_macro_f1_global']:.9f}",
|
||||
"selected": int(b == best[2]),
|
||||
"margin_to_runner_up": f"{gap:.9f}" if b == best[2] else "",
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def stage2(sel_rows: list[dict]) -> list[dict]:
|
||||
chosen = {(r["dataset"], r["alpha"]): r["beta"] for r in sel_rows if r["selected"]}
|
||||
rows = []
|
||||
for ds in DATASETS:
|
||||
for a in ALPHAS:
|
||||
for b in BETAS:
|
||||
t = load(rdir(RUNS, ds, a, b) / "test_global_metrics.json")
|
||||
rows.append(
|
||||
{
|
||||
"dataset": ds,
|
||||
"alpha": a,
|
||||
"beta": b,
|
||||
"selected_on_validation": int(chosen[(ds, a)] == b),
|
||||
"test_rare_f1_global": f"{t['rare_class_macro_f1']:.9f}",
|
||||
"test_rare_recall_global": f"{t['rare_class_recall']:.9f}",
|
||||
"test_macro_f1": f"{t['macro_f1']:.9f}",
|
||||
"test_accuracy": f"{t['accuracy']:.9f}",
|
||||
"test_rare_support": t["rare_support"],
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def stage3() -> list[dict]:
|
||||
rows = []
|
||||
for ds in DATASETS:
|
||||
for a in ALPHAS:
|
||||
for b in BETAS:
|
||||
new = rdir(RUNS, ds, a, b)
|
||||
old = rdir(PREV, ds, a, b)
|
||||
if not old.is_dir():
|
||||
rows.append({"dataset": ds, "alpha": a, "beta": b, "note": "no previous run"})
|
||||
continue
|
||||
nt = load(new / "test_global_metrics.json")
|
||||
ot = load(old / "test_global_metrics.json")
|
||||
row = {
|
||||
"dataset": ds,
|
||||
"alpha": a,
|
||||
"beta": b,
|
||||
"test_rare_f1_new": f"{nt['rare_class_macro_f1']:.9f}",
|
||||
"test_rare_f1_prev": f"{ot['rare_class_macro_f1']:.9f}",
|
||||
"abs_diff_test_rare_f1": f"{abs(nt['rare_class_macro_f1'] - ot['rare_class_macro_f1']):.3e}",
|
||||
"abs_diff_test_macro_f1": f"{abs(nt['macro_f1'] - ot['macro_f1']):.3e}",
|
||||
"abs_diff_test_accuracy": f"{abs(nt['accuracy'] - ot['accuracy']):.3e}",
|
||||
"note": "",
|
||||
}
|
||||
nv, ov = new / "val_selection.json", old / "val_selection.json"
|
||||
if nv.is_file() and ov.is_file():
|
||||
row["abs_diff_val_sel_metric"] = (
|
||||
f"{abs(load(nv)[SEL_KEY] - load(ov)[SEL_KEY]):.3e}"
|
||||
)
|
||||
else:
|
||||
row["abs_diff_val_sel_metric"] = ""
|
||||
row["note"] = "previous run has no val_selection.json"
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
type=Path,
|
||||
default=DEFAULT_REPO,
|
||||
help="repository root containing results/ (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
configure(args.repo_root)
|
||||
|
||||
s1 = stage1()
|
||||
write_csv(OUT / "selection.csv", s1)
|
||||
print(f"stage 1 written: {OUT / 'selection.csv'} ({len(s1)} rows); test not opened so far")
|
||||
s2 = stage2(s1)
|
||||
write_csv(OUT / "test_at_selected_beta.csv", s2)
|
||||
s3 = stage3()
|
||||
write_csv(OUT / "repro_vs_previous.csv", s3)
|
||||
print(f"stage 2/3 written under {OUT}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only checks on the sweep (1 thread) vs. the earlier grid (seed 0).
|
||||
|
||||
Three questions, no training:
|
||||
A. Where do paired runs first diverge (round-0 vs later rounds)?
|
||||
B. Is the sweep-minus-grid difference systematic (sign) or scattered?
|
||||
C. How stable is the beta ranking, per condition, between the two runs?
|
||||
|
||||
Writes results/sweep/analysis/noise_checks.json. Exploratory only: the 30
|
||||
pairs share partitions across betas, so they are NOT independent; the sign
|
||||
statistics are descriptive, not significance claims.
|
||||
|
||||
Paths are rooted at --repo-root, which defaults to the repository that contains
|
||||
this script (scripts/sweep/ -> repo root). No computation depends on the root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_REPO = SCRIPT_DIR.parents[1]
|
||||
|
||||
REPO = DEFAULT_REPO
|
||||
SWEEP = REPO / "results" / "sweep"
|
||||
RUNS = SWEEP / "runs"
|
||||
PREV = REPO / "results"
|
||||
OUT = SWEEP / "analysis" / "noise_checks.json"
|
||||
|
||||
|
||||
def configure(repo_root: Path) -> None:
|
||||
"""Rebind the path globals for an explicit repository root."""
|
||||
global REPO, SWEEP, RUNS, PREV, OUT
|
||||
REPO = repo_root.resolve()
|
||||
SWEEP = REPO / "results" / "sweep"
|
||||
RUNS = SWEEP / "runs"
|
||||
PREV = REPO / "results"
|
||||
OUT = SWEEP / "analysis" / "noise_checks.json"
|
||||
|
||||
DATASETS = ("xiiotid", "5gnidd")
|
||||
ALPHAS = ("0.25", "0.5", "0.75")
|
||||
BETAS = ("0", "0.25", "0.5", "0.75", "1")
|
||||
SEL = "val_rare_class_macro_f1_per_client_mean"
|
||||
|
||||
|
||||
def name(ds, a, b):
|
||||
return f"{ds}__alpha{a}__seed0__support_weighted__beta{b}"
|
||||
|
||||
|
||||
def new_dir(ds, a, b):
|
||||
n = name(ds, a, b)
|
||||
return RUNS / n / "results" / n
|
||||
|
||||
|
||||
def old_dir(ds, a, b):
|
||||
return PREV / name(ds, a, b)
|
||||
|
||||
|
||||
def jl(p):
|
||||
with p.open(encoding="utf-8") as h:
|
||||
return json.load(h)
|
||||
|
||||
|
||||
def spearman(x, y):
|
||||
def ranks(v):
|
||||
order = sorted(range(len(v)), key=lambda i: v[i])
|
||||
r = [0.0] * len(v)
|
||||
i = 0
|
||||
while i < len(order):
|
||||
j = i
|
||||
while j + 1 < len(order) and v[order[j + 1]] == v[order[i]]:
|
||||
j += 1
|
||||
for k in range(i, j + 1):
|
||||
r[order[k]] = (i + j) / 2 + 1
|
||||
i = j + 1
|
||||
return r
|
||||
|
||||
rx, ry = ranks(x), ranks(y)
|
||||
mx, my = sum(rx) / len(rx), sum(ry) / len(ry)
|
||||
num = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
|
||||
den = math.sqrt(sum((a - mx) ** 2 for a in rx) * sum((b - my) ** 2 for b in ry))
|
||||
return num / den if den else float("nan")
|
||||
|
||||
|
||||
def binom_two_sided(k, n):
|
||||
"""Exact two-sided sign-test p, p0 = 0.5."""
|
||||
tail = sum(math.comb(n, i) for i in range(0, min(k, n - k) + 1)) / 2**n
|
||||
return min(1.0, 2 * tail)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
type=Path,
|
||||
default=DEFAULT_REPO,
|
||||
help="repository root containing results/ (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
configure(args.repo_root)
|
||||
|
||||
res: dict = {"A_divergence": [], "B_sign": {}, "C_rank": []}
|
||||
|
||||
# A. per-round divergence (history.json: one dict per round)
|
||||
first_round_counts: dict = {}
|
||||
for ds in DATASETS:
|
||||
for a in ALPHAS:
|
||||
for b in BETAS:
|
||||
hn, ho = jl(new_dir(ds, a, b) / "history.json"), jl(old_dir(ds, a, b) / "history.json")
|
||||
diffs = [
|
||||
abs(x["rare_class_macro_f1"] - y["rare_class_macro_f1"])
|
||||
for x, y in zip(hn, ho)
|
||||
]
|
||||
first = next((i for i, d in enumerate(diffs) if d > 0), None)
|
||||
first_round_counts[str(first)] = first_round_counts.get(str(first), 0) + 1
|
||||
res["A_divergence"].append(
|
||||
{"dataset": ds, "alpha": a, "beta": b, "first_round_with_difference": first,
|
||||
"round0_abs_diff_rare_f1": diffs[0], "last_round_abs_diff_rare_f1": diffs[-1],
|
||||
"n_samples_equal_all_rounds": all(x["n_samples"] == y["n_samples"] for x, y in zip(hn, ho))}
|
||||
)
|
||||
res["A_summary"] = {"first_round_with_difference_counts": first_round_counts}
|
||||
|
||||
# B. signed differences (sweep - grid)
|
||||
for key, fname, field in (
|
||||
("test_rare_f1", "test_global_metrics.json", "rare_class_macro_f1"),
|
||||
("val_selection_metric", "val_selection.json", SEL),
|
||||
):
|
||||
d = []
|
||||
for ds in DATASETS:
|
||||
for a in ALPHAS:
|
||||
for b in BETAS:
|
||||
d.append(jl(new_dir(ds, a, b) / fname)[field] - jl(old_dir(ds, a, b) / fname)[field])
|
||||
pos = sum(1 for v in d if v > 0)
|
||||
neg = sum(1 for v in d if v < 0)
|
||||
res["B_sign"][key] = {
|
||||
"n": len(d), "n_sweep_higher": pos, "n_sweep_lower": neg,
|
||||
"mean_signed_diff": sum(d) / len(d),
|
||||
"mean_abs_diff": sum(abs(v) for v in d) / len(d),
|
||||
"sign_test_two_sided_p_NOT_independent_pairs": binom_two_sided(min(pos, neg), pos + neg),
|
||||
}
|
||||
by_ds = {}
|
||||
for ds in DATASETS:
|
||||
sub = [v for i, v in enumerate(d) if i // 15 == DATASETS.index(ds)]
|
||||
by_ds[ds] = {"mean_signed": sum(sub) / len(sub), "mean_abs": sum(abs(v) for v in sub) / len(sub)}
|
||||
res["B_sign"][key]["by_dataset"] = by_ds
|
||||
|
||||
# C. ranking stability
|
||||
for ds in DATASETS:
|
||||
for a in ALPHAS:
|
||||
s = [jl(new_dir(ds, a, b) / "val_selection.json")[SEL] for b in BETAS]
|
||||
o = [jl(old_dir(ds, a, b) / "val_selection.json")[SEL] for b in BETAS]
|
||||
mad = max(abs(x - y) for x, y in zip(s, o))
|
||||
best_s = max(s)
|
||||
close = [BETAS[i] for i, v in enumerate(s) if best_s - v <= mad]
|
||||
res["C_rank"].append(
|
||||
{"dataset": ds, "alpha": a, "spearman_sweep_vs_grid_over_5_betas": spearman(s, o),
|
||||
"max_abs_val_diff": mad,
|
||||
"betas_within_max_diff_of_sweep_best": close}
|
||||
)
|
||||
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
with OUT.open("w", encoding="utf-8") as h:
|
||||
json.dump(res, h, indent=2)
|
||||
print(f"written {OUT}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the two repro runs (xiiotid alpha0.25 seed0 beta0) with the sweep run and the old grid run.
|
||||
|
||||
Read-only. Writes results/sweep_repro/compare_repro.json and prints a compact table.
|
||||
|
||||
Paths are rooted at --repo-root, which defaults to the repository that contains
|
||||
this script (scripts/sweep/ -> repo root). No computation depends on the root.
|
||||
|
||||
Pairs:
|
||||
A_vs_sweep : a_1thread_repeat vs results/sweep run (same 1-thread setup)
|
||||
B_vs_grid : b_8thread vs results/<old grid run>
|
||||
A_vs_grid, B_vs_sweep, sweep_vs_grid : context
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_REPO = SCRIPT_DIR.parents[1]
|
||||
N = "xiiotid__alpha0.25__seed0__support_weighted__beta0"
|
||||
JSON_FILES = ("history.json", "test_global_metrics.json", "val_global_metrics.json", "val_selection.json")
|
||||
SEL = "val_rare_class_macro_f1_per_client_mean"
|
||||
|
||||
REPO = DEFAULT_REPO
|
||||
DIRS: dict[str, Path] = {}
|
||||
SNAP: dict[str, Path] = {}
|
||||
OUT = REPO / "results" / "sweep_repro" / "compare_repro.json"
|
||||
|
||||
|
||||
def configure(repo_root: Path) -> None:
|
||||
"""Rebind the path globals for an explicit repository root."""
|
||||
global REPO, DIRS, SNAP, OUT
|
||||
REPO = repo_root.resolve()
|
||||
DIRS = {
|
||||
"a_1thread_repeat": REPO / "results/sweep_repro/a_1thread_repeat/results" / N,
|
||||
"b_8thread": REPO / "results/sweep_repro/b_8thread/results" / N,
|
||||
"sweep": REPO / "results/sweep/runs" / N / "results" / N,
|
||||
"grid": REPO / "results" / N,
|
||||
}
|
||||
SNAP = {
|
||||
"a_1thread_repeat": REPO / "results/sweep_repro/a_1thread_repeat/analysis" / f"{N}__prototype_snapshot.json",
|
||||
"b_8thread": REPO / "results/sweep_repro/b_8thread/analysis" / f"{N}__prototype_snapshot.json",
|
||||
"sweep": REPO / "results/sweep/runs" / N / "analysis" / f"{N}__prototype_snapshot.json",
|
||||
"grid": REPO / "artifacts/analysis" / f"{N}__prototype_snapshot.json",
|
||||
}
|
||||
OUT = REPO / "results" / "sweep_repro" / "compare_repro.json"
|
||||
|
||||
|
||||
configure(DEFAULT_REPO)
|
||||
|
||||
|
||||
def jl(p: Path):
|
||||
with p.open(encoding="utf-8") as h:
|
||||
return json.load(h)
|
||||
|
||||
|
||||
def sha(p: Path) -> str:
|
||||
return hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def compare(x: str, y: str) -> dict:
|
||||
out: dict = {}
|
||||
for f in JSON_FILES:
|
||||
px, py = DIRS[x] / f, DIRS[y] / f
|
||||
if not (px.is_file() and py.is_file()):
|
||||
out[f] = "missing in " + ("x" if not px.is_file() else "y")
|
||||
continue
|
||||
out[f] = {"byte_identical": sha(px) == sha(py), "json_equal": jl(px) == jl(py)}
|
||||
hx, hy = jl(DIRS[x] / "history.json"), jl(DIRS[y] / "history.json")
|
||||
d = [abs(a["rare_class_macro_f1"] - b["rare_class_macro_f1"]) for a, b in zip(hx, hy)]
|
||||
out["history_rare_f1_abs_diff_by_round"] = d
|
||||
tx, ty = jl(DIRS[x] / "test_global_metrics.json"), jl(DIRS[y] / "test_global_metrics.json")
|
||||
out["test_rare_f1"] = {x: tx["rare_class_macro_f1"], y: ty["rare_class_macro_f1"],
|
||||
"abs_diff": abs(tx["rare_class_macro_f1"] - ty["rare_class_macro_f1"])}
|
||||
vx, vy = jl(DIRS[x] / "val_selection.json"), jl(DIRS[y] / "val_selection.json")
|
||||
out["val_selection_metric"] = {x: vx[SEL], y: vy[SEL], "abs_diff": abs(vx[SEL] - vy[SEL])}
|
||||
sx, sy = SNAP[x], SNAP[y]
|
||||
if sx.is_file() and sy.is_file():
|
||||
out["prototype_snapshot"] = {"byte_identical": sha(sx) == sha(sy), "json_equal": jl(sx) == jl(sy)}
|
||||
else:
|
||||
out["prototype_snapshot"] = "missing"
|
||||
return out
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
type=Path,
|
||||
default=DEFAULT_REPO,
|
||||
help="repository root containing results/ and artifacts/ (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
configure(args.repo_root)
|
||||
|
||||
pairs = [("a_1thread_repeat", "sweep"), ("b_8thread", "grid"), ("a_1thread_repeat", "grid"),
|
||||
("b_8thread", "sweep"), ("sweep", "grid"), ("a_1thread_repeat", "b_8thread")]
|
||||
res = {f"{x}__vs__{y}": compare(x, y) for x, y in pairs}
|
||||
out = OUT
|
||||
out.write_text(json.dumps(res, indent=2), encoding="utf-8")
|
||||
for k, v in res.items():
|
||||
t = v["test_rare_f1"]["abs_diff"]
|
||||
s = v["val_selection_metric"]["abs_diff"]
|
||||
h = v["history_rare_f1_abs_diff_by_round"]
|
||||
same = {f: v[f]["json_equal"] for f in JSON_FILES if isinstance(v[f], dict)}
|
||||
snap = v["prototype_snapshot"]
|
||||
snap_s = snap["byte_identical"] if isinstance(snap, dict) else snap
|
||||
print(f"{k:34s} test_rare={t:.3e} val_sel={s:.3e} round0={h[0]:.3e} last={h[-1]:.3e} "
|
||||
f"all_json_equal={all(same.values())} snapshot_identical={snap_s}")
|
||||
print(f"written {out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the 1-thread seed-0 validation selection with the earlier 108-run grid.
|
||||
|
||||
Read-only. Uses VALIDATION metrics and the already-fixed pooled beta only;
|
||||
no test file is opened.
|
||||
|
||||
Columns written to analysis/compare_with_grid.csv, per (dataset, alpha):
|
||||
beta_sweep_seed0 beta chosen from results/sweep (1 thread, seed 0)
|
||||
beta_grid_seed0 beta chosen from the earlier grid, seed 0 only (per-seed diagnostic)
|
||||
beta_grid_pooled PRIMARY pooled-validation beta of the earlier grid (3 seeds)
|
||||
sweep_equals_grid_s0 whether the two seed-0 choices coincide
|
||||
grid_s0_margin top-1 minus top-2 validation score of the earlier seed-0 run
|
||||
sweep_margin same for the sweep
|
||||
max_abs_val_diff_s0 largest |sweep - grid| validation difference over the 5 betas
|
||||
sweep_pick_rank_in_grid_s0 rank (1 = best) the sweep's beta has in the grid's seed-0 ranking
|
||||
|
||||
Paths are rooted at --repo-root, which defaults to the repository that contains
|
||||
this script (scripts/sweep/ -> repo root). No computation depends on the root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_REPO = SCRIPT_DIR.parents[1]
|
||||
|
||||
REPO = DEFAULT_REPO
|
||||
SWEEP = REPO / "results" / "sweep"
|
||||
AN = REPO / "artifacts" / "analysis"
|
||||
OUT = SWEEP / "analysis" / "compare_with_grid.csv"
|
||||
|
||||
|
||||
def configure(repo_root: Path) -> None:
|
||||
"""Rebind the path globals for an explicit repository root."""
|
||||
global REPO, SWEEP, AN, OUT
|
||||
REPO = repo_root.resolve()
|
||||
SWEEP = REPO / "results" / "sweep"
|
||||
AN = REPO / "artifacts" / "analysis"
|
||||
OUT = SWEEP / "analysis" / "compare_with_grid.csv"
|
||||
|
||||
DATASETS = ("xiiotid", "5gnidd")
|
||||
ALPHAS = ("0.25", "0.5", "0.75")
|
||||
|
||||
|
||||
def rows(path: Path) -> list[dict]:
|
||||
with path.open(newline="", encoding="utf-8") as h:
|
||||
return list(csv.DictReader(h))
|
||||
|
||||
|
||||
def f(x: str) -> float:
|
||||
return float(x)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
type=Path,
|
||||
default=DEFAULT_REPO,
|
||||
help="repository root containing results/ and artifacts/ (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
configure(args.repo_root)
|
||||
|
||||
sweep_sel = rows(SWEEP / "analysis" / "selection.csv")
|
||||
master = rows(AN / "final__six_condition_master.csv")
|
||||
out = []
|
||||
for ds in DATASETS:
|
||||
for a in ALPHAS:
|
||||
sw = {f(r["beta"]): f(r["val_rare_f1_per_client_mean"])
|
||||
for r in sweep_sel if r["dataset"] == ds and r["alpha"] == a}
|
||||
path = AN / f"{ds}__alpha{a}__multiseed__validation_per_seed.csv"
|
||||
gd = {f(r["beta"]): f(r["val_rare_macro_f1_per_client_mean"])
|
||||
for r in rows(path) if r["seed"] == "0"}
|
||||
assert set(sw) == set(gd) == {0.0, 0.25, 0.5, 0.75, 1.0}, (ds, a, set(sw), set(gd))
|
||||
pooled = [f(r["selected_beta"]) for r in master
|
||||
if r["dataset"] == ds and f(r["alpha"]) == f(a)]
|
||||
assert len(pooled) == 1, (ds, a, pooled)
|
||||
|
||||
def pick(d):
|
||||
best = max(d.items(), key=lambda kv: (kv[1], -kv[0]))
|
||||
srt = sorted(d.values(), reverse=True)
|
||||
return best[0], srt[0] - srt[1]
|
||||
|
||||
b_sw, m_sw = pick(sw)
|
||||
b_gd, m_gd = pick(gd)
|
||||
rank = 1 + sum(1 for v in gd.values() if v > gd[b_sw])
|
||||
out.append({
|
||||
"dataset": ds, "alpha": a,
|
||||
"beta_sweep_seed0": b_sw, "beta_grid_seed0": b_gd,
|
||||
"beta_grid_pooled": pooled[0],
|
||||
"sweep_equals_grid_s0": int(b_sw == b_gd),
|
||||
"sweep_equals_pooled": int(b_sw == pooled[0]),
|
||||
"grid_s0_equals_pooled": int(b_gd == pooled[0]),
|
||||
"sweep_margin": f"{m_sw:.6f}", "grid_s0_margin": f"{m_gd:.6f}",
|
||||
"max_abs_val_diff_s0": f"{max(abs(sw[b] - gd[b]) for b in sw):.6f}",
|
||||
"sweep_pick_rank_in_grid_s0": rank,
|
||||
})
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
with OUT.open("w", newline="", encoding="utf-8") as h:
|
||||
w = csv.DictWriter(h, fieldnames=list(out[0]))
|
||||
w.writeheader()
|
||||
w.writerows(out)
|
||||
print(f"written {OUT}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user