diff --git a/exploration_v2/.gitignore b/exploration_v2/.gitignore new file mode 100644 index 0000000..77263eb --- /dev/null +++ b/exploration_v2/.gitignore @@ -0,0 +1,11 @@ +# Transient driver runtime files (not part of the research record). +driver.lock +summary.lock +HEARTBEAT + +# Per-run result trees are regenerable/ignored by the repository-level +# `results/` rule; the analysis tables and REPORT_DATA.md are tracked instead. + +# Smoke-run prototype snapshots are bulky and not part of the scientific record. +smoke/analysis/*__prototype_snapshot.json +smoke/processed/ diff --git a/exploration_v2/ACCESS_LOG.txt b/exploration_v2/ACCESS_LOG.txt new file mode 100644 index 0000000..8bcc1bc --- /dev/null +++ b/exploration_v2/ACCESS_LOG.txt @@ -0,0 +1,52 @@ +2026-10-30T04:26:48Z /home/willtanoe/project/prototype-agregation/results/xiiotid__alpha0.25__seed0__original_protean/val_global_metrics.json +2026-10-30T04:26:48Z /home/willtanoe/project/prototype-agregation/results/xiiotid__alpha0.25__seed0__original_protean/val_selection.json +# ranking.py run at 2026-10-31T05:03:55Z +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean/results/xiiotid__alpha0.25__seed0__original_protean/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean/results/xiiotid__alpha0.25__seed0__original_protean/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0/results/xiiotid__alpha0.25__seed0__support_weighted__beta0/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0/results/xiiotid__alpha0.25__seed0__support_weighted__beta0/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta1/results/xiiotid__alpha0.25__seed0__support_weighted__beta1/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta1/results/xiiotid__alpha0.25__seed0__support_weighted__beta1/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean/results/5gnidd__alpha0.5__seed0__original_protean/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean/results/5gnidd__alpha0.5__seed0__original_protean/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0/results/5gnidd__alpha0.5__seed0__support_weighted__beta0/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0/results/5gnidd__alpha0.5__seed0__support_weighted__beta0/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta1/results/5gnidd__alpha0.5__seed0__support_weighted__beta1/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta1/results/5gnidd__alpha0.5__seed0__support_weighted__beta1/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/queue_screen.csv +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5/results/xiiotid__alpha0.25__seed0__v_beta__b1.5/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5/results/xiiotid__alpha0.25__seed0__v_beta__b1.5/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/results/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/results/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4/results/xiiotid__alpha0.25__seed0__v_cap__b1__c4/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4/results/xiiotid__alpha0.25__seed0__v_cap__b1__c4/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2/results/xiiotid__alpha0.25__seed0__v_cap__b1__c2/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2/results/xiiotid__alpha0.25__seed0__v_cap__b1__c2/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10/results/xiiotid__alpha0.25__seed0__v_minsup__t10/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10/results/xiiotid__alpha0.25__seed0__v_minsup__t10/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight/results/xiiotid__alpha0.25__seed0__v_logweight/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight/results/xiiotid__alpha0.25__seed0__v_logweight/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5/results/5gnidd__alpha0.5__seed0__v_beta__b1.5/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5/results/5gnidd__alpha0.5__seed0__v_beta__b1.5/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/results/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/results/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4/results/5gnidd__alpha0.5__seed0__v_cap__b1__c4/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4/results/5gnidd__alpha0.5__seed0__v_cap__b1__c4/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2/results/5gnidd__alpha0.5__seed0__v_cap__b1__c2/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2/results/5gnidd__alpha0.5__seed0__v_cap__b1__c2/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10/results/5gnidd__alpha0.5__seed0__v_minsup__t10/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10/results/5gnidd__alpha0.5__seed0__v_minsup__t10/val_selection.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight/results/5gnidd__alpha0.5__seed0__v_logweight/val_global_metrics.json +2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight/results/5gnidd__alpha0.5__seed0__v_logweight/val_selection.json diff --git a/exploration_v2/BUDGET_START b/exploration_v2/BUDGET_START new file mode 100644 index 0000000..82c7491 --- /dev/null +++ b/exploration_v2/BUDGET_START @@ -0,0 +1 @@ +2026-10-30T04:11:55Z diff --git a/exploration_v2/CANDIDATES.md b/exploration_v2/CANDIDATES.md new file mode 100644 index 0000000..2abcd2c --- /dev/null +++ b/exploration_v2/CANDIDATES.md @@ -0,0 +1,167 @@ +# Phase 3 — candidate modifications + +All candidates below are implemented **only inside `exploration_v2/`** +(`exploration_v2/scripts/variants.py`, `run_variant.py`, unit tests in +`exploration_v2/tests/`). No file under `src/`, `configs/`, `tests/`, +`scripts/`, `results/`, `artifacts/` or `planningawal/` was changed. + +Evidence codes used below are the Phase 2 measurements in +`exploration_v2/results/phase2_*.csv` (restored 108-run grid, read only). + +Baseline facts used by every candidate (measured, pooled over seeds 0–2): + +| condition | max weight share at β=1 | classes with `|S_j|50 % of a class's +weight on one client and the rare-class metric falls; (b) the deployed +per-client-mean selection metric disagrees with the global headline metric +(Phase 2: X-IIoTID α=0.25 and 5G-NIDD α=0.50). + +--- + +## C1. Extended beta grid (β = 1.5) — `v_beta` β=1.5 + +- **Mechanism.** Keep `w_i ∝ N_ij^β` but allow β > 1. Concentrates weight more + strongly than the frozen grid's endpoint. +- **Motivating evidence.** 5G-NIDD α=0.50 selected the boundary value β=1.00 and + its validation curve was still rising at the boundary (0.6872 → 0.6924), + so the frozen grid cannot tell whether the optimum lies beyond 1. Also tests + whether the X-IIoTID α=0.25 degradation is monotone in β. +- **Definition change.** Same operator, extended candidate range; not a new + method. Selection procedure unchanged apart from the extra candidate. +- **Cost.** One full run per condition, same as a normal beta run. +- **Failure modes.** At α=0.25 it should worsen concentration and rare-class + recall further; at 5G-NIDD α=0.50 the boundary gain may be seed noise. + +## C2. Shrinkage toward equal weighting — `v_shrink` β=1, λ=0.5 + +- **Mechanism.** `p_i = N_ij^β / Σ N_ij^β`; `w_i = (1−λ)/|S_j| + λ·p_i`. + λ=0 is the equal mean, λ=1 is the current method. λ=0.5 is fixed a priori. +- **Motivating evidence.** The largest losses occur where β=1 concentration is + highest (X-IIoTID α=0.25: max share 0.528; 5G-NIDD α=0.25: 0.454). Halving + the displacement should retain some support signal while bounding the tail. +- **Definition change.** New aggregation operator; would need its own name + (“λ-shrunk support-weighted aggregation”) if adopted. +- **Cost.** One run per condition. λ=0.5 is not tuned on screening data; if it + ranks top-2 it goes to confirmation unchanged. +- **Failure modes.** λ=0.5 may be too weak to prevent the α=0.25 loss, or too + strong to keep the 5G-NIDD α=0.75 gain; a single λ is unlikely to be right + everywhere. + +## C3. Weight capping at 4× median — `v_cap` β=1, c=4 + +- **Mechanism.** `w_i ∝ min(N_ij^β, c · median_k(N_kj^β))`, then normalise. + Caps the influence of a single high-support client without discarding it. +- **Motivating evidence.** In the worst cells the top client holds ~0.45–0.53 + of a class's weight; 53/162 class-cells have `|S_j|= 12 GiB` launch guard, OOM→137 mapping and +3→2→1 throttle. Two seed-3 confirmation runs were already in flight with the old +11 GiB cap; they are left to finish rather than killed. No test metric has been +read. + +## 2026-10-31T14:06:20Z — driver + +run `xiiotid__alpha0.25__seed4__original_protean` wrapper exit 1; see its logs. + +## 2026-10-31T14:06:31Z — driver + +run `xiiotid__alpha0.25__seed4__original_protean` wrapper exit 1; see its logs. + +## 2026-10-31T14:06:31Z — driver + +run `xiiotid__alpha0.25__seed4__original_protean` failed after 2 attempt(s); marked terminal failed. + +## 2026-10-31T14:06:41Z — driver + +run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` wrapper exit 1; see its logs. + +## 2026-10-31T14:06:52Z — driver + +run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` wrapper exit 1; see its logs. + +## 2026-10-31T14:06:53Z — driver + +run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` failed after 2 attempt(s); marked terminal failed. + +## 2026-10-31T14:07:03Z — driver + +run `xiiotid__alpha0.25__seed4__support_weighted__beta1` wrapper exit 1; see its logs. + +## 2026-10-31T14:07:03Z — driver + +stopping launches: failure rate 2/6 exceeds 20% + +## 2026-10-31T14:12Z — retry collision fixed (amendment A5) + +The 14:06Z failures were retries blocked by a stale transient scope unit name, +not new OOM kills. Evidence: stderr “Failed to start transient scope unit: Unit +… was already loaded” and zero-byte `time.txt`; the wrapper then read the old +scope's `Result=oom-kill`. Fixed with `reset-failed` before launch and a guard +that ignores the old Result when the scope never started. Seed-4 references are +running again with the 13 GiB cap. No test metric read. + +## 2026-10-31T14:30Z — A6: in-place wrapper edit caused exit 127 (results unaffected) + +Editing `run_one_v2.sh` while two wrappers were already running corrupted their +remaining shell commands (bash reads scripts incrementally); both exited 127 +after training had finished and were retried. The archived first attempts +(`results/attempts/*seed3*v_cap*attempt1`, `*v_shrink*attempt1`) are bit-identical +to the final runs for val/test metrics and the round-0 snapshot, so no result +changed. Lesson recorded: never edit a bash script while instances are running. diff --git a/exploration_v2/KEY_RESULTS.md b/exploration_v2/KEY_RESULTS.md new file mode 100644 index 0000000..b8de62b --- /dev/null +++ b/exploration_v2/KEY_RESULTS.md @@ -0,0 +1,78 @@ +# exploration_v2 — key results (one page) + +Status: complete. `DONE` = `status=complete`, 2026-11-01T22:35:36Z. +34/34 scientific runs, 0 terminal failures, 66.4 h of 96 h. +Plan hash `2118f31b…c978e`; ranking hash `f43ebc76…b054d3` (frozen before any +confirmatory run). + +## Question + +Which pre-specified modification to PROTEAN's server-side prototype aggregation +improves validation rare-class macro-F1 over (a) the original baseline and +(b) the current support-weighted method, and does it hold on seeds never used in +the analysis? + +## Screening (validation only; seed 0; 2 conditions) + +Score = mean over X-IIoTID α=0.25 and 5G-NIDD α=0.50 of the paired validation +difference in global rare-class macro-F1 vs the in-environment original. + +| rank | variant | score vs baseline | rank by deployed metric | +|---|---|---|---| +| 1 | `v_shrink` (β=1, λ=0.5) | **+0.0267** | 1 | +| 2 | `v_cap4` (β=1, cap 4×median) | **+0.0064** | 2 | +| 3 | `v_cap2` (cap 2×median) | −0.0024 | 3 | +| 4 | `v_logweight` | −0.0139 | 5 | +| 5 | `v_minsup10` | −0.0302 | 6 | +| 6 | `v_beta1.5` | −0.0485 | 4 | + +Top-2 margin: 0.0203. Only `v_shrink` was above the current method's best β on +validation, by +0.0038. + +## Confirmation (new seeds 3 and 4; 4 pairs) + +| variant | mean Δval vs baseline | sd | positive | mean Δtest | positive | pre-registered improved? | +|---|---|---|---|---|---|---| +| `v_shrink` | **+0.0180** | 0.0141 | **4/4** | +0.0184 | 4/4 | **yes** | +| `v_cap4` | +0.0047 | 0.0263 | 2/4 | +0.0059 | 2/4 | no | + +`v_shrink` per condition (validation, seeds 3/4): +X-IIoTID +0.0085 / +0.0329 (mean +0.0207); +5G-NIDD +0.0269 / +0.0038 (mean +0.0153). + +## Versus the current method + +Best of {original, β=0.5, β=1} per pair, validation: + +| variant | mean Δ vs current best | sd | pairs better | +|---|---|---|---| +| `v_shrink` | **−0.0153** | 0.0521 | 2/4 | +| `v_cap4` | −0.0286 | 0.0597 | 1/4 | + +The current method's β=1 is very seed-sensitive: mean −0.0163, sd 0.1040, +1/4 positive (5G-NIDD seed 3: +0.1158; X-IIoTID seed 4: −0.1384). +β=0.5 is more stable: +0.0109, sd 0.0190. + +## Bottom line + +- One modification, **`v_shrink` (half-way between equal and support-weighted)**, + met the pre-registered improvement rule over the original baseline on all four + confirmation pairs. The gain is modest (~+0.018 rare-class macro-F1) and the + test differences agree with validation. +- It does **not** beat the current method's per-pair best; it is more stable but + lower-peak. No superiority claim is made. +- The average hides heterogeneity: on X-IIoTID only 9/20 client-seed pairs + improve (mean +0.0295) and the gain is concentrated in C&C/Weaponization/ + Tampering; on 5G-NIDD 12/20 improve (mean +0.0119). See + `analysis/confirm_per_class_variant.csv` and `confirm_per_client_variant.csv`. +- The capping candidate (`v_cap4`) did not improve. Extended β=1.5, + minimum-support filtering, log-weights and 2× capping were negative in + screening. +- Four paired observations, no significance test; only two conditions and two + new seeds. See `REPORT.md` §6–8 for negative results, deviations and limits. + +## Where the numbers live + +`REPORT.md` (narrative) · `REPORT_DATA.md` (tables) · `analysis/*.csv` +(per-run/per-seed) · `ranking.json` + `ranking.sha256` · `ACCESS_LOG.txt` +(validation-only ranking reads). diff --git a/exploration_v2/PHASE12_ANALYSIS.md b/exploration_v2/PHASE12_ANALYSIS.md new file mode 100644 index 0000000..83b60c8 --- /dev/null +++ b/exploration_v2/PHASE12_ANALYSIS.md @@ -0,0 +1,351 @@ +# Phase 1 and Phase 2 — inventory and diagnosis (plain language) + +Scope: read-only analysis of the repository, the restored 108-run grid +(`results/`), the 30-run 1-thread sweep (`results/sweep/`), the reproducibility +runs (`results/sweep_repro/`), and the committed analysis tables +(`artifacts/analysis/`). No training was run for this document. Numeric tables +are reproduced in `exploration_v2/results/phase2_*.csv`. + +Labels used: **measured** (read or recomputed from an artifact), **inferred** +(a reading of measured values), **unknown** (not established by the available +files). + +--- + +## Phase 1 — what exists + +### 1.1 The research object + +The repository implements PROTEAN (Chennoufi et al., 2025) as a federated +intrusion detector. Each of 10 clients trains a small CNN on its local shard, +sends model weights and **class prototypes** (the mean embedding of each locally +observed class) to a server, and the server builds one **global prototype per +class** used for nearest-prototype classification. The thesis modifies exactly +one step: how the server combines the clients' class prototypes. + +The implemented operator is: + +``` +Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) / ( sum_{i in S_j} N_ij^beta ) +S_j = { clients with N_ij > 0 and a valid local prototype for class j } +``` + +`N_ij` is how many class-`j` training samples client `i` has. The baseline +`original_protean` uses equal weights `1/|S_j|`. The `S_j` rule itself is an +explicit **reproduction assumption**: no official PROTEAN code is available, so +the participant/denominator rule is a project decision +(`docs/research/EXPERIMENT_PROTOCOL.md`, `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). + +### 1.2 The beta grid + +`beta ∈ {0, 0.25, 0.50, 0.75, 1.00}`. The immutable specification +(`planningawal/PROTEAN_Revisi_Lurus.md`, line 78) says, in Indonesian, that +initial testing uses this set. It does **not** derive the spacing, does not say +why five points were chosen, and does not justify why not a finer or wider grid. +The choice is therefore **documented as the initial candidate set but its +rationale is undocumented**. This exploration does not invent one. The endpoints +have known semantics: β=0 is equal weighting over the shared `S_j`; β=1 is +proportional to class support. β=0 is not claimed to be bitwise identical to +`original_protean` (floating-point differences; project equivalence class B). + +### 1.3 Experiments in the repository and in the restored results + +| # | experiment | what it tests | runs | where | +|---|---|---|---|---| +| E1 | **108-run grid** | 2 datasets × 3 Dirichlet α × 3 seeds × (original + 5 betas). The main thesis experiment: does validation-selected β beat/equal the reproduced baseline on rare-class detection? | 108 | `results/__alpha<α>__seed__` (restored) | +| E2 | **30-run 1-thread sweep** | Re-runs seed 0 only at 1 thread/process, all 2×3×5 beta cells, to test deterministic threading and to sanity-check the seed-0 β selection. No `original_protean` runs. | 30 | `results/sweep/runs/` | +| E3 | **Reproducibility runs** | One 1-thread repeat and one 8-thread run for X-IIoTID α=0.25 seed 0 β=0, and for 5G-NIDD α=0.5 seed 0 β=0.75, to see whether thread count changes results. | 4 | `results/sweep_repro/{a_1thread_repeat,b_8thread}/` | +| E4 | **Smoke tests** | Tiny end-to-end runs (few clients/rounds/rows) to check the pipeline; excluded from science. | a few | `results/smoke/`, `report_smoke_run.md` | +| E5 | **Multiseed / cross-alpha / heterogeneity analyses** | Read-only reductions of E1: pooled validation curves, realized support imbalance, per-client deltas, selected-vs-original summaries. | 0 new | `scripts/analyze_*.py`, `artifacts/analysis/*multiseed*`, `*cross_alpha*`, `*partition*` | +| E6 | **Prototype-effect analysis** | Recomputes global prototypes from fixed round-0 `{C_ij, N_ij}` snapshots at every β, without retraining, and measures displacement. | 0 new | `scripts/analyze_prototype_effect.py`, 18 surviving snapshots | +| E7 | **Deep paired comparison** | original vs support_weighted(selected β) on test and validation, per seed/client/class. | 0 new | `scripts/deep_comparison.py`, `artifacts/analysis/deepcmp__*` | +| E8 | **Post-hoc statistical audit** | Sign-flip tests / paired summaries over the 3 seeds; explicitly exploratory (minimum attainable p = 0.25). | 0 new | `scripts/posthoc_statistical_audit.py`, `artifacts/analysis/posthoc__*` | +| E9 | **S_j participant-rule audit** | Counts how often `|S_j| < M` and compares the adopted `S_j` rule with a literal `1/M` hypothetical. | 0 new | `scripts/validate_sj_participant_rule.py`, `sj_*csv` | +| E10 | **β=0 equivalence audit** | Compares `original_protean` and `support_weighted` β=0 snapshots/metrics. | 0 new | `scripts/verify_beta0_equivalence.py`, `*beta0_equivalence.csv` | + +Restored-result inventory (measured): 108 run directories, 1,507 files, 36 runs +per seed {0,1,2}. One run, +`xiiotid__alpha0.5__seed0__original_protean`, is missing its five `val_*` files +and has an older `metadata.json` without the `beta` key. That run is a baseline, +not a β-selection candidate, so it does not affect β selection; it does mean that +the X-IIoTID α=0.5 baseline has no validation metrics from that directory. Only +18 prototype snapshots survived the restore (16 β=0 snapshots plus 2 +`original_protean` snapshots); the rest are represented only by committed +derived tables. + +### 1.4 Configs and scripts + +Configs (frozen): + +- `configs/datasets/datasets.yaml` — dataset paths, label columns, benign class, + dropped labels, feature/categorical/non-feature columns. +- `configs/experiments/experiment_protocol.yaml` — α grid, 10 clients, β + candidates, training hyperparameters, 70/10/20 split, seeds {0,1,2}, + selection metric. +- `configs/models/protean_cnn.yaml` — the frozen model architecture. + +Training and analysis scripts (`scripts/`): `run_baseline.py` (single run), +`generate_partitions.py`, `analyze_multiseed_beta.py`, +`analyze_cross_alpha.py`, `analyze_partition_heterogeneity.py`, +`analyze_prototype_effect.py`, `deep_comparison.py`, +`make_comparison_figures.py`, `make_final_figures.py`, +`posthoc_statistical_audit.py`, `validate_sj_participant_rule.py`, +`verify_beta0_equivalence.py`, plus `scripts/sweep/{analyze_sweep,check_noise, +compare_repro,compare_with_grid}.py`. + +Source modules (`src/protean_research/`): `data/` (loading, preprocessing, +splits, Dirichlet partition, manifest, rare/unseen, characterization, pipeline), +`federated/` (client, server, round), `aggregation/` (the two aggregation modes), +`models/` (CNN, prototypes), `evaluation/metrics.py`, `utils/`. + +Tests (`tests/`): 75 tests, including `test_baseline_aggregation.py`, +`test_support_weighted_aggregation.py`, `test_prototypes.py`, +`test_federated_smoke.py`, `test_metrics.py`, `test_data_pipeline.py`, +`test_partition.py`, `test_baseline_validation.py`, `test_prototype_effect.py`. +They cover aggregation invariants and the baseline/β=0 equivalence, not a full +sweep. + +### 1.5 MAE / MAPE + +**There is no MAE, MAPE, or any regression-error experiment or metric in this +repository.** A full-text search finds only the phrase “mean absolute +difference” in the sweep noise-check discussion +(`docs/research/SWEEP_1THREAD_SEED0_CHECK.md`, +`kisikisithesis/SWEEP_SUMMARY_REPORT.md`), which is a descriptive statistic +comparing two runs, not a model metric. Classification is evaluated with +accuracy, macro-F1, weighted-F1, per-class precision/recall/F1, and rare-class +macro-F1/recall. Any statement that MAE/MAPE was used would be false. + +--- + +## Phase 2 — why the proposed method falls below the baseline + +### 2.1 Where it falls below (measured) + +Pooled over seeds 0–2, test rare-class macro-F1 (mean): + +| condition | original | β=0 | β=0.25 | β=0.50 | β=0.75 | β=1.00 | lowest β | Δ lowest vs orig | +|---|---|---|---|---|---|---|---|---| +| X-IIoTID α=0.25 | 0.6328 | 0.6321 | 0.6368 | **0.6406** | 0.6272 | 0.5795 | 1.00 | **−0.0533** | +| X-IIoTID α=0.50 | 0.6425 | 0.6443 | 0.6574 | 0.6631 | **0.6655** | 0.6599 | 0.00 | +0.0018 | +| X-IIoTID α=0.75 | 0.6039 | 0.6059 | 0.6084 | **0.6052** | 0.6023 | 0.6024 | 0.75 | −0.0016 | +| 5G-NIDD α=0.25 | 0.8423 | 0.8412 | 0.8293 | 0.7644 | 0.7533 | **0.7817** | 0.75 | **−0.0890** | +| 5G-NIDD α=0.50 | 0.8805 | 0.8784 | 0.8573 | 0.8699 | 0.8754 | 0.8720 | 0.25 | **−0.0232** | +| 5G-NIDD α=0.75 | 0.8799 | 0.8765 | 0.8861 | **0.8967** | 0.8886 | 0.8945 | 0.00 | −0.0034 | + +Measured facts: + +- 41 of 90 (condition, seed, β>0) cells are **below** the original on test + rare-F1; 49 are above (the remaining are β=0 cells and FP ties). +- The damage is concentrated at α=0.25 in both datasets. 5G-NIDD α=0.25 is + below the baseline at **all three seeds** for β=1 (Δ −0.059, −0.104, −0.018). +- At the *pooled-selected* β the proposed method is mostly indistinguishable + from the baseline, because both α=0.25 conditions select β=0 by construction. + The large negatives appear when a positive β is used, not necessarily at the + deployed β. This nuance matters for how the result is phrased. + +### 2.2 The three-seed grid and the single-seed sweep are different experiments + +**Grid (3 seeds, pooled):** the validation-selected β is 0 at X-IIoTID α=0.25, +0 at 5G-NIDD α=0.25, 0.75 at X-IIoTID α=0.5, 1.0 at 5G-NIDD α=0.5, 0.5 at +X-IIoTID α=0.75, 0.5 at 5G-NIDD α=0.75. At the two α=0.25 cells the selected +"proposed method" is therefore equal weighting, so its difference from the +baseline is only floating-point divergence (measured |Δ| ≤ 0.001 in the pooled +table). + +**Sweep (seed 0 only, 1 thread, no baseline run):** validation-selected β is 0.75, +0.5, 0.5 (X-IIoTID α=0.25/0.5/0.75) and 0.25, 0.75, 0.25 (5G-NIDD +α=0.25/0.5/0.75). Within the sweep, selected-β test rare-F1 minus β=0 test +rare-F1 is +0.027, +0.031, **−0.020**, +0.026, +0.010, +0.004 (same order). So +the single-seed sweep shows a positive weighting effect in 5/6 cells, while the +pooled grid selects β=0 at both α=0.25 cells. These are **not contradictory**: +they are different selection units (one seed vs three pooled) and the sweep has +no `original_protean` reference and a different threading/host environment. + +Measured per-seed instability (grid): at 5G-NIDD α=0.5 the per-seed selected β +is 1.0 / 0.0 / 1.0; at X-IIoTID α=0.25 it is 0.75 / 0.5 / 0.0. The pooled +selection agrees with the per-seed choice in only 1–2 of 3 seeds per condition. +The sweep's seed-0 β equals the grid's seed-0 β in 4/6 conditions. None of this +is a significance test. + +### 2.3 Which β step hurts most (measured) + +Adjacent-step change in pooled validation per-client rare-F1: + +| condition | 0→0.25 | 0.25→0.50 | 0.50→0.75 | 0.75→1.00 | +|---|---|---|---|---| +| X-IIoTID α=0.25 | −0.003 | −0.003 | −0.006 | **−0.047** | +| X-IIoTID α=0.50 | +0.011 | +0.012 | +0.012 | −0.006 | +| X-IIoTID α=0.75 | +0.003 | +0.002 | −0.004 | −0.004 | +| 5G-NIDD α=0.25 | −0.012 | **−0.024** | −0.004 | +0.015 | +| 5G-NIDD α=0.50 | **−0.042** | +0.030 | +0.014 | +0.004 | +| 5G-NIDD α=0.75 | +0.012 | +0.019 | −0.018 | +0.013 | + +The worst single step is X-IIoTID α=0.25 from 0.75 to 1.00 (−0.047 validation, +−0.048 test) and 5G-NIDD α=0.25 from 0.25 to 0.50 (−0.024 validation, −0.065 +test). Both are sharp transitions, not a smooth trend. The response is +non-monotone in every condition. + +At β=1 the other test metrics move with rare-F1 (pooled deltas vs baseline): +X-IIoTID α=0.25 rare-F1 −0.053, rare-recall −0.114, macro-F1 −0.050, accuracy +−0.021; X-IIoTID α=0.50 +0.017 / −0.026 / +0.019 / −0.001; X-IIoTID α=0.75 +−0.002 / −0.009 / +0.005 / +0.007; 5G-NIDD α=0.25 −0.061 / −0.065 / −0.037 / +−0.011; 5G-NIDD α=0.50 −0.009 / −0.006 / −0.004 / −0.001; 5G-NIDD α=0.75 ++0.015 / +0.000 / +0.010 / +0.001. Accuracy is nearly flat while rare-F1 moves +by up to 0.06, so accuracy hides the effect. + +### 2.4 Per class: what actually degrades (measured) + +At β=1, pooled test-F1 deltas versus the baseline (mean over 3 seeds): + +- **X-IIoTID α=0.25:** Lateral_movement −0.159, Weaponization −0.139, + Tampering −0.106, C&C −0.036; the benign class and Reconnaissance are + essentially flat (+0.005, +0.003). The loss is concentrated in behavioural + attack classes, not in the majority class. +- **5G-NIDD α=0.25:** ICMPFlood F1 −0.363 with **no recall change** + (precision collapse), HTTPFlood −0.126 (recall −0.182), UDPFlood −0.047 + (recall −0.204), while Benign gains +0.096. This is the signature of the + decision regions shifting toward high-support traffic and cross-class + confusion. +- **X-IIoTID α=0.50:** C&C −0.079, Exfiltration −0.020, but Reconnaissance + +0.070 and Tampering +0.166; the aggregate is positive because gains and + losses partly cancel. +- **5G-NIDD α=0.50:** small losses (SYNFlood −0.043, SlowrateDoS −0.013) and a + small TCPConnectScan gain (+0.034); the condition-level loss is driven by + a few seeds, not by one class. + +### 2.5 Per client and per seed (measured) + +At β=1, validation per-client rare-F1 change versus the baseline (10 clients × +3 seeds; positive / negative client-seed cells): + +| condition | mean Δ | improve / degrade | +|---|---|---| +| X-IIoTID α=0.25 | −0.059 | 12 / 15 | +| X-IIoTID α=0.50 | +0.047 | 16 / 2 | +| X-IIoTID α=0.75 | −0.000 | 16 / 13 | +| 5G-NIDD α=0.25 | −0.022 | 10 / 11 | +| 5G-NIDD α=0.50 | +0.003 | 16 / 10 | +| 5G-NIDD α=0.75 | +0.019 | 14 / 11 | + +Even in the best condition some clients degrade; even in the worst some +improve. Client identities are per-seed realisations, so they are not pooled. + +Unseen classes (attack classes with `N_ij=0` for a client; never merged with +rare classes) also degrade at β=1. Mean recall over the union of unseen classes: +X-IIoTID α=0.25 −0.196, α=0.50 −0.029, α=0.75 −0.009; 5G-NIDD α=0.25 −0.017, +α=0.50 ~0.000, α=0.75 +0.000. Mean unseen F1: −0.073 (X-IIoTID α=0.25) and +−0.082 (5G-NIDD α=0.25). Table: `exploration_v2/results/phase2_unseen.csv`. + +### 2.6 Validation versus test, and the selection/headline mismatch (measured) + +The deployed selection metric is the **per-client mean** of rare-class macro-F1 +on validation. The headline metric is the **global pooled** rare-class macro-F1 +on test. These are different estimands: the per-client mean gives every client +equal weight regardless of how many rare samples it has; the global metric is +sample-weighted. + +| condition | validation ranking by per-client mean | validation ranking by global rare-F1 | test ranking | per-client ranking matches test? | +|---|---|---|---|---| +| X-IIoTID α=0.25 | 0 > .25 > .50 > .75 > 1 | **.50** > .25 > 0 > .75 > 1 | .50 > .25 > 0 > .75 > 1 | no (6/6 for global) | +| X-IIoTID α=0.50 | .75 > 1 > .50 > .25 > 0 | .75 > .50 > 1 > .25 > 0 | .75 > .50 > 1 > .25 > 0 | yes | +| X-IIoTID α=0.75 | .50 > .25 > .75 > 0 > 1 | .25 > 0 > .50 > .75 > 1 | .25 > 0 > .50 > 1 > .75 | partly | +| 5G-NIDD α=0.25 | 0 > .25 > 1 > .50 > .75 | same order | same order | yes | +| 5G-NIDD α=0.50 | **1** > .75 > 0 > .50 > .25 | **0** > .75 > 1 > .50 > .25 | 0 > .75 > 1 > .50 > .25 | no | +| 5G-NIDD α=0.75 | .50 > 1 > .75 > .25 > 0 | same order | same order | yes | + +Measured: the per-client-mean ranking matches the test ranking in 4/6 +conditions, the global-validation ranking in 6/6. The two clearest failures are +exactly the two conditions with the largest proposed-vs-baseline loss: +X-IIoTID α=0.25 (selection picks β=0, test prefers 0.50) and 5G-NIDD α=0.50 +(selection picks β=1, test prefers 0). This is a **selection-metric problem**, +not only a weighting problem. It is the direct motivation for candidate C7. + +Validation-to-test *level* gaps are small for the global metric (e.g. 5G-NIDD +α=0.25 validation 0.8424 vs test 0.8423), so the mismatch is about **ranking**, +not about a systematic validation/test scale shift. + +### 2.7 Support heterogeneity and the participant rule (measured) + +| condition | mean support CV | classes with `|S_j| Among a small set of pre-specified modifications to the server-side +> aggregation, which ones improve validation rare-class macro-F1 over the +> original baseline and over the current proposed method, and do any of them +> keep that improvement on seeds that were never used in the analysis? + +There is no target score. A modification that does not improve is a valid result +and stays in the tables. + +## 2. Frozen inputs and seeds + +- Existing partition manifests for seeds 0, 1, 2 are used **read-only** from + `artifacts/partitions/` for screening. +- Seeds 0, 1, 2 are **burned for confirmation** (Phase 2 read their test + metrics; the 108-run grid and the 30-run sweep both used them). +- Confirmatory seeds: **3 and 4**. New manifests are generated inside + `exploration_v2/partitions/` with the frozen procedure + (`protean_research.data.pipeline.create_partition`, + `per_class_dirichlet_multinomial`), never in `artifacts/partitions/`. +- Confirmatory manifest files: + `exploration_v2/partitions/xiiotid__alpha0.25__seed{3,4}.json`, + `exploration_v2/partitions/5gnidd__alpha0.5__seed{3,4}.json`. + +## 3. Screening conditions (2) + +Chosen from the Phase 2 diagnosis, before any new result is seen: + +| id | dataset | alpha | seed | why | +|---|---|---|---|---| +| S1 | xiiotid | 0.25 | 0 | most heterogeneous cell; largest β-step validation loss (0.75→1.00: −0.047); maximum weight share 0.528 | +| S2 | 5gnidd | 0.50 | 0 | validation-selected β=1 lost to the baseline on test; deployed per-client metric and global metric disagree | + +Screening uses these two conditions only. Both are existing frozen partitions; +all new runs in a condition reuse the same manifest, split and initialisation +policy, so comparisons are paired. + +## 4. Screening variants (6) and references + +| id | mode | fixed parameters | source | +|---|---|---|---| +| V1 `v_beta1.5` | `v_beta` | β=1.5 | extended grid | +| V2 `v_shrink0.5` | `v_shrink` | β=1, λ=0.5 | shrinkage toward equal | +| V3 `v_cap4` | `v_cap` | β=1, cap=4×median | capping | +| V4 `v_cap2` | `v_cap` | β=1, cap=2×median | aggressive capping (expected negative) | +| V5 `v_minsup10` | `v_minsup` | β=0, t=10 | participant-rule change (expected negative) | +| V6 `v_logweight` | `v_logweight` | β=0 | different weight function | + +Hyperparameters are fixed above and are **not** tuned on screening outcomes. + +Reference runs on the screening conditions are the **existing** seed-0 results +from the restored grid, read under identical conditions (same manifest, same +split, same initialisation policy): + +- `original_protean`; +- `support_weighted` at β ∈ {0, 0.25, 0.50, 0.75, 1.00} (the current method's + beta grid). + +No new reference training is needed for screening. + +Analysis-level candidates with **zero** training cost: + +- **C7 selection metric:** re-rank the same validation files using + `val_rare_class_macro_f1_global` instead of the deployed per-client mean. +- **C8 finer beta grid:** reported from the existing coarse grid only; no new + β points are run in this exploration. + +## 5. Screening ranking metric (validation only) + +Primary score for variant V: + +``` +score(V) = mean over c in {S1, S2} of + [ val_global_rare_f1(V, c, seed 0) + - val_global_rare_f1(original, c, seed 0) ] +``` + +where `val_global_rare_f1` is `rare_class_macro_f1` from +`val_global_metrics.json` (global pooled validation, the same definition as the +headline test metric). Ranking is by `score(V)`, descending. Tie-break: +larger minimum per-condition difference, then lexical id. + +Justification (recorded before new results): the headline metric is the global +pooled rare-class macro-F1; Phase 2 measured that the deployed per-client mean +ranks β differently in 2/6 conditions while the global validation metric +matches the test ranking in 6/6. This is a metric-definition argument, not a +test-result-driven choice. + +Secondary sensitivity ranking: the same score computed with +`val_rare_class_macro_f1_per_client_mean` from `val_selection.json` (the +deployed metric). Both rankings are reported. + +The screening analysis code reads **only** `val_*` files and logs every path it +opens to `exploration_v2/ACCESS_LOG.txt`. It must not open any +`test_*`, `per_client.csv` or `history.json` file. + +## 6. Confirmatory design + +The **top 2** variants by the primary screening score go to confirmation. + +Confirmation conditions: **S1 and S2** (same two conditions), seeds **3 and 4**. +Confirmation configurations per (condition, seed): + +1. `original_protean` (baseline reference); +2. `support_weighted` β=1.00 (canonical current method / maximal support weight); +3. `support_weighted` β=0.50 (interior point of the current beta grid); +4. top-1 variant (fixed hyperparameters above); +5. top-2 variant (fixed hyperparameters above). + +This is 5 configs × 2 conditions × 2 seeds = **20 confirmatory runs**. + +Deviation from the literal "current proposed method with its beta grid": only +β=0.50 and β=1.00 are run on the new seeds, not all five betas, because the +full grid would add 12 runs and push the planned total past the conservative +96-hour capacity (§7). The full five-beta grid is the screening reference on +the existing seeds. This truncation is recorded here before any confirmation +result exists. + +## 7. Budget, duration and maximum run count + +Measured per-run wall times from `results/sweep/summary.csv` (30 completed +1-thread runs, seed 0): + +| dataset | n | mean wall | min | max | mean peak RSS | max peak RSS | memory cap | +|---|---|---|---|---|---|---|---| +| xiiotid | 15 | 3.53 h | 2.94 h | 3.98 h | 7.02 GiB | 8.38 GiB | 11 G | +| 5gnidd | 15 | 5.49 h | 4.47 h | 6.09 h | 10.67 GiB | 13.12 GiB | 15 G | +| all | 30 | 4.51 h | | | | | | + +Allowed concurrency (from the copied `run_one.sh` caps, sum ≤ 36 GiB, ≤ 3 +processes): 3 × X-IIoTID = 33 GiB; 2 × 5G-NIDD = 30 GiB; 1 × 5G-NIDD + 1 × +X-IIoTID = 26 GiB (a third process would be 37 GiB > 36). 5G-NIDD is never +allowed above 2 concurrent. + +Conservative capacity bound (2 usable slots, mean run 4.51 h): + +``` +96 h / 4.51 h × 2 slots ≈ 42 runs +``` + +Planned: 12 screening + 20 confirmatory = **32 new runs**; 8 of the 32 are +X-IIoTID (S1) and 12 are 5G-NIDD (S2): screening 6+6, confirmation 10+10. +This leaves a margin below the bound. The run list is fixed by this plan. + +Budget clock: the first scientific run's UTC start time is written to +`exploration_v2/BUDGET_START`; no new run may be launched after +`BUDGET_START + 96 h`. + +If the budget turns out to be too small, reduce in this order: + +1. drop screening condition S2 (keep S1) → 6 screening runs; +2. drop variants in this order: V4 `v_cap2`, V5 `v_minsup10`, V6 + `v_logweight`, V3 `v_cap4`, V1 `v_beta1.5`. Never go below 3 variants; + the retained minimum set is V2 `v_shrink0.5`, V1 `v_beta1.5`, V3 `v_cap4`; +3. if still too small, reduce confirmatory conditions to S1 only (10 + confirmatory runs), keeping 2 new seeds. + +## 8. Improvement definition (confirmation) + +For variant V and condition c ∈ {S1, S2}, seed s ∈ {3, 4}: + +``` +d(c,s) = val_global_rare_f1(V, c, s) - val_global_rare_f1(original, c, s) +``` + +- **Improvement over the original baseline**: mean over the four `d(c,s)` is + strictly positive **and** every one of the four differences is strictly + positive (same sign on every confirmatory seed and condition). +- **No improvement / mixed**: any other outcome. Mixed is reported, never + recoded as success. +- Secondary paired differences vs `support_weighted` β=1.00 and β=0.50 are + reported with mean, sample standard deviation and per-seed values. They are + descriptive; no significance test and no superiority claim. + +Selection margins (best minus runner-up validation score within a condition) +are reported for the screening table. They are not used to claim a winner when +they are smaller than between-seed spread. + +## 9. Stop rules + +The driver stops launching new runs and records the reason in +`exploration_v2/ISSUES.md` and `exploration_v2/STOP_REASON` if any of: + +1. `exploration_v2/STOP` exists; +2. more than 20 % of finished runs have failed (failed / (done + failed)), with + at least 5 finished runs; +3. the 96-hour budget is exhausted; +4. two memory-related kills (exit 137) occur in a row — then lower concurrency + by one and continue (this is a throttle, not a full stop, unless the + concurrency is already 1); +5. smoke test or the unit tests fail; +6. the pre-confirmation ranking script fails to produce a valid + `ranking.json`. + +Runs already in flight are allowed to finish; the driver does not kill them. + +## 10. Freezing and honesty notes + +- The pre-confirmation ranking file is `exploration_v2/ranking.json`. Its + sha256 and UTC time are written to `exploration_v2/ranking.sha256` **before** + the first confirmatory run is launched. +- No test metric from any new run is read before that freeze. The screening + code reads only `val_*` files. +- After the freeze, confirmatory test metrics may be read by the final analysis + script only. +- Screening six variants raises the chance that the top one is lucky. The + confirmation step on unseen seeds exists precisely because of that; the + exploration does not adjust the plan after seeing confirmatory results. +- Amendments after the plan hash is recorded are allowed only as dated entries + in `exploration_v2/PLAN_AMENDMENTS.md`, each stating the reason and exactly + what had already been seen at that point. Silent edits to this file are not + permitted; the hash in `PLAN.sha256` is the frozen reference. +- No statistical significance, no superiority, no universal-beta claim will be + made. Differences are reported with sample standard deviations and per-seed + values. +- Every variant and every run is reported, including failures and negative + results. + +## 11. Exact run list + +Screening (12): + +``` +xiiotid__alpha0.25__seed0__v_beta__b1.5 +xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5 +xiiotid__alpha0.25__seed0__v_cap__b1__c4 +xiiotid__alpha0.25__seed0__v_cap__b1__c2 +xiiotid__alpha0.25__seed0__v_minsup__t10 +xiiotid__alpha0.25__seed0__v_logweight +5gnidd__alpha0.5__seed0__v_beta__b1.5 +5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5 +5gnidd__alpha0.5__seed0__v_cap__b1__c4 +5gnidd__alpha0.5__seed0__v_cap__b1__c2 +5gnidd__alpha0.5__seed0__v_minsup__t10 +5gnidd__alpha0.5__seed0__v_logweight +``` + +Confirmation (20; the two variant labels are filled from `ranking.json`): + +``` +{d}__alpha{a}__seed{s}__original_protean +{d}__alpha{a}__seed{s}__support_weighted__beta0.5 +{d}__alpha{a}__seed{s}__support_weighted__beta1 +{d}__alpha{a}__seed{s}__{top1_label} +{d}__alpha{a}__seed{s}__{top2_label} +``` + +for `(d,a) ∈ {(xiiotid,0.25),(5gnidd,0.5)}` and `s ∈ {3,4}`. + +## 12. What will be reported + +- Full Phase 1 inventory and Phase 2 plain-language diagnosis. +- Plan hash, every variant, every run (including failures), wall time and peak + memory. +- Screening ranking table with mean and per-condition paired differences over + the baseline and over the current method. +- Confirmatory per-seed differences over the baseline, β=0.5 and β=1.0, with + mean and sample standard deviation, and direction agreement. +- Negative results and deviations with timestamps. +- Explicit separation of measured / inferred / unknown statements. diff --git a/exploration_v2/PLAN.sha256 b/exploration_v2/PLAN.sha256 new file mode 100644 index 0000000..f8f3a35 --- /dev/null +++ b/exploration_v2/PLAN.sha256 @@ -0,0 +1,2 @@ +2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e PLAN.md +2026-10-30T04:07:47Z diff --git a/exploration_v2/PLAN_AMENDMENTS.md b/exploration_v2/PLAN_AMENDMENTS.md new file mode 100644 index 0000000..cef771a --- /dev/null +++ b/exploration_v2/PLAN_AMENDMENTS.md @@ -0,0 +1,114 @@ +# Plan amendments + +## 2026-10-30T04:10Z — A1 (engineering only) + +- **Reason:** `run_one_v2.sh` / `run_one_inner_v2.sh` used `$HERE/../..` as the + repository root while `$HERE` is already `exploration_v2/`, so the first six + wrapper attempts exited 127 before Python started. +- **What had been seen:** the non-scientific smoke test (PASS) and six + exit-127 wrapper failures with empty run trees. No validation or test metric + of any screening or confirmatory run existed. +- **Change:** repository root corrected to `$HERE/..`; failed attempt + directories archived under `exploration_v2/results/_driver_bug_20261030T0410Z/`. +- **Unchanged:** variants, conditions, seeds, ranking metric, budget, + concurrency, improvement definition, stop rules, PLAN.md content (hash still + valid for the methodological plan). + +## 2026-10-30T04:32Z — A2 (reference runs moved in-environment) + +- **Reason:** the plan's screening references were the restored 108-run grid, + which was produced on a different host/library environment. The project's own + reproducibility audit measured that a 1-thread rerun on this host is *not* + bitwise identical to that grid: over 30 seed-0 pairs the absolute test + rare-F1 difference has median 0.0022 and max 0.0127; a 1-thread *repeat* in + the current environment is bitwise identical to the sweep. Pairing new + variant runs against a restored baseline would therefore confound the + variant effect with an environment effect of the same order as the + differences being ranked. +- **What had been seen:** no validation or test metric from any screening or + confirmatory run existed. Only the non-scientific smoke test (PASS) and the + six archived exit-127 wrapper attempts existed. +- **Change:** two `original_protean` screening reference runs were appended to + the front of `queue_screen.csv` (X-IIoTID α=0.25 seed 0; 5G-NIDD α=0.5 seed 0) + so that the primary baseline is measured in the same environment and with the + same frozen partition as the variants. The ranking script now prefers: + 1. the new in-environment original run for the baseline; + 2. the current-environment 1-thread sweep for the β-grid reference (same host, + same partitions); + 3. the restored grid as a fallback and sensitivity reference. +- **Cost:** +2 runs (14 screening instead of 12; 34 total instead of 32). + Conservative capacity was ≈42 runs, so the budget still holds. +- **Unchanged:** variants, hyperparameters, ranking metric definition, seeds, + confirmation design, improvement definition, stop rules. The primary metric + is still “paired validation difference versus the original baseline”. + +## 2026-10-31T08:35Z — A3 (OOM recovery in confirmation) + +- **Reason:** during confirmation, `xiiotid__alpha0.25__seed4__original_protean` + and `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` were killed by + systemd cgroup OOM (`Result: oom-kill`, MemoryMax=11 GiB) after ~5m39s, twice + each. The wrapper saw exit 143 and the driver's 20 % failure-rate stop rule + fired. The plan's memory-kill rule says: two memory-related kills in a row → + lower concurrency by one and continue. These are memory kills, so the throttle + rule is the applicable one. +- **What had been seen:** 14/14 screening runs done; ranking frozen at + 05:03:55Z; 5 confirmation runs finished (3 seed-3 references); the two failed + runs produced **zero-byte stdout/stderr/time files and no result artifacts**. + No validation or test metric from the failed attempts existed, and none was + read. +- **Change:** + 1. the two OOM-killed attempts were archived to + `results/attempts/__attempt2` and requeued as clean directories; + 2. `max_procs` lowered from 3 to 2 (memory throttle); + 3. `run_one_v2.sh` now queries the transient scope's `Result` and records a + cgroup OOM as exit 137, so the driver's throttle logic sees memory kills; + 4. the failure-rate stop is skipped when the last two exits were memory kills, + so the throttle rule wins over the full stop. +- **Retry accounting:** this gives the two environmental OOM victims one more + attempt than the nominal "one retry" rule. They are references, not variants, + and no result was ever produced; the deviation is recorded here rather than + silently hidden. +- **Unchanged:** ranking (frozen), variants, seeds, metrics, improvement + definition, confirmation design, 96 h budget. + +## 2026-10-31T09:50Z — A4 (OOM prevention: cap, concurrency, host guard) + +- **Reason:** three seed-4 confirmation reference runs were killed by + `oom-kill` inside 11 GiB systemd scopes; the user asked explicitly to prevent + further OOM kills. Memory safety is an operational setting, not a scientific + parameter. +- **What had been seen:** 14/14 screening done, ranking frozen, 3/20 + confirmation done, three seed-4 references OOM-killed with zero-byte outputs. + No new test metric had been read. +- **Changes:** + 1. X-IIoTID per-run systemd cap raised from 11 GiB to **13 GiB** (5G-NIDD + stays 15 GiB). Two concurrent X-IIoTID runs therefore request 26 GiB, + still below the 36 GiB campaign budget. + 2. `MAX_PROCS` fixed at **2** (not 3). + 3. New host-level guard: the driver refuses to launch unless + `/proc/meminfo MemAvailable` is at least **12 GiB**; this prevents starting + a run while another service is squeezing the host. + 4. The wrapper maps a cgroup `oom-kill` to exit 137; two consecutive memory + kills lower `max_procs` to 1 and continue. +- **Unchanged:** ranking (frozen), variants, seeds, metrics, improvement + definition, 96 h budget, total run list. + +## 2026-10-31T14:12Z — A5 (retry unit-name collision, not a new OOM) + +- **Reason:** the 14:06Z failures were **not** new OOM kills. A failed/OOM-killed + transient systemd scope keeps its deterministic unit name loaded, so + `systemd-run --unit=` refused to start the retry with “Unit … was + already loaded”. The wrapper's new OOM check then read the *old* scope's + `Result=oom-kill` and mislabelled the failed start as 137. This consumed the + retry and tripped the 20 % failure-rate stop. +- **What had been seen:** 14/14 screening; 5/20 confirmation (all of seed-3 S1); + ranking frozen; failed retries had zero-byte logs, no artifacts, no metrics. +- **Change:** + 1. `run_one_v2.sh` now runs `systemctl --user reset-failed .scope` and a + one-second pause before every launch, so a reused unit name is free; + 2. if `stderr.log` contains “Failed to start transient scope unit”, the wrapper + does **not** map the old `Result` to an OOM; it records a failed start + instead. +- **Recovery:** the three blocked seed-4 reference attempts were archived + (`attempt2`/`attempt4`), requeued, and are running with the 13 GiB cap. +- **Unchanged:** ranking, variants, seeds, metrics, budget, improvement rule. diff --git a/exploration_v2/PLAN_CHECK.md b/exploration_v2/PLAN_CHECK.md new file mode 100644 index 0000000..78b439b --- /dev/null +++ b/exploration_v2/PLAN_CHECK.md @@ -0,0 +1,34 @@ +# PLAN self-check (written before the plan hash is recorded) + +Re-read `exploration_v2/PLAN.md` and checked each required item: + +| requirement | where satisfied | verdict | +|---|---|---| +| at most 8 variants screened | PLAN §4: 6 run-based (V1–V6) + 2 analysis-level (C7, C8) = 8 | PASS | +| documented drop order if budget too small | PLAN §7: drop S2 first; then V4, V5, V6, V3, V1 down to 3; then one confirmation condition | PASS | +| reference runs: original + current method beta grid, identical conditions | PLAN §4 (screening: existing seed-0 grid, same manifests/splits); PLAN §6 + deviation note for the truncated confirmation grid (β=0.5, 1.0) | PASS, with the truncation stated as a deviation before any confirmation result | +| per-run duration estimate from sweep logs | PLAN §7: xiiotid mean 3.53 h, 5gnidd mean 5.49 h, all mean 4.51 h, from `results/sweep/summary.csv` | PASS | +| maximum runs at allowed concurrency within 96 h | PLAN §7: conservative bound ≈42 runs; planned 32 | PASS | +| exact validation ranking metric | PLAN §5: mean over S1/S2 of paired difference in `val_rare_class_macro_f1_global` vs original; tie-break defined | PASS | +| screening conditions and seeds | PLAN §3: S1 xiiotid α=0.25 seed 0; S2 5gnidd α=0.5 seed 0 | PASS | +| separate confirmatory seeds | PLAN §2/§6: seeds 3 and 4, new manifests inside exploration_v2/partitions | PASS | +| top 2 variants go to confirmation | PLAN §6 | PASS | +| improvement definition | PLAN §8: positive mean paired difference AND strictly positive on all four confirmation pairs | PASS | +| no-improvement definition | PLAN §8: any other outcome, mixed reported separately | PASS | +| stop rules | PLAN §9: STOP file, >20 % failure, budget exhausted, two consecutive memory kills → throttle, smoke/ranking failure | PASS | +| note that many variants raise luck risk | PLAN §10 | PASS | +| amendment rule | PLAN §10: dated entries in PLAN_AMENDMENTS.md only | PASS | +| no test metrics read before ranking freeze | PLAN §5/§10; ranking script restricted to val_* with ACCESS_LOG | PASS | +| all new outputs under exploration_v2 | PLAN header; runner defaults forced inside exploration_v2 | PASS | +| new seeds via frozen partition procedure | PLAN §2; `make_manifest_v2.py` calls the project's `create_partition` | PASS | +| one thread per process | PLAN §7 and the copied `run_one.sh` capping mechanism; `thread1_launcher.py` pins torch | PASS | +| memory caps ≤36 GiB and ≤3 processes | PLAN §7; driver scheduler enforces both | PASS | +| all new variants implemented only as exploration_v2 copies with unit tests | PLAN header; `variants.py`, `run_variant.py`, `tests/test_variants.py` (14 tests) | PASS | + +Open caveat recorded deliberately: a non-scientific engineering smoke run was +executed before this self-check (3000 rows, 2 rounds, 4 clients). No screening +or confirmatory run has been launched, no new run's test metric has been read, +and no ranking decision depends on the smoke numbers. The driver repeats the +smoke after the hash. + +Decision: proceed to `sha256sum PLAN.md > PLAN.sha256` and start the driver. diff --git a/exploration_v2/README.md b/exploration_v2/README.md new file mode 100644 index 0000000..47b873d --- /dev/null +++ b/exploration_v2/README.md @@ -0,0 +1,45 @@ +# exploration_v2 + +Diagnostic follow-up to the completed 108-run PROTEAN beta grid. Everything in +this directory is new; the tracked research code, configs, tests, scripts, +results, partition manifests and the immutable research specification are +untouched. + +## Layout + +| path | purpose | +|---|---| +| `PLAN.md` | pre-registered plan (hash in `PLAN.sha256`) | +| `PLAN_CHECK.md` | self-check against the task constraints | +| `CANDIDATES.md` | Phase 3 candidate catalogue | +| `scripts/` | copied runner, variant implementations, ranking/analysis, driver | +| `tests/` | unit tests for the variants (baseline and β=0 unchanged) | +| `queue_screen.csv` | 12 screening runs | +| `queue_confirm.csv` | generated after the ranking freeze (20 runs) | +| `partitions/` | new-seed manifests (seeds 3 and 4) | +| `results/runs/` | per-run outputs, logs, COMPLETE markers | +| `analysis/` | final CSV tables | +| `REPORT_DATA.md` | all generated tables (written after confirmation) | +| `REPORT.md` | narrative report (Phase 6) | +| `ACCESS_LOG.txt` | every file opened by the validation-only ranking script | +| `HEARTBEAT`, `state.json`, `summary.csv` | driver state | +| `ISSUES.md` | failures, deviations and permission problems | +| `STOP`, `DONE` | stop request / completion marker | + +## Entry points + +```bash +# start / resume the detached driver in tmux session explore_v2 +bash exploration_v2/resume.sh + +# short status (returns immediately) +bash exploration_v2/status.sh + +# manual re-run of one stage (only when the driver is not running) +.venv/bin/python exploration_v2/scripts/rank_screening.py +.venv/bin/python exploration_v2/scripts/analyze_results.py +``` + +The driver is idempotent: it reconstructs queue state from `COMPLETE`, +`.failed`, `.running` and live systemd scopes, retries a failed run once from a +clean directory, and never kills a process it did not start. diff --git a/exploration_v2/REPORT.md b/exploration_v2/REPORT.md new file mode 100644 index 0000000..a595847 --- /dev/null +++ b/exploration_v2/REPORT.md @@ -0,0 +1,662 @@ +# exploration_v2 — final report + +Status: **complete**. The driver finished on 2026-11-01T22:35:36Z with +**34/34 scientific runs done, 0 terminal failures**, in **66.4 h** of the 96 h +budget (`DONE` has `"status": "complete"`). This narrative covers Phase 1 +(inventory), Phase 2 (diagnosis), the pre-registered design, and the screening +and confirmatory results. All numeric tables are in +`exploration_v2/REPORT_DATA.md` and `exploration_v2/analysis/*.csv`. + +Plan hash: `2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e` +(recorded 2026-10-30T04:07:47Z in `exploration_v2/PLAN.sha256`). +Ranking hash: `f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3` +(recorded 2026-10-31T05:03:55Z in `exploration_v2/ranking.sha256`, before any +confirmatory run). + +Labels: **measured** = read/recomputed from a file; **inferred** = a reading of +measured values; **unknown** = not established. + +--- + +## 1. Phase 1 — what the repository contains and what each experiment tests + +### 1.1 The method under study + +PROTEAN is a federated intrusion detector. Ten clients train a CNN locally and +send the server (a) model weights and (b) one **prototype per locally observed +class** — the mean embedding of that class's training samples. The server +aggregates the local prototypes into one global prototype per class, and +classification is nearest global prototype. The thesis changes only the +server-side aggregation: + +``` +Cbar_j(beta) = sum_{i in S_j} N_ij^beta C_ij / sum_{i in S_j} N_ij^beta +S_j = { clients with N_ij > 0 and a valid local prototype for class j } +``` + +`N_ij` is the class-specific training support. The baseline +(`original_protean`) uses equal weights `1/|S_j|`. The `S_j` participant rule is +an explicit reproduction assumption, because no official PROTEAN code is +available. `N_ij` is support, not quality; nothing here assumes a larger +support means a better prototype. + +### 1.2 The beta grid + +The candidates are β ∈ {0, 0.25, 0.50, 0.75, 1.00}. The immutable specification +(`planningawal/PROTEAN_Revisi_Lurus.md`, line 78) names this set as the initial +test grid. It gives **no derivation of the spacing and no rationale for why +these five points** rather than a finer or wider grid. The endpoints have known +semantics (β=0 equal weighting; β=1 support-proportional), but the choice of +the intermediate points is **undocumented**. This report does not invent a +rationale. + +### 1.3 Experiments (measured inventory) + +| experiment | what it tests | runs | evidence | +|---|---|---|---| +| 108-run grid | 2 datasets × 3 α × 3 seeds × (baseline + 5 β): does validation-selected β change rare-class detection? | 108 | restored `results/` | +| 30-run 1-thread sweep | seed 0 only, 2×3×5 β cells, deterministic threading and seed-0 selection | 30 | `results/sweep/` | +| reproducibility runs | 1-thread repeat vs 8-thread on two cells | 4 | `results/sweep_repro/` | +| smoke tests | pipeline sanity only, excluded from science | several | `results/smoke/`, `report_smoke_run.md` | +| multiseed / cross-alpha / heterogeneity analyses | reductions of the grid | 0 | `scripts/analyze_*.py`, `artifacts/analysis/*` | +| prototype-effect analysis | recomputes prototypes from fixed round-0 `{C_ij,N_ij}` snapshots | 0 | `scripts/analyze_prototype_effect.py` | +| deep paired comparison | original vs selected-β per seed/client/class | 0 | `scripts/deep_comparison.py` | +| post-hoc audit | exploratory sign-flip tests, n=3 | 0 | `scripts/posthoc_statistical_audit.py` | +| `S_j` audit | how often `|S_j| < M`; literal `1/M` comparison | 0 | `scripts/validate_sj_participant_rule.py` | +| β=0 equivalence | baseline vs β=0 snapshots/metrics | 0 | `scripts/verify_beta0_equivalence.py` | + +Restored results: 108 directories, 1,507 files, 36 per seed {0,1,2}. One +baseline run (`xiiotid__alpha0.5__seed0__original_protean`) lacks its five +`val_*` files and has an older metadata schema; it is not used for β selection. +Only 18 prototype snapshots survived the restore; most displacement claims rely +on the committed derived tables. + +Configs: `configs/datasets/datasets.yaml`, `configs/experiments/ +experiment_protocol.yaml`, `configs/models/protean_cnn.yaml`. +Scripts: `scripts/{run_baseline,generate_partitions,analyze_multiseed_beta, +analyze_cross_alpha,analyze_partition_heterogeneity,analyze_prototype_effect, +deep_comparison,make_comparison_figures,make_final_figures, +posthoc_statistical_audit,validate_sj_participant_rule, +verify_beta0_equivalence}.py` and `scripts/sweep/*.py`. +Tests: 75 tests covering aggregation invariants, the data pipeline, metrics and +smoke training. + +### 1.4 MAE / MAPE + +**No MAE, MAPE or any regression-error experiment or metric exists in this +repository.** A full-text search finds only the phrase “mean absolute +difference” in the sweep noise-check text, which compares two runs and is not a +model metric. Evaluation uses accuracy, macro-F1, weighted-F1, per-class +precision/recall/F1, and rare-class macro-F1/recall. Saying otherwise would be +false. + +--- + +## 2. Phase 2 — why the proposed method scores below the baseline + +All numbers below are **measured** from the restored grid / committed tables; +tables are in `exploration_v2/results/phase2_*.csv` and the long form in +`exploration_v2/PHASE12_ANALYSIS.md`. + +### 2.1 Where it loses + +Pooled test rare-class macro-F1 over seeds 0–2: + +| condition | original | β=0 | β=0.25 | β=0.50 | β=0.75 | β=1.00 | worst β | Δ worst vs orig | +|---|---|---|---|---|---|---|---|---| +| X-IIoTID α=0.25 | 0.6328 | 0.6321 | 0.6368 | **0.6406** | 0.6272 | 0.5795 | 1.00 | **−0.0533** | +| X-IIoTID α=0.50 | 0.6425 | 0.6443 | 0.6574 | 0.6631 | **0.6655** | 0.6599 | 0.00 | +0.0018 | +| X-IIoTID α=0.75 | 0.6039 | 0.6059 | 0.6084 | **0.6052** | 0.6023 | 0.6024 | 0.75 | −0.0016 | +| 5G-NIDD α=0.25 | 0.8423 | 0.8412 | 0.8293 | 0.7644 | 0.7533 | **0.7817** | 0.75 | **−0.0890** | +| 5G-NIDD α=0.50 | 0.8805 | 0.8784 | 0.8573 | 0.8699 | 0.8754 | 0.8720 | 0.25 | **−0.0232** | +| 5G-NIDD α=0.75 | 0.8799 | 0.8765 | 0.8861 | **0.8967** | 0.8886 | 0.8945 | 0.00 | −0.0034 | + +41 of 90 (condition, seed, β>0) cells are below the original; 49 are above. +5G-NIDD α=0.25 is below at all three seeds for β=1 (Δ −0.059, −0.104, −0.018). +At the **pooled-selected** β the proposed method is ~equal to the baseline at +both α=0.25 cells because those cells select β=0; the large losses appear when +a positive β is used. This distinction must be kept in every claim. + +### 2.2 The three-seed grid and the single-seed sweep are separate evidence + +- Grid (3 seeds, pooled): selected β = 0, 0.75, 0.50 (X-IIoTID α = .25/.50/.75) + and 0, 1.00, 0.50 (5G-NIDD). At both α=0.25 cells the selected method is equal + weighting, so its difference from the baseline is floating-point only. +- Sweep (seed 0, 1 thread, no baseline run): selected β = 0.75, 0.50, 0.50 and + 0.25, 0.75, 0.25. Within the sweep, selected-β minus β=0 test rare-F1 is + +0.027, +0.031, **−0.020**, +0.026, +0.010, +0.004. So the single seed gives a + positive weighting effect in 5/6 cells, unlike the pooled grid at α=0.25. + They are not contradictory: different selection units, and the sweep has no + `original_protean` reference. Per-seed grid selections are unstable (e.g. + 5G-NIDD α=0.50: 1.0 / 0.0 / 1.0; X-IIoTID α=0.25: 0.75 / 0.5 / 0.0). + +### 2.3 Which β step hurts most + +Validation per-client rare-F1 adjacent-step change: + +| condition | 0→.25 | .25→.50 | .50→.75 | .75→1.00 | +|---|---|---|---|---| +| X-IIoTID α=0.25 | −0.003 | −0.003 | −0.006 | **−0.047** | +| X-IIoTID α=0.50 | +0.011 | +0.012 | +0.012 | −0.006 | +| X-IIoTID α=0.75 | +0.003 | +0.002 | −0.004 | −0.004 | +| 5G-NIDD α=0.25 | −0.012 | **−0.024** | −0.004 | +0.015 | +| 5G-NIDD α=0.50 | **−0.042** | +0.030 | +0.014 | +0.004 | +| 5G-NIDD α=0.75 | +0.012 | +0.019 | −0.018 | +0.013 | + +The worst steps are X-IIoTID α=0.25 0.75→1.00 (−0.047 validation, −0.048 test) +and 5G-NIDD α=0.25 0.25→0.50 (−0.024 validation, −0.065 test). Response is +non-monotone everywhere. + +At β=1 the other metrics move with the same sign as rare-F1 (pooled test deltas +versus the baseline): + +| condition | rare-F1 | rare-recall | macro-F1 | accuracy | +|---|---|---|---|---| +| X-IIoTID α=0.25 | −0.053 | −0.114 | −0.050 | −0.021 | +| X-IIoTID α=0.50 | +0.017 | −0.026 | +0.019 | −0.001 | +| X-IIoTID α=0.75 | −0.002 | −0.009 | +0.005 | +0.007 | +| 5G-NIDD α=0.25 | −0.061 | −0.065 | −0.037 | −0.011 | +| 5G-NIDD α=0.50 | −0.009 | −0.006 | −0.004 | −0.001 | +| 5G-NIDD α=0.75 | +0.015 | +0.000 | +0.010 | +0.001 | + +The key asymmetry: in X-IIoTID α=0.50 rare-F1 rises while rare-recall falls +(+precision, −recall). Overall accuracy barely moves even when rare-F1 moves by +0.05–0.06, which is why accuracy alone hides the effect. + +### 2.4 Per class, per client, per seed + +- X-IIoTID α=0.25 at β=1: Lateral_movement −0.159 F1, Weaponization −0.139, + Tampering −0.106, C&C −0.036; benign +0.005, Reconnaissance +0.003. +- 5G-NIDD α=0.25 at β=1: ICMPFlood −0.363 F1 with **no recall change** + (precision collapse), HTTPFlood −0.126 (recall −0.182), UDPFlood −0.047 + (recall −0.204); benign +0.096. The decision regions shift toward + high-support traffic. +- X-IIoTID α=0.50 at β=1: C&C −0.079, Exfiltration −0.020, but + Reconnaissance +0.070, Tampering +0.166; gains and losses partly cancel. +- Per-client β=1 validation deltas: worst condition X-IIoTID α=0.25 mean −0.059 + (12 client-seed cells improve, 15 degrade); best X-IIoTID α=0.50 +0.047 + (16 improve, 2 degrade). Even the worst condition has improving clients and + the best has degrading clients. Client identities are per-seed realisations. +- Unseen-class behaviour (attack classes with `N_ij=0` for a client; **not** + merged with rare classes): at β=1 the mean recall over the union of unseen + classes falls by 0.196 on X-IIoTID α=0.25, 0.029 on X-IIoTID α=0.50, 0.009 on + X-IIoTID α=0.75, 0.017 on 5G-NIDD α=0.25, and is ~0 on the other two + conditions; mean unseen F1 falls by 0.073 on X-IIoTID α=0.25 and 0.082 on + 5G-NIDD α=0.25. The weighting also damages classes a client never trained on, + not only its two designated rare classes. Full table: + `exploration_v2/results/phase2_unseen.csv`. + +### 2.5 Validation vs test and the selection/headline mismatch +The deployed selector maximises the **per-client mean** validation rare-F1; the +headline is the **global pooled** test rare-F1. The two rankings agree with the +test ranking in 4/6 and 6/6 conditions respectively. The two failures are the +two conditions with the largest proposed-vs-baseline loss: + +- X-IIoTID α=0.25: per-client selector picks β=0; global validation (and test) + prefer β=0.50. +- 5G-NIDD α=0.50: per-client selector picks β=1; global validation (and test) + prefer β=0 (the baseline). + +Validation-to-test *levels* are close for the global metric (e.g. 5G-NIDD +α=0.25 validation 0.8424 vs test 0.8423), so the problem is ranking, not a +global scale shift. This is the direct motivation for the selection-metric +candidate. + +### 2.6 Support heterogeneity and the participant rule + +| condition | mean support CV | classes `|S_j| +## 5. Results of the new runs + +### 5.1 Screening ranking (validation only) + +Top-1 minus top-2 validation score (selection margin): 0.020278 (top2 = ['v_shrink__b1__lam0.5', 'v_cap__b1__c4']). + +Score = mean over S1,S2 of the paired validation difference in global rare-class macro-F1 versus the in-environment original baseline. `rank_secondary` uses the deployed per-client-mean metric. + +| rank | variant | β | λ | cap | min N | score | min cond Δ | S1 Δ | S2 Δ | rank_secondary | +|---|---|---|---|---|---|---|---|---|---| +| 1 | `v_shrink__b1__lam0.5` | 1.00 | 0.50 | | | 0.0267 | 0.0142 | 0.0391 | 0.0142 | 1 | +| 2 | `v_cap__b1__c4` | 1.00 | | 4.0 | | 0.0064 | 0.0035 | 0.0035 | 0.0093 | 2 | +| 3 | `v_cap__b1__c2` | 1.00 | | 2.0 | | -0.0024 | -0.0056 | -0.0056 | 0.0007 | 3 | +| 4 | `v_logweight` | 0.00 | | | | -0.0139 | -0.0346 | 0.0067 | -0.0346 | 5 | +| 5 | `v_minsup__t10` | 0.00 | | | 10 | -0.0302 | -0.0438 | -0.0166 | -0.0438 | 6 | +| 6 | `v_beta__b1.5` | 1.50 | | | | -0.0485 | -0.0918 | -0.0918 | -0.0052 | 4 | + +### 5.2 Confirmatory runs, per seed + +| dataset | α | seed | config | val rare-F1 | test rare-F1 | Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 | +|---|---|---|---|---|---|---|---|---|---| +| xiiotid | 0.25 | 3 | `v_shrink__b1__lam0.5` | 0.6605 | 0.6611 | 0.0085 | 0.0094 | 0.0261 | 0.0241 | +| xiiotid | 0.25 | 3 | `v_cap__b1__c4` | 0.6930 | 0.6929 | 0.0410 | 0.0413 | 0.0587 | 0.0559 | +| xiiotid | 0.25 | 3 | `support_weighted__beta0.5` | 0.6695 | 0.6697 | 0.0175 | 0.0181 | 0.0352 | 0.0327 | +| xiiotid | 0.25 | 3 | `support_weighted__beta1` | 0.6343 | 0.6370 | -0.0177 | -0.0146 | 0.0000 | 0.0000 | +| xiiotid | 0.25 | 4 | `v_shrink__b1__lam0.5` | 0.7155 | 0.7164 | 0.0329 | 0.0332 | 0.1713 | 0.1745 | +| xiiotid | 0.25 | 4 | `v_cap__b1__c4` | 0.6612 | 0.6628 | -0.0214 | -0.0204 | 0.1170 | 0.1209 | +| xiiotid | 0.25 | 4 | `support_weighted__beta0.5` | 0.6745 | 0.6745 | -0.0082 | -0.0087 | 0.1302 | 0.1326 | +| xiiotid | 0.25 | 4 | `support_weighted__beta1` | 0.5443 | 0.5419 | -0.1384 | -0.1413 | 0.0000 | 0.0000 | +| 5gnidd | 0.50 | 3 | `v_shrink__b1__lam0.5` | 0.8314 | 0.8276 | 0.0269 | 0.0276 | -0.0888 | -0.0895 | +| 5gnidd | 0.50 | 3 | `v_cap__b1__c4` | 0.8064 | 0.8044 | 0.0020 | 0.0044 | -0.1138 | -0.1128 | +| 5gnidd | 0.50 | 3 | `support_weighted__beta0.5` | 0.8389 | 0.8370 | 0.0344 | 0.0370 | -0.0813 | -0.0801 | +| 5gnidd | 0.50 | 3 | `support_weighted__beta1` | 0.9202 | 0.9172 | 0.1158 | 0.1172 | 0.0000 | 0.0000 | +| 5gnidd | 0.50 | 4 | `v_shrink__b1__lam0.5` | 0.9297 | 0.9265 | 0.0038 | 0.0032 | 0.0285 | 0.0262 | +| 5gnidd | 0.50 | 4 | `v_cap__b1__c4` | 0.9231 | 0.9216 | -0.0029 | -0.0017 | 0.0219 | 0.0213 | +| 5gnidd | 0.50 | 4 | `support_weighted__beta0.5` | 0.9258 | 0.9255 | -0.0001 | 0.0023 | 0.0246 | 0.0253 | +| 5gnidd | 0.50 | 4 | `support_weighted__beta1` | 0.9012 | 0.9002 | -0.0248 | -0.0231 | 0.0000 | 0.0000 | + +### 5.3 Confirmatory summary + +A variant “improves” under the pre-registered rule only if the mean validation difference over the four pairs is positive **and** all four are positive. + +| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | mean Δtest vs current best | improved | +|---|---|---|---|---|---|---|---|---| +| `v_shrink__b1__lam0.5` | 0.0180 | 0.0141 | 4/4 | 0.0184 | 0.0143 | 0.0343 | 0.0338 | -0.0153 | -0.0155 | True | +| `v_cap__b1__c4` | 0.0047 | 0.0263 | 2/4 | 0.0059 | 0.0258 | 0.0210 | 0.0213 | -0.0286 | -0.0279 | False | + +### 5.4 Reference configurations on the confirmation seeds + +| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive | +|---|---|---|---|---|---|---| +| `support_weighted__beta0.5` | 0.0109 | 0.0190 | 2 | 0.0122 | 0.0199 | 3 | +| `support_weighted__beta1` | -0.0163 | 0.1040 | 1 | -0.0155 | 0.1057 | 1 | + +### 5.5 Reference consistency (environment check) + +| condition | new original val | restored original val | sweep β=0 val | new − restored | new − sweep β=0 | +|---|---|---|---|---|---| +| S1 | 0.6380 | 0.6416 | 0.6433 | -0.0035 | -0.0052 | +| S2 | 0.8615 | 0.8694 | 0.8677 | -0.0079 | -0.0061 | + +### 5.6 Wall time and memory + +| phase | run | status | wall h | peak GiB | +|---|---|---|---|---| +| screening | `xiiotid__alpha0.25__seed0__original_protean` | done | 3.02 | 8.38 | +| screening | `5gnidd__alpha0.5__seed0__original_protean` | done | 4.67 | 10.30 | +| screening | `xiiotid__alpha0.25__seed0__v_beta__b1.5` | done | 3.07 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5` | done | 3.05 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c4` | done | 3.06 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c2` | done | 3.04 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_minsup__t10` | done | 3.03 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_logweight` | done | 2.96 | 8.38 | +| screening | `5gnidd__alpha0.5__seed0__v_beta__b1.5` | done | 4.88 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5` | done | 4.85 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c4` | done | 4.55 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c2` | done | 4.41 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_minsup__t10` | done | 4.51 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_logweight` | done | 4.82 | 10.31 | +| confirm | `xiiotid__alpha0.25__seed3__original_protean` | done | 3.08 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta0.5` | done | 3.08 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta1` | done | 3.12 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5` | done | 2.97 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__v_cap__b1__c4` | done | 2.95 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed4__original_protean` | done | 3.11 | 12.75 | +| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` | done | 3.10 | 12.76 | +| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta1` | done | 3.20 | 12.76 | +| confirm | `xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5` | done | 3.18 | 12.76 | +| confirm | `xiiotid__alpha0.25__seed4__v_cap__b1__c4` | done | 2.82 | 12.76 | +| confirm | `5gnidd__alpha0.5__seed3__original_protean` | done | 4.41 | 9.04 | +| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta0.5` | done | 4.48 | 9.07 | +| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta1` | done | 4.66 | 9.04 | +| confirm | `5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5` | done | 4.57 | 9.06 | +| confirm | `5gnidd__alpha0.5__seed3__v_cap__b1__c4` | done | 4.73 | 9.05 | +| confirm | `5gnidd__alpha0.5__seed4__original_protean` | done | 4.81 | 11.12 | +| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta0.5` | done | 4.77 | 11.16 | +| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta1` | done | 4.88 | 11.13 | +| confirm | `5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5` | done | 4.79 | 11.15 | +| confirm | `5gnidd__alpha0.5__seed4__v_cap__b1__c4` | done | 4.56 | 11.14 | + +### 5.7 Failed or missing runs + +None. + + + +### 5.8 What the confirmation actually shows + +**Screening (validation only, seed 0).** `v_shrink` (λ=0.5) ranked first +(+0.0267 mean over S1/S2 vs the in-environment baseline) and `v_cap4` second +(+0.0064), with a selection margin of 0.0203 between them. The remaining four +candidates were at or below the baseline on the two-condition mean: `v_cap2` −0.0024, +`v_logweight` −0.0139, `v_minsup` −0.0302, `v_beta1.5` −0.0485. + +The same screening scores expressed against the current method's best β on each +condition (β=0.75 for S1, +0.0310 vs baseline; β=1.00 for S2, +0.0147; mean ++0.0229): + +| variant | Δ vs original (mean) | Δ vs current-method best (mean) | +|---|---|---| +| `v_shrink` | +0.0267 | **+0.0038** | +| `v_cap4` | +0.0064 | −0.0165 | +| `v_cap2` | −0.0024 | −0.0253 | +| `v_logweight` | −0.0139 | −0.0368 | +| `v_minsup10` | −0.0302 | −0.0531 | +| `v_beta1.5` | −0.0485 | −0.0714 | + +Only `v_shrink` is above the current method's validation-selected best in this +screening comparison, and by a small margin (+0.0038). The confirmation then +shows that this small margin does not survive as a general advantage (§5.8). + +**Top candidate, `v_shrink` (λ=0.5).** It meets the pre-registered improvement +rule against the original baseline: + +| statistic (4 confirmatory pairs) | validation | test | +|---|---|---| +| mean paired difference vs baseline | **+0.0180** | **+0.0184** | +| sample sd of paired differences | 0.0141 | 0.0143 | +| positive pairs | **4/4** | **4/4** | +| X-IIoTID α=0.25, mean (seeds 3,4) | +0.0207 (+0.0085, +0.0329) | +0.0213 | +| 5G-NIDD α=0.50, mean (seeds 3,4) | +0.0153 (+0.0269, +0.0038) | +0.0154 | + +The direction is the same on every confirmatory seed and both conditions, and +the test differences track the validation differences closely (mean +validation−test = −0.0004). That is the strongest statement this design +supports: a modest, direction-consistent gain over the **baseline** on the two +new seeds. + +**Second candidate, `v_cap4`.** It does **not** meet the rule: mean validation +difference +0.0047, sd 0.0263, only **2/4** positive. The gain comes entirely +from X-IIoTID seed 3 (+0.0410) and reverses on seed 4 (−0.0214); on 5G-NIDD it +is +0.0020 / −0.0029. Mean test difference +0.0059 with the same mixed pattern. +So the capping candidate is a negative/mixed result and remains in the tables. + +**Versus the current proposed method.** Comparing each variant with the best +validation score among {original, β=0.5, β=1} on the same pair: + +- `v_shrink`: mean **−0.0153** (sd 0.0521, 2/4 pairs better). It improves on + both seed-4 pairs (+0.0329 X-IIoTID, +0.0038 5G-NIDD) but loses on seed 3, + mainly 5G-NIDD (−0.0888) where β=1 gains +0.1158 over the baseline. +- `v_cap4`: mean **−0.0286** (sd 0.0597, 1/4 pairs better). + +So `v_shrink` is **more stable than the current method but does not dominate +it**. The current method's β=1 has a much higher peak on 5G-NIDD seed 3 and a +much deeper failure on X-IIoTID seed 4 (−0.1384). This is a measured trade-off, +not a win for either method. + +**Reference behaviour on the new seeds.** β=0.5 has mean validation +0.0109 +(sd 0.0190, 2/4 positive) and test +0.0122 (3/4). β=1 has mean validation +−0.0163 (sd 0.1040, 1/4 positive) and test −0.0155 (1/4): X-IIoTID −0.0780 +(−0.0177, −0.1384), 5G-NIDD +0.0455 (+0.1158, −0.0248). The large β=1 spread is +the clearest measured instance of the seed sensitivity already visible in the +old grid. + +**No significance claim.** Four paired observations (two conditions × two +seeds), no inferential test, and the sample standard deviations are of the same +order as the means for most statistics. `v_shrink` passing the pre-registered +rule means “positive on these four pairs”, not “superior in general”. + +### 5.9 Where the `v_shrink` gain comes from (descriptive) + +Condition-level means hide heterogeneity. Pooling the two confirmation seeds +per dataset, test-F1 class deltas for `v_shrink` versus the original: + +| dataset | largest gains | losses | +|---|---|---| +| X-IIoTID α=0.25 | C&C +0.072, Weaponization +0.061, Tampering +0.054 | Exploitation −0.010, Exfiltration −0.003, Lateral −0.002 | +| 5G-NIDD α=0.50 | ICMPFlood +0.069 (seed 3 +0.143, seed 4 −0.004), SlowrateDoS +0.026, HTTPFlood +0.013 | SYNFlood −0.014 (both seeds) | + +Per-client validation (10 clients × 2 seeds per dataset): X-IIoTID mean +0.0295 +but only **9 improve / 11 degrade**; 5G-NIDD mean +0.0119 with **12 improve / +4 degrade**. So the positive condition-level result is not “every client wins”: +on X-IIoTID the mean is pulled up by a few large attack-class/client gains while +the majority of client-seed pairs lose a little. + +The same view explains the other results: + +- `v_cap4` on X-IIoTID gains Tampering +0.157 but loses **Normal −0.234** + (seed 4 −0.475) and C&C −0.123; client pairs are 8 improve / 12 degrade. That + is why its average is mixed and flips sign between seeds. +- β=1 on X-IIoTID loses Tampering −0.366 and Normal −0.245; client pairs are + 3 improve / 17 degrade. On 5G-NIDD it gains ICMPFlood +0.200 and 11/5 client + pairs improve, but seed 4 reverses the aggregate. This is the microscale + origin of the large β=1 seed variance. + +Full rows: `analysis/confirm_per_class_variant.csv` and +`analysis/confirm_per_client_variant.csv`. + +--- + +## 6. Negative results, operational deviations and honesty notes + +### 6.1 Negative and mixed results (measured) + +- **`v_cap4` did not improve** under the pre-registered rule: mean validation + +0.0047, sd 0.0263, 2/4 positive; the gain is seed-3 only (+0.0410 on + X-IIoTID, −0.0214 on seed 4). It stays in the tables as a mixed/negative + result. +- **Expected-negative controls behaved as expected but not identically.** + `v_minsup10` was worst on screening (−0.0302; S1 −0.0166, S2 −0.0438), + consistent with removing low-support rare-class evidence. `v_cap2` was near + zero (−0.0024). `v_logweight` was mixed (+0.0067 S1, −0.0346 S2) and overall + negative. `v_beta1.5` was the worst (−0.0485; S1 −0.0918), confirming that + extending β beyond 1 concentrates weight in the wrong direction at α=0.25. +- **The current method's β=1 is strongly seed-dependent** on the new seeds: + mean validation −0.0163, sd 0.1040, 1/4 positive. It ranges from +0.1158 + (5G-NIDD seed 3) to −0.1384 (X-IIoTID seed 4). β=0.5 is more stable + (+0.0109, sd 0.0190) but weaker. +- **The analysis-level selection-metric candidate (C7) was not confirmed.** + Its exploratory gain (+0.0082 vs +0.0052 mean test difference on the old + grid) uses the burned seeds and is not part of the new confirmation. It + remains a hypothesis for future work. +- **No variant was tested at α=0.75.** The confirmation conditions were fixed + to α=0.25 (X-IIoTID) and α=0.50 (5G-NIDD); the report does not extrapolate. + +### 6.2 Operational deviations (with timestamps) + +The ranking was frozen at 2026-10-31T05:03:55Z before any confirmatory run, and +was never recomputed. The following operational events are recorded because they +changed execution, not the scientific design: + +| when (UTC) | id | event | action | +|---|---|---|---| +| 2026-10-30T04:10Z | A1 | wrapper resolved repo root one level too high; six attempts exited 127 before Python started | path fixed; attempts archived; no science | +| 2026-10-30T04:32Z | A2 | restored grid came from another host; environment difference measured at median 0.0022 / max 0.0127 test rare-F1 | +2 in-environment baseline runs; 34 runs total | +| 2026-10-31T08:35Z | A3 | two seed-4 reference runs OOM-killed twice at 11 GiB | requeued, `max_procs` 3→2, OOM→137 mapping | +| 2026-10-31T09:50Z | A4 | three OOM kills total; user asked to prevent further OOM | X-IIoTID cap 11→13 GiB, `max_procs=2`, host `MemAvailable ≥ 12 GiB` guard | +| 2026-10-31T14:12Z | A5 | retries blocked by stale systemd unit names, misread as OOM | `reset-failed` before launch; failed-start guard | +| 2026-10-31T09:41/11:08Z | A6 | editing `run_one_v2.sh` while two wrappers were running caused exit 127 after training finished | both retried; archived first attempts are bit-identical to the final runs | +| 2026-10-31T22:35Z | — | driver wrote `DONE` `status=complete` | 34/34 done, 0 terminal failures, 66.4 h | + +The A6 retries are a free reproducibility check: for both affected seed-3 +variants, the archived first attempt and the final run have identical +`val_global_metrics.json`, `test_global_metrics.json`, `val_selection.json` and +a byte-identical round-0 snapshot. The retry did not change any result. + +A partial `DONE` (`status=stopped`) was produced at 2026-10-31T14:07Z during the +A5 incident; it was archived to `logs/DONE.partial_20261031T1412Z` and the run +resumed. The final `DONE` is `complete`. All failed attempts are kept under +`results/attempts/` and the pre-campaign bug attempts under +`results/_driver_bug_20261030T0410Z/`. + +### 6.3 Honesty notes + +- Screening six variants raises the chance that the top one is lucky; the + confirmation on seeds never used in any analysis exists for that reason. +- No test metric of any new run was read before the ranking was frozen. The + ranking code opened only `val_*` files (see `ACCESS_LOG.txt`); the + post-freeze analysis read test metrics only after `ranking.sha256` existed. +- No significance test is performed and no superiority is claimed. +- A modification that did not improve is reported as such (`v_cap4`, and the + four candidates that never reached confirmation). + +## 7. Limitations + +1. Two datasets, two conditions, ten clients, one architecture and one training + configuration; only α=0.25 (X-IIoTID) and α=0.50 (5G-NIDD) reached + confirmation. +2. Confirmation has **four paired observations** (two conditions × two seeds). + The sample standard deviations are of the same order as the means; no + inferential test and no generalization claim. +3. The `S_j` participant rule remains a reproduction assumption; all results are + conditional on it. +4. The paper comparison is limited by the reproduction gap in §2.8. +5. Host/library differences between the restored grid and this environment are + why A2 added in-environment baselines. Residual environment effects for the + β-grid reference (taken from the 1-thread sweep) cannot be excluded. +6. The confirmation β-grid reference is truncated to β=0.5, β=1.0 and the + original (a β=0 proxy); the full five-point grid was not run on the new + seeds, for budget reasons stated before any confirmation run. +7. X-IIoTID seed-4 confirmation runs peaked at **12.76 GiB**, above the old + 11 GiB cap and close to the final 13 GiB cap. The results are conditional on + the 13 GiB cap; a smaller cap would have killed them. +8. Rare-class support differs across seeds, so absolute metric levels are not + comparable across seeds; only paired differences are used. +9. The “current best” comparison uses the per-pair best of + {original, β=0.5, β=1} on validation. It is a fair proxy for the deployed + method on these seeds, but it is not the pooled-across-seeds selection used + in the original thesis protocol. + +## 8. What the evidence supports and what it does not + +**Supports (measured):** + +- `v_shrink` (β=1, λ=0.5, half-way between equal and support-weighted) improved + over the **original baseline** on validation in 4/4 confirmation pairs: + mean +0.0180 (sd 0.0141), with test differences of the same size and + direction (mean +0.0184, 4/4). This is a modest, direction-consistent gain on + two unseen seeds, not a large one. +- `v_cap4` did **not** improve (2/4, mean +0.0047): capping at 4×median is not + enough, or is the wrong shape, on these conditions. +- The current method's β=1 is highly seed-sensitive (sd 0.1040 across the four + pairs, 1/4 positive); β=0.5 is more stable but weaker. +- From Phase 2: support weighting can lose substantially at high + heterogeneity; the loss is class- and client-specific; the deployed + per-client selection metric can disagree with the global headline metric; + displacement is not a detection-quality proxy. + +**Does not support:** + +- Any claim that `v_shrink` is better than the current proposed method. Against + the per-pair best of {original, β=0.5, β=1} it is **−0.0153** on validation + (2/4 pairs better), mainly because β=1 gains +0.1158 on 5G-NIDD seed 3. +- Any claim of statistical significance, superiority, or generalisation beyond + these two datasets, two conditions and two new seeds. +- Any universal or globally optimal β, or any causal α→β rule. +- Any causal link from prototype displacement to detection performance. +- Any statement that support weighting always helps or always hurts. + +**Unknown:** + +- Why β=1 is so seed-sensitive on 5G-NIDD seed 3 versus X-IIoTID seed 4. +- Whether a pooled-across-seeds selection would change the current-method + comparison. +- Whether `v_shrink` helps at α=0.75 or on other datasets; not tested. +- Whether the C7 selection-metric change would survive new seeds; not + confirmed. diff --git a/exploration_v2/REPORT_DATA.md b/exploration_v2/REPORT_DATA.md new file mode 100644 index 0000000..a316b8f --- /dev/null +++ b/exploration_v2/REPORT_DATA.md @@ -0,0 +1,102 @@ +# exploration_v2 — report data +Generated: 2026-11-02T03:36:03Z +This file is generated by `exploration_v2/scripts/analyze_results.py`. All numbers come from files on disk. Missing rows mean a run did not produce the metric. +## Plan and ranking freeze +| item | value | +|---|---| +| `PLAN.sha256` | `2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e PLAN.md` at 2026-10-30T04:07:47Z | +| `ranking.sha256` | `f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3 ranking.json` at 2026-10-31T05:03:55Z | + +## Screening ranking (validation only) +Top-1 minus top-2 validation score (selection margin): 0.020278 (top2 = ['v_shrink__b1__lam0.5', 'v_cap__b1__c4']). + +Score = mean over S1,S2 of the paired validation difference in global rare-class macro-F1 versus the original baseline. + +| rank | variant | β | λ | cap | min N | score | min cond Δ | S1 Δ | S2 Δ | secondary rank | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | `v_shrink__b1__lam0.5` | 1.00 | 0.50 | | | 0.0267 | 0.0142 | 0.0391 | 0.0142 | 1 | +| 2 | `v_cap__b1__c4` | 1.00 | | 4.0 | | 0.0064 | 0.0035 | 0.0035 | 0.0093 | 2 | +| 3 | `v_cap__b1__c2` | 1.00 | | 2.0 | | -0.0024 | -0.0056 | -0.0056 | 0.0007 | 3 | +| 4 | `v_logweight` | 0.00 | | | | -0.0139 | -0.0346 | 0.0067 | -0.0346 | 5 | +| 5 | `v_minsup__t10` | 0.00 | | | 10 | -0.0302 | -0.0438 | -0.0166 | -0.0438 | 6 | +| 6 | `v_beta__b1.5` | 1.50 | | | | -0.0485 | -0.0918 | -0.0918 | -0.0052 | 4 | + +## Confirmatory results (per seed) +| dataset | α | seed | config | val rare-F1 | test rare-F1 | Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 | +|---|---|---|---|---|---|---|---|---|---| +| xiiotid | 0.25 | 3 | `v_shrink__b1__lam0.5` | 0.6605 | 0.6611 | 0.0085 | 0.0094 | 0.0261 | 0.0241 | +| xiiotid | 0.25 | 3 | `v_cap__b1__c4` | 0.6930 | 0.6929 | 0.0410 | 0.0413 | 0.0587 | 0.0559 | +| xiiotid | 0.25 | 3 | `support_weighted__beta0.5` | 0.6695 | 0.6697 | 0.0175 | 0.0181 | 0.0352 | 0.0327 | +| xiiotid | 0.25 | 3 | `support_weighted__beta1` | 0.6343 | 0.6370 | -0.0177 | -0.0146 | 0.0000 | 0.0000 | +| xiiotid | 0.25 | 4 | `v_shrink__b1__lam0.5` | 0.7155 | 0.7164 | 0.0329 | 0.0332 | 0.1713 | 0.1745 | +| xiiotid | 0.25 | 4 | `v_cap__b1__c4` | 0.6612 | 0.6628 | -0.0214 | -0.0204 | 0.1170 | 0.1209 | +| xiiotid | 0.25 | 4 | `support_weighted__beta0.5` | 0.6745 | 0.6745 | -0.0082 | -0.0087 | 0.1302 | 0.1326 | +| xiiotid | 0.25 | 4 | `support_weighted__beta1` | 0.5443 | 0.5419 | -0.1384 | -0.1413 | 0.0000 | 0.0000 | +| 5gnidd | 0.50 | 3 | `v_shrink__b1__lam0.5` | 0.8314 | 0.8276 | 0.0269 | 0.0276 | -0.0888 | -0.0895 | +| 5gnidd | 0.50 | 3 | `v_cap__b1__c4` | 0.8064 | 0.8044 | 0.0020 | 0.0044 | -0.1138 | -0.1128 | +| 5gnidd | 0.50 | 3 | `support_weighted__beta0.5` | 0.8389 | 0.8370 | 0.0344 | 0.0370 | -0.0813 | -0.0801 | +| 5gnidd | 0.50 | 3 | `support_weighted__beta1` | 0.9202 | 0.9172 | 0.1158 | 0.1172 | 0.0000 | 0.0000 | +| 5gnidd | 0.50 | 4 | `v_shrink__b1__lam0.5` | 0.9297 | 0.9265 | 0.0038 | 0.0032 | 0.0285 | 0.0262 | +| 5gnidd | 0.50 | 4 | `v_cap__b1__c4` | 0.9231 | 0.9216 | -0.0029 | -0.0017 | 0.0219 | 0.0213 | +| 5gnidd | 0.50 | 4 | `support_weighted__beta0.5` | 0.9258 | 0.9255 | -0.0001 | 0.0023 | 0.0246 | 0.0253 | +| 5gnidd | 0.50 | 4 | `support_weighted__beta1` | 0.9012 | 0.9002 | -0.0248 | -0.0231 | 0.0000 | 0.0000 | + +## Confirmatory summary +| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | mean Δtest vs current best | improved per plan | +|---|---|---|---|---|---|---|---|---| +| `v_shrink__b1__lam0.5` | 0.0180 | 0.0141 | 4/4 | 0.0184 | 0.0143 | 0.0343 | 0.0338 | -0.0153 | -0.0155 | True | +| `v_cap__b1__c4` | 0.0047 | 0.0263 | 2/4 | 0.0059 | 0.0258 | 0.0210 | 0.0213 | -0.0286 | -0.0279 | False | + +## Reference configurations on the confirmation seeds +| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive | +|---|---|---|---|---|---|---| +| `support_weighted__beta0.5` | 0.0109 | 0.0190 | 2 | 0.0122 | 0.0199 | 3 | +| `support_weighted__beta1` | -0.0163 | 0.1040 | 1 | -0.0155 | 0.1057 | 1 | + +## Reference consistency (environment check) +| condition | new original val | restored original val | sweep β=0 val | new − restored | new − sweep β=0 | +|---|---|---|---|---|---| +| S1 | 0.6380 | 0.6416 | 0.6433 | -0.0035 | -0.0052 | +| S2 | 0.8615 | 0.8694 | 0.8677 | -0.0079 | -0.0061 | + +## Wall time and memory (new runs) +| phase | run | status | wall h | peak GiB | +|---|---|---|---|---| +| screening | `xiiotid__alpha0.25__seed0__original_protean` | done | 3.02 | 8.38 | +| screening | `5gnidd__alpha0.5__seed0__original_protean` | done | 4.67 | 10.30 | +| screening | `xiiotid__alpha0.25__seed0__v_beta__b1.5` | done | 3.07 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5` | done | 3.05 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c4` | done | 3.06 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c2` | done | 3.04 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_minsup__t10` | done | 3.03 | 8.38 | +| screening | `xiiotid__alpha0.25__seed0__v_logweight` | done | 2.96 | 8.38 | +| screening | `5gnidd__alpha0.5__seed0__v_beta__b1.5` | done | 4.88 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5` | done | 4.85 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c4` | done | 4.55 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c2` | done | 4.41 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_minsup__t10` | done | 4.51 | 10.29 | +| screening | `5gnidd__alpha0.5__seed0__v_logweight` | done | 4.82 | 10.31 | +| confirm | `xiiotid__alpha0.25__seed3__original_protean` | done | 3.08 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta0.5` | done | 3.08 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta1` | done | 3.12 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5` | done | 2.97 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed3__v_cap__b1__c4` | done | 2.95 | 10.48 | +| confirm | `xiiotid__alpha0.25__seed4__original_protean` | done | 3.11 | 12.75 | +| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` | done | 3.10 | 12.76 | +| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta1` | done | 3.20 | 12.76 | +| confirm | `xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5` | done | 3.18 | 12.76 | +| confirm | `xiiotid__alpha0.25__seed4__v_cap__b1__c4` | done | 2.82 | 12.76 | +| confirm | `5gnidd__alpha0.5__seed3__original_protean` | done | 4.41 | 9.04 | +| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta0.5` | done | 4.48 | 9.07 | +| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta1` | done | 4.66 | 9.04 | +| confirm | `5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5` | done | 4.57 | 9.06 | +| confirm | `5gnidd__alpha0.5__seed3__v_cap__b1__c4` | done | 4.73 | 9.05 | +| confirm | `5gnidd__alpha0.5__seed4__original_protean` | done | 4.81 | 11.12 | +| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta0.5` | done | 4.77 | 11.16 | +| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta1` | done | 4.88 | 11.13 | +| confirm | `5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5` | done | 4.79 | 11.15 | +| confirm | `5gnidd__alpha0.5__seed4__v_cap__b1__c4` | done | 4.56 | 11.14 | + +## Failed or missing runs +None. + diff --git a/exploration_v2/SETUP_CHECK.md b/exploration_v2/SETUP_CHECK.md new file mode 100644 index 0000000..2037b70 --- /dev/null +++ b/exploration_v2/SETUP_CHECK.md @@ -0,0 +1,19 @@ +# exploration_v2 setup check + +Generated: 2026-11-02T03:38:12.876063Z + +| check | result | detail | +|---|---|---| +| plan_hash_matches | PASS | 2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e | +| no_tracked_modifications | PASS | | +| planningawal_unchanged | PASS | | +| no_raw_dataset_in_repo | PASS | | +| original_tests_pass | PASS | ... [100%] | +| variant_tests_pass | PASS | .............. [100%] | +| confirm_seeds_new | PASS | restored seeds=[0, 1, 2] | +| manifest_ok_5gnidd__alpha0.5__seed4.json | PASS | seed=4 | +| manifest_ok_xiiotid__alpha0.25__seed4.json | PASS | seed=4 | +| manifest_ok_xiiotid__alpha0.25__seed3.json | PASS | seed=3 | +| manifest_ok_5gnidd__alpha0.5__seed3.json | PASS | seed=3 | + +Overall: PASS diff --git a/exploration_v2/STATUS_AT_SESSION_END.md b/exploration_v2/STATUS_AT_SESSION_END.md new file mode 100644 index 0000000..8054d49 --- /dev/null +++ b/exploration_v2/STATUS_AT_SESSION_END.md @@ -0,0 +1,55 @@ +# Final status — exploration_v2 + +UTC: 2026-11-02T03:36Z +Repository: `/home/willtanoe/project/prototype-agregation` + +## Outcome + +- Driver finished 2026-11-01T22:35:36Z; `DONE` says `status=complete`. +- **34/34 scientific runs done**, 0 terminal failures. +- Wall time: **66.4 h** of the 96 h budget. +- Screening: 14 runs (12 variants + 2 in-environment baselines). +- Confirmation: 20 runs (top-2 variants + original + β=0.5 + β=1 on seeds 3 + and 4, two conditions). +- Frozen plan hash: `2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e`. +- Frozen ranking hash: `f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3`. + +## Headline result + +- `v_shrink` (β=1, λ=0.5) improved over the original baseline on validation in + **4/4** confirmation pairs: mean **+0.0180** (sd 0.0141); test mean +0.0184 + (4/4). It therefore meets the pre-registered improvement rule. +- `v_cap4` did **not**: mean +0.0047, 2/4. +- Neither beats the current method's per-pair best + {original, β=0.5, β=1}: `v_shrink` mean **−0.0153** on validation (2/4 pairs + better), mainly because β=1 gains +0.1158 on 5G-NIDD seed 3. +- β=1 was highly seed-sensitive (mean −0.0163, sd 0.1040, 1/4 positive). + +## Files + +- `REPORT.md` — complete narrative (Phase 1, Phase 2, design, results, + negative results, deviations, limitations, support/does-not-support). +- `REPORT_DATA.md`, `analysis/*.csv` — all numeric tables. +- `PHASE12_ANALYSIS.md`, `CANDIDATES.md`, `PLAN.md`, `PLAN_CHECK.md`, + `PLAN_AMENDMENTS.md` (A1–A5), `ISSUES.md`, `HANDOFF.md`. +- `results/runs/` (34 COMPLETE), `results/attempts/` (archived failures), + `results/_driver_bug_20261030T0410Z/` (pre-campaign path bug). +- `ranking.json` / `ranking.sha256`, `ACCESS_LOG.txt` (52 validation-only + opens; zero test/per-client/history opens). + +## Integrity + +- No tracked file modified; `git diff --stat` empty; `planningawal/` unchanged. +- No raw dataset copied into the repository. +- No commit or push performed. +- No statistical significance or superiority claim is made. + +## Open issues / limitations + +- Four confirmatory pairs only; no inferential test. +- Confirmation conditions limited to X-IIoTID α=0.25 and 5G-NIDD α=0.50. +- X-IIoTID seed-4 runs peaked at 12.76 GiB, close to the 13 GiB cap adopted + after the OOM incident. +- The current-method reference on new seeds is truncated to β=0.5, β=1 and the + original (β=0 proxy). +- C7 (global selection metric) remains exploratory on burned seeds. diff --git a/exploration_v2/analysis/confirm_paired.csv b/exploration_v2/analysis/confirm_paired.csv new file mode 100644 index 0000000..7240add --- /dev/null +++ b/exploration_v2/analysis/confirm_paired.csv @@ -0,0 +1,17 @@ +dataset,alpha,seed,config,run_id,val_global_rare_f1,val_per_client_rare_f1,test_rare_f1,test_rare_recall,baseline_val_global,baseline_test_rare_f1,delta_val_vs_original,delta_test_vs_original,delta_val_vs_beta1,delta_test_vs_beta1,delta_val_vs_beta05,delta_test_vs_beta05,current_best_config,current_best_val_global,delta_val_vs_current_best,delta_test_vs_current_best +xiiotid,0.25,3,v_shrink__b1__lam0.5,xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,0.6604599138123004,0.6835769827947561,0.6611166614366842,0.834631327602874,0.6520038277014277,0.6516753883176936,0.008456086110872674,0.009441273118990678,0.026139916018731668,0.024068007268394864,-0.009056466053729562,-0.008628509118876138,support_weighted__beta0.5,0.66951637986603,-0.009056466053729562,-0.008628509118876138 +xiiotid,0.25,3,v_cap__b1__c4,xiiotid__alpha0.25__seed3__v_cap__b1__c4,0.6930347111029873,0.6974933387696897,0.6929355556853871,0.8169158784862602,0.6520038277014277,0.6516753883176936,0.04103088340155958,0.041260167367693557,0.058714713309418576,0.05588690151709774,0.023518331236957346,0.02319038512982674,support_weighted__beta0.5,0.66951637986603,0.023518331236957346,0.02319038512982674 +xiiotid,0.25,3,support_weighted__beta0.5,xiiotid__alpha0.25__seed3__support_weighted__beta0.5,0.66951637986603,0.6806478241186118,0.6697451705555604,0.8147471250008732,0.6520038277014277,0.6516753883176936,0.017512552164602235,0.018069782237866816,0.03519638207246123,0.032696516387271,0.0,0.0,support_weighted__beta0.5,0.66951637986603,0.0,0.0 +xiiotid,0.25,3,support_weighted__beta1,xiiotid__alpha0.25__seed3__support_weighted__beta1,0.6343199977935687,0.6443454840057614,0.6370486541682894,0.7953041343152203,0.6520038277014277,0.6516753883176936,-0.017683829907858994,-0.014626734149404186,0.0,0.0,-0.03519638207246123,-0.032696516387271,support_weighted__beta0.5,0.66951637986603,-0.03519638207246123,-0.032696516387271 +xiiotid,0.25,4,v_shrink__b1__lam0.5,xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,0.7155211309912932,0.5875322150240144,0.7163859069449825,0.8549205204777552,0.6826579058578224,0.6832331434606697,0.03286322513347084,0.03315276348431284,0.17126530669398643,0.17449076882237846,0.04105079335795048,0.04186336031153037,original_protean,0.6826579058578224,0.03286322513347084,0.03315276348431284 +xiiotid,0.25,4,v_cap__b1__c4,xiiotid__alpha0.25__seed4__v_cap__b1__c4,0.6612274834490823,0.5895548490218416,0.662809680890142,0.8345629884018604,0.6826579058578224,0.6832331434606697,-0.021430422408740046,-0.02042346257052763,0.11697165915177554,0.12091454276753799,-0.01324285418426041,-0.011712865743310097,original_protean,0.6826579058578224,-0.021430422408740046,-0.02042346257052763 +xiiotid,0.25,4,support_weighted__beta0.5,xiiotid__alpha0.25__seed4__support_weighted__beta0.5,0.6744703376333427,0.5946075229514161,0.6745225466334521,0.8293580156927381,0.6826579058578224,0.6832331434606697,-0.008187568224479636,-0.008710596827217532,0.13021451333603595,0.1326274085108481,0.0,0.0,original_protean,0.6826579058578224,-0.008187568224479636,-0.008710596827217532 +xiiotid,0.25,4,support_weighted__beta1,xiiotid__alpha0.25__seed4__support_weighted__beta1,0.5442558242973068,0.42910855853255436,0.541895138122604,0.6966345934869704,0.6826579058578224,0.6832331434606697,-0.1384020815605156,-0.14133800533806562,0.0,0.0,-0.13021451333603595,-0.1326274085108481,original_protean,0.6826579058578224,-0.1384020815605156,-0.14133800533806562 +5gnidd,0.5,3,v_shrink__b1__lam0.5,5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,0.831360062139045,0.8667377269112777,0.8276438788886984,0.9892275573304957,0.8044578347431527,0.7999980487126149,0.02690222739589232,0.027645830176083463,-0.08884958289572842,-0.08952434009919907,-0.007525129739533343,-0.009401964142520147,support_weighted__beta1,0.9202096450347734,-0.08884958289572842,-0.08952434009919907 +5gnidd,0.5,3,v_cap__b1__c4,5gnidd__alpha0.5__seed3__v_cap__b1__c4,0.8064198040760951,0.8437023476357212,0.8043697046783258,0.9921040309667984,0.8044578347431527,0.7999980487126149,0.0019619693329424592,0.0043716559657108744,-0.11378984095867828,-0.11279851430957166,-0.0324653878024832,-0.032676138352892736,support_weighted__beta1,0.9202096450347734,-0.11378984095867828,-0.11279851430957166 +5gnidd,0.5,3,support_weighted__beta0.5,5gnidd__alpha0.5__seed3__support_weighted__beta0.5,0.8388851918785784,0.8727486080528835,0.8370458430312185,0.9909209257334471,0.8044578347431527,0.7999980487126149,0.03442735713542566,0.03704779431860361,-0.08132445315619508,-0.08012237595667893,0.0,0.0,support_weighted__beta1,0.9202096450347734,-0.08132445315619508,-0.08012237595667893 +5gnidd,0.5,3,support_weighted__beta1,5gnidd__alpha0.5__seed3__support_weighted__beta1,0.9202096450347734,0.9429509748552004,0.9171682189878975,0.9866951889574849,0.8044578347431527,0.7999980487126149,0.11575181029162074,0.11717017027528254,0.0,0.0,0.08132445315619508,0.08012237595667893,support_weighted__beta1,0.9202096450347734,0.0,0.0 +5gnidd,0.5,4,v_shrink__b1__lam0.5,5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,0.9296850930111598,0.8880565443840929,0.9264737095074091,0.9657604065424229,0.9259273982940985,0.9232963015030996,0.003757694717061222,0.0031774080043095454,0.028523543149283515,0.026239840648658586,0.0038793926743494467,0.0009244400791833529,original_protean,0.9259273982940985,0.003757694717061222,0.0031774080043095454 +5gnidd,0.5,4,v_cap__b1__c4,5gnidd__alpha0.5__seed4__v_cap__b1__c4,0.9230772912397688,0.884532344077232,0.9215621117759981,0.967188414469838,0.9259273982940985,0.9232963015030996,-0.0028501070543297535,-0.0017341897271014872,0.02191574137789254,0.021328242917247553,-0.0027284090970415287,-0.00398715765222768,original_protean,0.9259273982940985,-0.0028501070543297535,-0.0017341897271014872 +5gnidd,0.5,4,support_weighted__beta0.5,5gnidd__alpha0.5__seed4__support_weighted__beta0.5,0.9258057003368103,0.88601978325071,0.9255492694282258,0.9668784318423409,0.9259273982940985,0.9232963015030996,-0.00012169795728822486,0.0022529679251261925,0.024644150474934068,0.025315400569475233,0.0,0.0,original_protean,0.9259273982940985,-0.00012169795728822486,0.0022529679251261925 +5gnidd,0.5,4,support_weighted__beta1,5gnidd__alpha0.5__seed4__support_weighted__beta1,0.9011615498618762,0.8562921340984854,0.9002338688587506,0.9672615025827914,0.9259273982940985,0.9232963015030996,-0.024765848432222293,-0.02306243264434904,0.0,0.0,-0.024644150474934068,-0.025315400569475233,original_protean,0.9259273982940985,-0.024765848432222293,-0.02306243264434904 diff --git a/exploration_v2/analysis/confirm_per_class_variant.csv b/exploration_v2/analysis/confirm_per_class_variant.csv new file mode 100644 index 0000000..3e38b06 --- /dev/null +++ b/exploration_v2/analysis/confirm_per_class_variant.csv @@ -0,0 +1,145 @@ +dataset,alpha,seed,variant,class_id,class_name,test_support,base_f1,variant_f1,delta_f1,base_recall,variant_recall,delta_recall +xiiotid,0.25,3,v_shrink__b1__lam0.5,0,C&C,573,0.19558797985061663,0.29688838138738227,0.10130040153676564,0.9825479930191972,0.9075043630017452,-0.07504363001745196 +xiiotid,0.25,3,v_shrink__b1__lam0.5,1,Exfiltration,4427,0.9995482267901513,0.9995481247175779,-1.0207257339356346e-07,0.9995482267901513,0.999322340185227,-0.0002258866049242858 +xiiotid,0.25,3,v_shrink__b1__lam0.5,2,Exploitation,227,0.03455925277291302,0.029429192178550265,-0.005130060594362752,0.6519823788546255,0.6563876651982379,0.004405286343612369 +xiiotid,0.25,3,v_shrink__b1__lam0.5,3,Lateral_movement,6319,0.7004691086934823,0.6687793907176446,-0.03168971797583775,0.5553093843962652,0.5176451970248457,-0.03766418737141952 +xiiotid,0.25,3,v_shrink__b1__lam0.5,4,Normal,83651,0.9427787226987763,0.9586015558416595,0.01582283314288324,0.9177415691384443,0.9479384585958327,0.03019688945738841 +xiiotid,0.25,3,v_shrink__b1__lam0.5,5,RDOS,28252,0.9952464259838485,0.9942155631423466,-0.0010308628415018894,0.9967435933739204,0.9946906413705224,-0.0020529520033979454 +xiiotid,0.25,3,v_shrink__b1__lam0.5,6,Reconnaissance,25474,0.7694047991528076,0.7669666336944198,-0.00243816545838782,0.6274632959095549,0.6239695375677161,-0.0034937583418387375 +xiiotid,0.25,3,v_shrink__b1__lam0.5,7,Tampering,1024,0.6162227602905569,0.5744920993227992,-0.04173066096775768,0.994140625,0.994140625,0.0 +xiiotid,0.25,3,v_shrink__b1__lam0.5,8,Weaponization,12884,0.9023645530071718,0.9586139063327532,0.05624935332558145,0.9863396460726482,0.9833902514746973,-0.0029493945979509606 +xiiotid,0.25,3,v_cap__b1__c4,0,C&C,573,0.19558797985061663,0.1510164569215876,-0.04457152292902902,0.9825479930191972,0.680628272251309,-0.3019197207678882 +xiiotid,0.25,3,v_cap__b1__c4,1,Exfiltration,4427,0.9995482267901513,0.9808931860036832,-0.01865504078646807,0.9995482267901513,0.9625028235825616,-0.037045403207589755 +xiiotid,0.25,3,v_cap__b1__c4,2,Exploitation,227,0.03455925277291302,0.027462035100481952,-0.007097217672431065,0.6519823788546255,0.6651982378854625,0.013215859030836996 +xiiotid,0.25,3,v_cap__b1__c4,3,Lateral_movement,6319,0.7004691086934823,0.8096490432574589,0.10917993456397657,0.5553093843962652,0.706440892546289,0.15113150815002374 +xiiotid,0.25,3,v_cap__b1__c4,4,Normal,83651,0.9427787226987763,0.948727631530862,0.005948908832085764,0.9177415691384443,0.931034894980335,0.01329332584189069 +xiiotid,0.25,3,v_cap__b1__c4,5,RDOS,28252,0.9952464259838485,0.9936917461105007,-0.0015546798733477862,0.9967435933739204,0.9924607107461418,-0.004282882627778539 +xiiotid,0.25,3,v_cap__b1__c4,6,Reconnaissance,25474,0.7694047991528076,0.7667060839346477,-0.0026987152181598972,0.6274632959095549,0.6235769804506556,-0.003886315458899281 +xiiotid,0.25,3,v_cap__b1__c4,7,Tampering,1024,0.6162227602905569,0.8412353197042193,0.22501255941366238,0.994140625,0.9443359375,-0.0498046875 +xiiotid,0.25,3,v_cap__b1__c4,8,Weaponization,12884,0.9023645530071718,0.972830574450517,0.07046602144334524,0.9863396460726482,0.9601831729276622,-0.026156473144986037 +xiiotid,0.25,3,support_weighted__beta0.5,0,C&C,573,0.19558797985061663,0.20091789903110657,0.005329919180489945,0.9825479930191972,0.68760907504363,-0.29493891797556715 +xiiotid,0.25,3,support_weighted__beta0.5,1,Exfiltration,4427,0.9995482267901513,0.9994349644027574,-0.00011326238739395311,0.9995482267901513,0.9988705669753783,-0.0006776598147729684 +xiiotid,0.25,3,support_weighted__beta0.5,2,Exploitation,227,0.03455925277291302,0.027697741425783063,-0.006861511347129954,0.6519823788546255,0.6563876651982379,0.004405286343612369 +xiiotid,0.25,3,support_weighted__beta0.5,3,Lateral_movement,6319,0.7004691086934823,0.7297718853577696,0.029302776664287267,0.5553093843962652,0.592340560215224,0.03703117581895876 +xiiotid,0.25,3,support_weighted__beta0.5,4,Normal,83651,0.9427787226987763,0.9550678067643374,0.012289084065561173,0.9177415691384443,0.9433479575856833,0.02560638844723906 +xiiotid,0.25,3,support_weighted__beta0.5,5,RDOS,28252,0.9952464259838485,0.9928932210899425,-0.0023532048939060024,0.9967435933739204,0.9915050261928359,-0.005238567181084508 +xiiotid,0.25,3,support_weighted__beta0.5,6,Reconnaissance,25474,0.7694047991528076,0.7662707609115489,-0.0031340382412586854,0.6274632959095549,0.6230274004867709,-0.0044358954227839975 +xiiotid,0.25,3,support_weighted__beta0.5,7,Tampering,1024,0.6162227602905569,0.6616688396349413,0.04544607934438438,0.994140625,0.9912109375,-0.0029296875 +xiiotid,0.25,3,support_weighted__beta0.5,8,Weaponization,12884,0.9023645530071718,0.9793060525906333,0.07694149958346153,0.9863396460726482,0.9770257683949084,-0.009313877677739835 +xiiotid,0.25,3,support_weighted__beta1,0,C&C,573,0.19558797985061663,0.15957228048529715,-0.03601569936531948,0.9825479930191972,0.6771378708551483,-0.30541012216404884 +xiiotid,0.25,3,support_weighted__beta1,1,Exfiltration,4427,0.9995482267901513,0.9992087713349157,-0.0003394554552356155,0.9995482267901513,0.9984187937655297,-0.001129433024621651 +xiiotid,0.25,3,support_weighted__beta1,2,Exploitation,227,0.03455925277291302,0.0239511332583186,-0.010608119514594418,0.6519823788546255,0.6563876651982379,0.004405286343612369 +xiiotid,0.25,3,support_weighted__beta1,3,Lateral_movement,6319,0.7004691086934823,0.6436087092405458,-0.05686039945293653,0.5553093843962652,0.4888431713878778,-0.0664662130083874 +xiiotid,0.25,3,support_weighted__beta1,4,Normal,83651,0.9427787226987763,0.9457320693052946,0.002953346606518359,0.9177415691384443,0.923635102987412,0.005893533848967736 +xiiotid,0.25,3,support_weighted__beta1,5,RDOS,28252,0.9952464259838485,0.9929812123360511,-0.002265213647797415,0.9967435933739204,0.9915050261928359,-0.005238567181084508 +xiiotid,0.25,3,support_weighted__beta1,6,Reconnaissance,25474,0.7694047991528076,0.7662337662337663,-0.0031710329190413056,0.6274632959095549,0.6230274004867709,-0.0044358954227839975 +xiiotid,0.25,3,support_weighted__beta1,7,Tampering,1024,0.6162227602905569,0.5461868958109559,-0.07003586447960097,0.994140625,0.9931640625,-0.0009765625 +xiiotid,0.25,3,support_weighted__beta1,8,Weaponization,12884,0.9023645530071718,0.9646464646464645,0.06228191163929275,0.9863396460726482,0.9339490841353617,-0.05239056193728653 +xiiotid,0.25,4,v_shrink__b1__lam0.5,0,C&C,573,0.21977597528003087,0.26217401338564505,0.04239803810561418,0.9930191972076788,0.9912739965095986,-0.0017452006980802626 +xiiotid,0.25,4,v_shrink__b1__lam0.5,1,Exfiltration,4427,0.9766797506349573,0.9704651162790697,-0.006214634355887538,0.9555003388299074,0.9426248023492206,-0.012875536480686733 +xiiotid,0.25,4,v_shrink__b1__lam0.5,2,Exploitation,227,0.04720962951383461,0.03136033229491174,-0.01584929721892287,0.6651982378854625,0.6651982378854625,0.0 +xiiotid,0.25,4,v_shrink__b1__lam0.5,3,Lateral_movement,6319,0.8849000425350915,0.9127720870678617,0.027872044532770257,0.8230732710871973,0.9025162209210318,0.07944294983383449 +xiiotid,0.25,4,v_shrink__b1__lam0.5,4,Normal,83651,0.9508570036837458,0.9540897802551446,0.0032327765713987544,0.9334257809231211,0.9391997704749495,0.005773989551828396 +xiiotid,0.25,4,v_shrink__b1__lam0.5,5,RDOS,28252,0.9948602009996997,0.995265184975796,0.00040498397609622927,0.99684978054651,0.9969913634432961,0.0001415828967861188 +xiiotid,0.25,4,v_shrink__b1__lam0.5,6,Reconnaissance,25474,0.7627515886631068,0.7634216004260057,0.0006700117628989366,0.6196121535683442,0.6190625736044595,-0.0005495799638847165 +xiiotid,0.25,4,v_shrink__b1__lam0.5,7,Tampering,1024,0.6814384308027607,0.8310552214324768,0.1496167906297161,0.916015625,0.7421875,-0.173828125 +xiiotid,0.25,4,v_shrink__b1__lam0.5,8,Weaponization,12884,0.8982495292558755,0.964573699698093,0.0663241704422175,0.999689537410742,0.9795094691089724,-0.02018006830176955 +xiiotid,0.25,4,v_cap__b1__c4,0,C&C,573,0.21977597528003087,0.019225284772433692,-0.2005506905075972,0.9930191972076788,1.0,0.006980802792321161 +xiiotid,0.25,4,v_cap__b1__c4,1,Exfiltration,4427,0.9766797506349573,0.9561424192407451,-0.02053733139421221,0.9555003388299074,0.91597018296815,-0.03953015586175734 +xiiotid,0.25,4,v_cap__b1__c4,2,Exploitation,227,0.04720962951383461,0.025953936060501892,-0.021255693453332718,0.6651982378854625,0.6651982378854625,0.0 +xiiotid,0.25,4,v_cap__b1__c4,3,Lateral_movement,6319,0.8849000425350915,0.8543103448275863,-0.030589697707505192,0.8230732710871973,0.7841430606108561,-0.038930210476341154 +xiiotid,0.25,4,v_cap__b1__c4,4,Normal,83651,0.9508570036837458,0.47602587077348524,-0.4748311329102606,0.9334257809231211,0.31278765346499143,-0.6206381274581296 +xiiotid,0.25,4,v_cap__b1__c4,5,RDOS,28252,0.9948602009996997,0.9941342217654043,-0.0007259792342954041,0.99684978054651,0.992814667988107,-0.004035112558402942 +xiiotid,0.25,4,v_cap__b1__c4,6,Reconnaissance,25474,0.7627515886631068,0.7467183551291804,-0.016033233533926383,0.6196121535683442,0.597354165031012,-0.022257988537332185 +xiiotid,0.25,4,v_cap__b1__c4,7,Tampering,1024,0.6814384308027607,0.7708719851576995,0.08943355435493883,0.916015625,0.8115234375,-0.1044921875 +xiiotid,0.25,4,v_cap__b1__c4,8,Weaponization,12884,0.8982495292558755,0.9351209001675844,0.03687137091170889,0.999689537410742,0.9095001552312947,-0.09018938217944727 +xiiotid,0.25,4,support_weighted__beta0.5,0,C&C,573,0.21977597528003087,0.025710176195613086,-0.1940657990844178,0.9930191972076788,0.9982547993019197,0.005235602094240899 +xiiotid,0.25,4,support_weighted__beta0.5,1,Exfiltration,4427,0.9766797506349573,0.9771256931608133,0.00044594252585605787,0.9555003388299074,0.9552744522249831,-0.0002258866049242858 +xiiotid,0.25,4,support_weighted__beta0.5,2,Exploitation,227,0.04720962951383461,0.023436262866191604,-0.023773366647643006,0.6651982378854625,0.6519823788546255,-0.013215859030836996 +xiiotid,0.25,4,support_weighted__beta0.5,3,Lateral_movement,6319,0.8849000425350915,0.8859115567976328,0.0010115142625413132,0.8230732710871973,0.8528248140528565,0.029751542965659183 +xiiotid,0.25,4,support_weighted__beta0.5,4,Normal,83651,0.9508570036837458,0.652634283007811,-0.2982227206759348,0.9334257809231211,0.4859356134415608,-0.44749016748156034 +xiiotid,0.25,4,support_weighted__beta0.5,5,RDOS,28252,0.9948602009996997,0.9940503594574495,-0.000809841542250278,0.99684978054651,0.9935225824720374,-0.0033271980744725704 +xiiotid,0.25,4,support_weighted__beta0.5,6,Reconnaissance,25474,0.7627515886631068,0.7680698201456193,0.0053182314825125365,0.6196121535683442,0.6253042317657219,0.005692078197377715 +xiiotid,0.25,4,support_weighted__beta0.5,7,Tampering,1024,0.6814384308027607,0.8291891891891893,0.1477507583864286,0.916015625,0.7490234375,-0.1669921875 +xiiotid,0.25,4,support_weighted__beta0.5,8,Weaponization,12884,0.8982495292558755,0.8926873152551087,-0.005562214000766841,0.999689537410742,0.808677429369761,-0.191012108040981 +xiiotid,0.25,4,support_weighted__beta1,0,C&C,573,0.21977597528003087,0.0738276919063458,-0.14594828337368507,0.9930191972076788,0.987783595113438,-0.005235602094240788 +xiiotid,0.25,4,support_weighted__beta1,1,Exfiltration,4427,0.9766797506349573,0.8429168844746471,-0.1337628661603102,0.9555003388299074,0.7284843008809577,-0.22701603794894964 +xiiotid,0.25,4,support_weighted__beta1,2,Exploitation,227,0.04720962951383461,0.005032216385304568,-0.042177413128530045,0.6651982378854625,0.6519823788546255,-0.013215859030836996 +xiiotid,0.25,4,support_weighted__beta1,3,Lateral_movement,6319,0.8849000425350915,0.8383040711492754,-0.04659597138581606,0.8230732710871973,0.8651685393258427,0.0420952682386454 +xiiotid,0.25,4,support_weighted__beta1,4,Normal,83651,0.9508570036837458,0.45723444826220333,-0.4936225554215425,0.9334257809231211,0.2966850366403271,-0.636740744282794 +xiiotid,0.25,4,support_weighted__beta1,5,RDOS,28252,0.9948602009996997,0.9947536334633108,-0.00010656753638893512,0.99684978054651,0.9932748124026618,-0.0035749681438481673 +xiiotid,0.25,4,support_weighted__beta1,6,Reconnaissance,25474,0.7627515886631068,0.7785080905375368,0.015756501874430007,0.6196121535683442,0.6393185208447829,0.019706367276438708 +xiiotid,0.25,4,support_weighted__beta1,7,Tampering,1024,0.6814384308027607,0.019342359767891684,-0.662096071034869,0.916015625,0.009765625,-0.90625 +xiiotid,0.25,4,support_weighted__beta1,8,Weaponization,12884,0.8982495292558755,0.7824761572965205,-0.115773371959355,0.999689537410742,0.6972989754734554,-0.30239056193728653 +5gnidd,0.5,3,v_shrink__b1__lam0.5,0,Benign,95547,0.5529540050583834,0.5519122784416843,-0.0010417266166991013,0.3821260740787257,0.38113179900991134,-0.000994275068814332 +5gnidd,0.5,3,v_shrink__b1__lam0.5,1,HTTPFlood,28162,0.8921519550586796,0.9118123892470686,0.01966043418838903,0.8543427313401036,0.8587458277111001,0.004403096370996451 +5gnidd,0.5,3,v_shrink__b1__lam0.5,2,ICMPFlood,231,0.3996539792387543,0.5422535211267606,0.14259954188800633,1.0,1.0,0.0 +5gnidd,0.5,3,v_shrink__b1__lam0.5,3,SYNFlood,1944,0.8737464643867319,0.858948432760364,-0.014798031626367836,0.8739711934156379,0.8739711934156379,0.0 +5gnidd,0.5,3,v_shrink__b1__lam0.5,4,SYNScan,4009,0.9931412894375857,0.9941389200648459,0.0009976306272602153,0.9932651534048391,0.9942629084559741,0.0009977550511349742 +5gnidd,0.5,3,v_shrink__b1__lam0.5,5,SlowrateDoS,14625,0.8143951383174021,0.8544430422604574,0.04004790394305535,0.8796581196581197,0.9490598290598291,0.06940170940170942 +5gnidd,0.5,3,v_shrink__b1__lam0.5,6,TCPConnectScan,4010,0.848619957537155,0.8292531120331951,-0.01936684550395995,0.9967581047381546,0.9967581047381546,0.0 +5gnidd,0.5,3,v_shrink__b1__lam0.5,7,UDPFlood,91468,0.7627768118117493,0.7623921650343822,-0.0003846467773670481,1.0,1.0,0.0 +5gnidd,0.5,3,v_shrink__b1__lam0.5,8,UDPScan,3181,0.9814011159330439,0.9833825128125485,0.0019813968795046044,0.9952845017290161,0.9952845017290161,0.0 +5gnidd,0.5,3,v_cap__b1__c4,0,Benign,95547,0.5529540050583834,0.5550438944717918,0.0020898894134083745,0.3821260740787257,0.3841250902697102,0.001999016190984526 +5gnidd,0.5,3,v_cap__b1__c4,1,HTTPFlood,28162,0.8921519550586796,0.8941099122094598,0.0019579571507801763,0.8543427313401036,0.8209289112989134,-0.0334138200411902 +5gnidd,0.5,3,v_cap__b1__c4,2,ICMPFlood,231,0.3996539792387543,0.3902027027027027,-0.0094512765360516,1.0,1.0,0.0 +5gnidd,0.5,3,v_cap__b1__c4,3,SYNFlood,1944,0.8737464643867319,0.9018046709129511,0.028058206526219265,0.8739711934156379,0.8739711934156379,0.0 +5gnidd,0.5,3,v_cap__b1__c4,4,SYNScan,4009,0.9931412894375857,0.9931464174454829,5.128007897203801e-06,0.9932651534048391,0.9940134696931904,0.0007483162883512584 +5gnidd,0.5,3,v_cap__b1__c4,5,SlowrateDoS,14625,0.8143951383174021,0.8365808225658615,0.02218568424845946,0.8796581196581197,0.9673162393162393,0.08765811965811965 +5gnidd,0.5,3,v_cap__b1__c4,6,TCPConnectScan,4010,0.848619957537155,0.8594792339143534,0.010859276377198412,0.9967581047381546,0.9960099750623441,-0.0007481296758105049 +5gnidd,0.5,3,v_cap__b1__c4,7,UDPFlood,91468,0.7627768118117493,0.7626623308944614,-0.00011448091728782916,1.0,1.0,0.0 +5gnidd,0.5,3,v_cap__b1__c4,8,UDPScan,3181,0.9814011159330439,0.9841467205470935,0.0027456046140496015,0.9952845017290161,0.9952845017290161,0.0 +5gnidd,0.5,3,support_weighted__beta0.5,0,Benign,95547,0.5529540050583834,0.5518354603049501,-0.0011185447534333148,0.3821260740787257,0.38105853663641975,-0.0010675374423059214 +5gnidd,0.5,3,support_weighted__beta0.5,1,HTTPFlood,28162,0.8921519550586796,0.9147573805791134,0.022605425520433764,0.8543427313401036,0.8598466018038492,0.005503870463745564 +5gnidd,0.5,3,support_weighted__beta0.5,2,ICMPFlood,231,0.3996539792387543,0.6015625,0.2019085207612457,1.0,1.0,0.0 +5gnidd,0.5,3,support_weighted__beta0.5,3,SYNFlood,1944,0.8737464643867319,0.8679438058748404,-0.005802658511891434,0.8739711934156379,0.8739711934156379,0.0 +5gnidd,0.5,3,support_weighted__beta0.5,4,SYNScan,4009,0.9931412894375857,0.994517817094443,0.0013765276568572515,0.9932651534048391,0.9955101022698928,0.002244948865053664 +5gnidd,0.5,3,support_weighted__beta0.5,5,SlowrateDoS,14625,0.8143951383174021,0.8599392507593654,0.045544112441963325,0.8796581196581197,0.9582222222222222,0.07856410256410251 +5gnidd,0.5,3,support_weighted__beta0.5,6,TCPConnectScan,4010,0.848619957537155,0.8175959079283888,-0.031024049608766258,0.9967581047381546,0.9965087281795512,-0.00024937655860346464 +5gnidd,0.5,3,support_weighted__beta0.5,7,UDPFlood,91468,0.7627768118117493,0.7623667475141483,-0.00041006429760093166,1.0,1.0,0.0 +5gnidd,0.5,3,support_weighted__beta0.5,8,UDPScan,3181,0.9814011159330439,0.9862928348909656,0.004891718957921731,0.9952845017290161,0.9952845017290161,0.0 +5gnidd,0.5,3,support_weighted__beta1,0,Benign,95547,0.5529540050583834,0.5562078318487121,0.0032538267903287066,0.3821260740787257,0.38524495797879577,0.0031188839000700908 +5gnidd,0.5,3,support_weighted__beta1,1,HTTPFlood,28162,0.8921519550586796,0.9776701074362755,0.08551815237759586,0.8543427313401036,0.9887792060222995,0.13443647468219588 +5gnidd,0.5,3,support_weighted__beta1,2,ICMPFlood,231,0.3996539792387543,0.9850746268656716,0.5854206476269173,1.0,1.0,0.0 +5gnidd,0.5,3,support_weighted__beta1,3,SYNFlood,1944,0.8737464643867319,0.8328431372549019,-0.04090332713182998,0.8739711934156379,0.8739711934156379,0.0 +5gnidd,0.5,3,support_weighted__beta1,4,SYNScan,4009,0.9931412894375857,0.9968824042898118,0.0037411148522260573,0.9932651534048391,0.9970067348465952,0.00374158144175607 +5gnidd,0.5,3,support_weighted__beta1,5,SlowrateDoS,14625,0.8143951383174021,0.9538612336343907,0.13946609531698861,0.8796581196581197,0.9315555555555556,0.051897435897435895 +5gnidd,0.5,3,support_weighted__beta1,6,TCPConnectScan,4010,0.848619957537155,0.8183587747157054,-0.030261182821449628,0.9967581047381546,0.9960099750623441,-0.0007481296758105049 +5gnidd,0.5,3,support_weighted__beta1,7,UDPFlood,91468,0.7627768118117493,0.7631491122680549,0.000372300456305652,1.0,1.0,0.0 +5gnidd,0.5,3,support_weighted__beta1,8,UDPScan,3181,0.9814011159330439,0.9856831621537504,0.0042820462207064924,0.9952845017290161,0.995598868280415,0.0003143665513989369 +5gnidd,0.5,4,v_shrink__b1__lam0.5,0,Benign,95547,0.5564678777807154,0.5582255083179297,0.0017576305372143386,0.38550660931269426,0.3871916439030006,0.0016850345903063335 +5gnidd,0.5,4,v_shrink__b1__lam0.5,1,HTTPFlood,28162,0.9678798586572438,0.9741026322786492,0.006222773621405442,0.972622683048079,0.9756764434344152,0.0030537603863362017 +5gnidd,0.5,4,v_shrink__b1__lam0.5,2,ICMPFlood,231,0.9148514851485148,0.911242603550296,-0.003608881598218816,1.0,1.0,0.0 +5gnidd,0.5,4,v_shrink__b1__lam0.5,3,SYNFlood,1944,0.870687384777456,0.8573651452282157,-0.013322239549240278,0.8503086419753086,0.8503086419753086,0.0 +5gnidd,0.5,4,v_shrink__b1__lam0.5,4,SYNScan,4009,0.997880034917072,0.9980044898977302,0.00012445498065816185,0.9980044898977302,0.9980044898977302,0.0 +5gnidd,0.5,4,v_shrink__b1__lam0.5,5,SlowrateDoS,14625,0.935706878672658,0.9481568007695744,0.012449922096916421,0.9254700854700855,0.9435213675213675,0.018051282051282036 +5gnidd,0.5,4,v_shrink__b1__lam0.5,6,TCPConnectScan,4010,0.7895882818685669,0.8084287306250634,0.018840448756496486,0.9947630922693267,0.9950124688279302,0.00024937655860346464 +5gnidd,0.5,4,v_shrink__b1__lam0.5,7,UDPFlood,91468,0.7635727672291811,0.7634739785484747,-9.878868070645197e-05,1.0,1.0,0.0 +5gnidd,0.5,4,v_shrink__b1__lam0.5,8,UDPScan,3181,0.9864801864801864,0.9880155642023346,0.001535377722148179,0.9977994341402074,0.9977994341402074,0.0 +5gnidd,0.5,4,v_cap__b1__c4,0,Benign,95547,0.5564678777807154,0.5587993514084241,0.002331473627708691,0.38550660931269426,0.38773587867750947,0.0022292693648152118 +5gnidd,0.5,4,v_cap__b1__c4,1,HTTPFlood,28162,0.9678798586572438,0.9785105856572297,0.010630726999985907,0.972622683048079,0.9830622825083446,0.010439599460265625 +5gnidd,0.5,4,v_cap__b1__c4,2,ICMPFlood,231,0.9148514851485148,0.8816793893129771,-0.03317209583553771,1.0,1.0,0.0 +5gnidd,0.5,4,v_cap__b1__c4,3,SYNFlood,1944,0.870687384777456,0.82025819265144,-0.05042919212601604,0.8503086419753086,0.8497942386831275,-0.0005144032921811093 +5gnidd,0.5,4,v_cap__b1__c4,4,SYNScan,4009,0.997880034917072,0.997880563520758,5.28603686045237e-07,0.9980044898977302,0.9982539286605139,0.0002494387627837158 +5gnidd,0.5,4,v_cap__b1__c4,5,SlowrateDoS,14625,0.935706878672658,0.9564676616915423,0.020760783018884288,0.9254700854700855,0.9464615384615385,0.020991452991452997 +5gnidd,0.5,4,v_cap__b1__c4,6,TCPConnectScan,4010,0.7895882818685669,0.827664869348818,0.03807658748025111,0.9947630922693267,0.9952618453865336,0.0004987531172069293 +5gnidd,0.5,4,v_cap__b1__c4,7,UDPFlood,91468,0.7635727672291811,0.7635313366055628,-4.1430623618277096e-05,1.0,1.0,0.0 +5gnidd,0.5,4,v_cap__b1__c4,8,UDPScan,3181,0.9864801864801864,0.9884735202492211,0.0019933337690346553,0.9977994341402074,0.9974850675888085,-0.0003143665513989369 +5gnidd,0.5,4,support_weighted__beta0.5,0,Benign,95547,0.5564678777807154,0.5593643278777508,0.00289645009703543,0.38550660931269426,0.3882801134520184,0.0027735041393241455 +5gnidd,0.5,4,support_weighted__beta0.5,1,HTTPFlood,28162,0.9678798586572438,0.9747545831922894,0.006874724535045562,0.972622683048079,0.9713798735885235,-0.0012428094595554606 +5gnidd,0.5,4,support_weighted__beta0.5,2,ICMPFlood,231,0.9148514851485148,0.9041095890410958,-0.010741896107418936,1.0,1.0,0.0 +5gnidd,0.5,4,support_weighted__beta0.5,3,SYNFlood,1944,0.870687384777456,0.8461735346813412,-0.024513850096114842,0.8503086419753086,0.8503086419753086,0.0 +5gnidd,0.5,4,support_weighted__beta0.5,4,SYNScan,4009,0.997880034917072,0.998129442573887,0.0002494076568150261,0.9980044898977302,0.9982539286605139,0.0002494387627837158 +5gnidd,0.5,4,support_weighted__beta0.5,5,SlowrateDoS,14625,0.935706878672658,0.950336757602558,0.014629878929899975,0.9254700854700855,0.9551452991452991,0.02967521367521364 +5gnidd,0.5,4,support_weighted__beta0.5,6,TCPConnectScan,4010,0.7895882818685669,0.8173254147040754,0.027737132835508493,0.9947630922693267,0.9952618453865336,0.0004987531172069293 +5gnidd,0.5,4,support_weighted__beta0.5,7,UDPFlood,91468,0.7635727672291811,0.7636588298156559,8.606258647481102e-05,1.0,1.0,0.0 +5gnidd,0.5,4,support_weighted__beta0.5,8,UDPScan,3181,0.9864801864801864,0.9880155642023346,0.001535377722148179,0.9977994341402074,0.9977994341402074,0.0 +5gnidd,0.5,4,support_weighted__beta1,0,Benign,95547,0.5564678777807154,0.5587993514084241,0.002331473627708691,0.38550660931269426,0.38773587867750947,0.0022292693648152118 +5gnidd,0.5,4,support_weighted__beta1,1,HTTPFlood,28162,0.9678798586572438,0.9742469286149218,0.0063670699576779954,0.972622683048079,0.9658404942830765,-0.006782188765002473 +5gnidd,0.5,4,support_weighted__beta1,2,ICMPFlood,231,0.9148514851485148,0.7287066246056783,-0.18614486054283652,1.0,1.0,0.0 +5gnidd,0.5,4,support_weighted__beta1,3,SYNFlood,1944,0.870687384777456,0.8210735586481114,-0.04961382612934462,0.8503086419753086,0.8497942386831275,-0.0005144032921811093 +5gnidd,0.5,4,support_weighted__beta1,4,SYNScan,4009,0.997880034917072,0.998003992015968,0.00012395709889601036,0.9980044898977302,0.9977550511349463,-0.0002494387627838268 +5gnidd,0.5,4,support_weighted__beta1,5,SlowrateDoS,14625,0.935706878672658,0.9501515661839003,0.014444687511242282,0.9254700854700855,0.9644444444444444,0.03897435897435897 +5gnidd,0.5,4,support_weighted__beta1,6,TCPConnectScan,4010,0.7895882818685669,0.8417501317870323,0.0521618499184654,0.9947630922693267,0.9955112219451372,0.0007481296758105049 +5gnidd,0.5,4,support_weighted__beta1,7,UDPFlood,91468,0.7635727672291811,0.7633561029347331,-0.00021666429444799906,1.0,1.0,0.0 +5gnidd,0.5,4,support_weighted__beta1,8,UDPScan,3181,0.9864801864801864,0.9877042801556419,0.0012240936754555065,0.9977994341402074,0.9974850675888085,-0.0003143665513989369 diff --git a/exploration_v2/analysis/confirm_per_client_variant.csv b/exploration_v2/analysis/confirm_per_client_variant.csv new file mode 100644 index 0000000..a4ccb02 --- /dev/null +++ b/exploration_v2/analysis/confirm_per_client_variant.csv @@ -0,0 +1,161 @@ +dataset,alpha,seed,variant,client_id,n_samples,base_val_rare_f1,variant_val_rare_f1,delta_val_rare_f1 +xiiotid,0.25,3,v_shrink__b1__lam0.5,0,2140,0.8561962585138841,0.8375523410734678,-0.01864391744041638 +xiiotid,0.25,3,v_shrink__b1__lam0.5,1,18610,0.28169014084507044,0.3895131086142322,0.10782296776916178 +xiiotid,0.25,3,v_shrink__b1__lam0.5,2,1581,0.5283208020050125,0.536380531594081,0.00805972958906842 +xiiotid,0.25,3,v_shrink__b1__lam0.5,3,3178,0.8866432337434095,0.8850220264317181,-0.0016212073116913528 +xiiotid,0.25,3,v_shrink__b1__lam0.5,4,12935,0.5414469291532702,0.5653124500337164,0.023865520880446245 +xiiotid,0.25,3,v_shrink__b1__lam0.5,5,5628,0.6122881355932204,0.6888020833333333,0.07651394774011289 +xiiotid,0.25,3,v_shrink__b1__lam0.5,6,13609,0.6420664206642066,0.626865671641791,-0.015200749022415638 +xiiotid,0.25,3,v_shrink__b1__lam0.5,7,2268,0.9933548127265405,0.9943227899432279,0.000967977216687399 +xiiotid,0.25,3,v_shrink__b1__lam0.5,8,5176,0.47174934993403267,0.4315775645103299,-0.040171785423702755 +xiiotid,0.25,3,v_shrink__b1__lam0.5,9,16289,0.8829885544314278,0.8804212607716639,-0.002567293659763825 +xiiotid,0.25,3,v_cap__b1__c4,0,2140,0.8561962585138841,0.9140103780578206,0.05781411954393645 +xiiotid,0.25,3,v_cap__b1__c4,1,18610,0.28169014084507044,0.21875,-0.06294014084507044 +xiiotid,0.25,3,v_cap__b1__c4,2,1581,0.5283208020050125,0.5383390750600271,0.01001827305501457 +xiiotid,0.25,3,v_cap__b1__c4,3,3178,0.8866432337434095,0.886103781882146,-0.0005394518612634736 +xiiotid,0.25,3,v_cap__b1__c4,4,12935,0.5414469291532702,0.5196141652815423,-0.021832763871727945 +xiiotid,0.25,3,v_cap__b1__c4,5,5628,0.6122881355932204,0.629029029029029,0.016740893435808646 +xiiotid,0.25,3,v_cap__b1__c4,6,13609,0.6420664206642066,0.726643598615917,0.08457717795171038 +xiiotid,0.25,3,v_cap__b1__c4,7,2268,0.9933548127265405,0.9796953677153082,-0.013659445011232285 +xiiotid,0.25,3,v_cap__b1__c4,8,5176,0.47174934993403267,0.6828009194286191,0.21105156949458642 +xiiotid,0.25,3,v_cap__b1__c4,9,16289,0.8829885544314278,0.8799470726264877,-0.003041481804940016 +xiiotid,0.25,3,support_weighted__beta0.5,0,2140,0.8561962585138841,0.8707592406090758,0.01456298209519169 +xiiotid,0.25,3,support_weighted__beta0.5,1,18610,0.28169014084507044,0.2828282828282828,0.0011381419832123818 +xiiotid,0.25,3,support_weighted__beta0.5,2,1581,0.5283208020050125,0.5399982514425599,0.011677449437547338 +xiiotid,0.25,3,support_weighted__beta0.5,3,3178,0.8866432337434095,0.8850220264317181,-0.0016212073116913528 +xiiotid,0.25,3,support_weighted__beta0.5,4,12935,0.5414469291532702,0.5417999072331323,0.0003529780798621207 +xiiotid,0.25,3,support_weighted__beta0.5,5,5628,0.6122881355932204,0.647567517573067,0.03527938197984659 +xiiotid,0.25,3,support_weighted__beta0.5,6,13609,0.6420664206642066,0.6666666666666666,0.024600246002460024 +xiiotid,0.25,3,support_weighted__beta0.5,7,2268,0.9933548127265405,0.9930838079739626,-0.000271004752577908 +xiiotid,0.25,3,support_weighted__beta0.5,8,5176,0.47174934993403267,0.4994349873234336,0.02768563738940094 +xiiotid,0.25,3,support_weighted__beta0.5,9,16289,0.8829885544314278,0.8793175531042194,-0.003671001327208323 +xiiotid,0.25,3,support_weighted__beta1,0,2140,0.8561962585138841,0.8255265538183227,-0.030669704695561406 +xiiotid,0.25,3,support_weighted__beta1,1,18610,0.28169014084507044,0.23098591549295774,-0.05070422535211269 +xiiotid,0.25,3,support_weighted__beta1,2,1581,0.5283208020050125,0.531031056751803,0.0027102547467904303 +xiiotid,0.25,3,support_weighted__beta1,3,3178,0.8866432337434095,0.8850220264317181,-0.0016212073116913528 +xiiotid,0.25,3,support_weighted__beta1,4,12935,0.5414469291532702,0.5323556211995784,-0.00909130795369184 +xiiotid,0.25,3,support_weighted__beta1,5,5628,0.6122881355932204,0.5699022093444405,-0.042385926248779904 +xiiotid,0.25,3,support_weighted__beta1,6,13609,0.6420664206642066,0.6060606060606061,-0.03600581460360053 +xiiotid,0.25,3,support_weighted__beta1,7,2268,0.9933548127265405,0.9808013355592654,-0.012553477167275062 +xiiotid,0.25,3,support_weighted__beta1,8,5176,0.47174934993403267,0.40165827142049576,-0.07009107851353691 +xiiotid,0.25,3,support_weighted__beta1,9,16289,0.8829885544314278,0.8801112439784262,-0.002877310453001547 +xiiotid,0.25,4,v_shrink__b1__lam0.5,0,2737,0.6914893617021277,0.8,0.10851063829787233 +xiiotid,0.25,4,v_shrink__b1__lam0.5,1,16337,0.030266608391608392,0.13058659217877094,0.10031998378716256 +xiiotid,0.25,4,v_shrink__b1__lam0.5,2,20397,0.9916883116883117,0.9899736147757255,-0.0017146969125861888 +xiiotid,0.25,4,v_shrink__b1__lam0.5,3,6530,0.8631578947368421,0.8400000000000001,-0.023157894736842044 +xiiotid,0.25,4,v_shrink__b1__lam0.5,4,7342,0.48807096365416736,0.4368949709744338,-0.05117599267973355 +xiiotid,0.25,4,v_shrink__b1__lam0.5,5,11939,0.4859075535512965,0.4811858608893957,-0.0047216926619008115 +xiiotid,0.25,4,v_shrink__b1__lam0.5,6,6280,0.8491131545208317,0.878584907422977,0.02947175290214532 +xiiotid,0.25,4,v_shrink__b1__lam0.5,7,7900,0.45493042952208107,0.4273399014778325,-0.027590528044248552 +xiiotid,0.25,4,v_shrink__b1__lam0.5,8,1256,0.19672131147540983,0.5714285714285715,0.3747072599531617 +xiiotid,0.25,4,v_shrink__b1__lam0.5,9,696,0.3725490196078431,0.31932773109243695,-0.05322128851540614 +xiiotid,0.25,4,v_cap__b1__c4,0,2737,0.6914893617021277,0.7638888888888888,0.07239952718676113 +xiiotid,0.25,4,v_cap__b1__c4,1,16337,0.030266608391608392,0.2650974577580082,0.2348308493663998 +xiiotid,0.25,4,v_cap__b1__c4,2,20397,0.9916883116883117,0.9509185632026322,-0.04076974848567949 +xiiotid,0.25,4,v_cap__b1__c4,3,6530,0.8631578947368421,0.8172043010752689,-0.045953593661573255 +xiiotid,0.25,4,v_cap__b1__c4,4,7342,0.48807096365416736,0.4566807056324985,-0.03139025802166884 +xiiotid,0.25,4,v_cap__b1__c4,5,11939,0.4859075535512965,0.4727694090382387,-0.013138144513057781 +xiiotid,0.25,4,v_cap__b1__c4,6,6280,0.8491131545208317,0.5698830559991757,-0.27923009852165603 +xiiotid,0.25,4,v_cap__b1__c4,7,7900,0.45493042952208107,0.3986569227730594,-0.05627350674902165 +xiiotid,0.25,4,v_cap__b1__c4,8,1256,0.19672131147540983,0.923076923076923,0.7263556116015132 +xiiotid,0.25,4,v_cap__b1__c4,9,696,0.3725490196078431,0.2773722627737226,-0.09517675683412047 +xiiotid,0.25,4,support_weighted__beta0.5,0,2737,0.6914893617021277,0.8125,0.12101063829787229 +xiiotid,0.25,4,support_weighted__beta0.5,1,16337,0.030266608391608392,0.40527704485488125,0.37501043646327287 +xiiotid,0.25,4,support_weighted__beta0.5,2,20397,0.9916883116883117,0.8921739130434782,-0.09951439864483358 +xiiotid,0.25,4,support_weighted__beta0.5,3,6530,0.8631578947368421,0.8247422680412372,-0.03841562669560494 +xiiotid,0.25,4,support_weighted__beta0.5,4,7342,0.48807096365416736,0.4370553291427782,-0.05101563451138913 +xiiotid,0.25,4,support_weighted__beta0.5,5,11939,0.4859075535512965,0.4858757062146893,-3.184733660721584e-05 +xiiotid,0.25,4,support_weighted__beta0.5,6,6280,0.8491131545208317,0.6092359988392819,-0.23987715568154977 +xiiotid,0.25,4,support_weighted__beta0.5,7,7900,0.45493042952208107,0.4038434194839712,-0.05108701003810989 +xiiotid,0.25,4,support_weighted__beta0.5,8,1256,0.19672131147540983,0.8333333333333334,0.6366120218579235 +xiiotid,0.25,4,support_weighted__beta0.5,9,696,0.3725490196078431,0.24203821656050953,-0.13051080304733356 +xiiotid,0.25,4,support_weighted__beta1,0,2737,0.6914893617021277,0.0,-0.6914893617021277 +xiiotid,0.25,4,support_weighted__beta1,1,16337,0.030266608391608392,0.13121299272542716,0.10094638433381878 +xiiotid,0.25,4,support_weighted__beta1,2,20397,0.9916883116883117,0.827670645852464,-0.16401766583584776 +xiiotid,0.25,4,support_weighted__beta1,3,6530,0.8631578947368421,0.7321428571428571,-0.13101503759398503 +xiiotid,0.25,4,support_weighted__beta1,4,7342,0.48807096365416736,0.034514697996392535,-0.45355626565777485 +xiiotid,0.25,4,support_weighted__beta1,5,11939,0.4859075535512965,0.4083224967490247,-0.07758505680227179 +xiiotid,0.25,4,support_weighted__beta1,6,6280,0.8491131545208317,0.6507254064416463,-0.19838774807918536 +xiiotid,0.25,4,support_weighted__beta1,7,7900,0.45493042952208107,0.4015292906670286,-0.05340113885505249 +xiiotid,0.25,4,support_weighted__beta1,8,1256,0.19672131147540983,0.9090909090909091,0.7123695976154992 +xiiotid,0.25,4,support_weighted__beta1,9,696,0.3725490196078431,0.1958762886597938,-0.17667273094804928 +5gnidd,0.5,3,v_shrink__b1__lam0.5,0,8233,0.997275204359673,0.997275204359673,0.0 +5gnidd,0.5,3,v_shrink__b1__lam0.5,1,11855,1.0,1.0,0.0 +5gnidd,0.5,3,v_shrink__b1__lam0.5,2,3482,0.822289156626506,0.8629737609329445,0.0406846043064385 +5gnidd,0.5,3,v_shrink__b1__lam0.5,3,23984,0.5995379807039,0.6332307954013865,0.03369281469748653 +5gnidd,0.5,3,v_shrink__b1__lam0.5,4,23835,0.9813571961222968,0.9835575485799701,0.00220035245767336 +5gnidd,0.5,3,v_shrink__b1__lam0.5,5,14697,0.9188260179799048,0.9300813008130081,0.011255282833103286 +5gnidd,0.5,3,v_shrink__b1__lam0.5,6,13864,0.8424967110833235,0.8844401120994274,0.04194340101610394 +5gnidd,0.5,3,v_shrink__b1__lam0.5,7,2087,0.7710740265297389,0.8266129032258064,0.055538876696067496 +5gnidd,0.5,3,v_shrink__b1__lam0.5,8,14920,0.6668387602336143,0.6606425938471879,-0.006196166386426394 +5gnidd,0.5,3,v_shrink__b1__lam0.5,9,4632,0.804147465437788,0.8885630498533725,0.0844155844155845 +5gnidd,0.5,3,v_cap__b1__c4,0,8233,0.997275204359673,0.997275204359673,0.0 +5gnidd,0.5,3,v_cap__b1__c4,1,11855,1.0,1.0,0.0 +5gnidd,0.5,3,v_cap__b1__c4,2,3482,0.822289156626506,0.8698140200286122,0.047524863402106154 +5gnidd,0.5,3,v_cap__b1__c4,3,23984,0.5995379807039,0.5987810299244285,-0.0007569507794714658 +5gnidd,0.5,3,v_cap__b1__c4,4,23835,0.9813571961222968,0.9835575485799701,0.00220035245767336 +5gnidd,0.5,3,v_cap__b1__c4,5,14697,0.9188260179799048,0.9047131147540984,-0.014112903225806384 +5gnidd,0.5,3,v_cap__b1__c4,6,13864,0.8424967110833235,0.8319189061763319,-0.010577804906991561 +5gnidd,0.5,3,v_cap__b1__c4,7,2087,0.7710740265297389,0.8,0.028925973470261157 +5gnidd,0.5,3,v_cap__b1__c4,8,14920,0.6668387602336143,0.6608854511655734,-0.005953309068040924 +5gnidd,0.5,3,v_cap__b1__c4,9,4632,0.804147465437788,0.7900782013685239,-0.01406926406926412 +5gnidd,0.5,3,support_weighted__beta0.5,0,8233,0.997275204359673,0.997275204359673,0.0 +5gnidd,0.5,3,support_weighted__beta0.5,1,11855,1.0,1.0,0.0 +5gnidd,0.5,3,support_weighted__beta0.5,2,3482,0.822289156626506,0.869942196531792,0.04765303990528591 +5gnidd,0.5,3,support_weighted__beta0.5,3,23984,0.5995379807039,0.6328921430582934,0.03335416235439348 +5gnidd,0.5,3,support_weighted__beta0.5,4,23835,0.9813571961222968,0.9894894894894894,0.008132293367192678 +5gnidd,0.5,3,support_weighted__beta0.5,5,14697,0.9188260179799048,0.9424360400444939,0.023610022064589065 +5gnidd,0.5,3,support_weighted__beta0.5,6,13864,0.8424967110833235,0.8964569058781104,0.053960194794786975 +5gnidd,0.5,3,support_weighted__beta0.5,7,2087,0.7710740265297389,0.8444444444444446,0.07337041791470567 +5gnidd,0.5,3,support_weighted__beta0.5,8,14920,0.6668387602336143,0.6606102627831439,-0.006228497450470383 +5gnidd,0.5,3,support_weighted__beta0.5,9,4632,0.804147465437788,0.893939393939394,0.089791928501606 +5gnidd,0.5,3,support_weighted__beta1,0,8233,0.997275204359673,0.997275204359673,0.0 +5gnidd,0.5,3,support_weighted__beta1,1,11855,1.0,1.0,0.0 +5gnidd,0.5,3,support_weighted__beta1,2,3482,0.822289156626506,0.9487603305785123,0.12647117395200624 +5gnidd,0.5,3,support_weighted__beta1,3,23984,0.5995379807039,0.9202334630350195,0.32069548233111955 +5gnidd,0.5,3,support_weighted__beta1,4,23835,0.9813571961222968,0.9894894894894894,0.008132293367192678 +5gnidd,0.5,3,support_weighted__beta1,5,14697,0.9188260179799048,0.956,0.03717398202009514 +5gnidd,0.5,3,support_weighted__beta1,6,13864,0.8424967110833235,0.9996466431095407,0.1571499320262172 +5gnidd,0.5,3,support_weighted__beta1,7,2087,0.7710740265297389,0.9444444444444444,0.17337041791470553 +5gnidd,0.5,3,support_weighted__beta1,8,14920,0.6668387602336143,0.679036517621346,0.012197757387731656 +5gnidd,0.5,3,support_weighted__beta1,9,4632,0.804147465437788,0.9946236559139785,0.19047619047619047 +5gnidd,0.5,4,v_shrink__b1__lam0.5,0,13066,0.9896907216494846,0.9795918367346939,-0.010098884914790718 +5gnidd,0.5,4,v_shrink__b1__lam0.5,1,17365,0.7715385486244764,0.772027397260274,0.0004888486357976785 +5gnidd,0.5,4,v_shrink__b1__lam0.5,2,1194,0.9745762711864406,0.9806866952789699,0.006110424092529265 +5gnidd,0.5,4,v_shrink__b1__lam0.5,3,5360,0.7272727272727272,0.7272727272727272,0.0 +5gnidd,0.5,4,v_shrink__b1__lam0.5,4,17032,0.9676691729323308,0.9648729446935725,-0.002796228238758247 +5gnidd,0.5,4,v_shrink__b1__lam0.5,5,3520,0.9864864864864865,0.9864864864864865,0.0 +5gnidd,0.5,4,v_shrink__b1__lam0.5,6,33983,0.6752988047808764,0.6904761904761905,0.015177385695314016 +5gnidd,0.5,4,v_shrink__b1__lam0.5,7,3343,0.9150326797385621,0.8705882352941177,-0.0444444444444444 +5gnidd,0.5,4,v_shrink__b1__lam0.5,8,17387,0.987171958429151,0.9893973501613671,0.002225391732216142 +5gnidd,0.5,4,v_shrink__b1__lam0.5,9,9339,0.9104573073233131,0.9191655801825294,0.00870827285921627 +5gnidd,0.5,4,v_cap__b1__c4,0,13066,0.9896907216494846,0.9696969696969697,-0.01999375195251485 +5gnidd,0.5,4,v_cap__b1__c4,1,17365,0.7715385486244764,0.7212821201539976,-0.05025642847047873 +5gnidd,0.5,4,v_cap__b1__c4,2,1194,0.9745762711864406,0.9806866952789699,0.006110424092529265 +5gnidd,0.5,4,v_cap__b1__c4,3,5360,0.7272727272727272,0.7272727272727272,0.0 +5gnidd,0.5,4,v_cap__b1__c4,4,17032,0.9676691729323308,0.966966966966967,-0.0007022059653637447 +5gnidd,0.5,4,v_cap__b1__c4,5,3520,0.9864864864864865,0.9864864864864865,0.0 +5gnidd,0.5,4,v_cap__b1__c4,6,33983,0.6752988047808764,0.71256038647343,0.03726158169255356 +5gnidd,0.5,4,v_cap__b1__c4,7,3343,0.9150326797385621,0.8705882352941177,-0.0444444444444444 +5gnidd,0.5,4,v_cap__b1__c4,8,17387,0.987171958429151,0.9914354230900994,0.004263464660948402 +5gnidd,0.5,4,v_cap__b1__c4,9,9339,0.9104573073233131,0.9183474300585557,0.007890122735242588 +5gnidd,0.5,4,support_weighted__beta0.5,0,13066,0.9896907216494846,0.9795918367346939,-0.010098884914790718 +5gnidd,0.5,4,support_weighted__beta0.5,1,17365,0.7715385486244764,0.7563099668362826,-0.015228581788193751 +5gnidd,0.5,4,support_weighted__beta0.5,2,1194,0.9745762711864406,0.9806866952789699,0.006110424092529265 +5gnidd,0.5,4,support_weighted__beta0.5,3,5360,0.7272727272727272,0.7272727272727272,0.0 +5gnidd,0.5,4,support_weighted__beta0.5,4,17032,0.9676691729323308,0.9662668665667167,-0.0014023063656141144 +5gnidd,0.5,4,support_weighted__beta0.5,5,3520,0.9864864864864865,0.9864864864864865,0.0 +5gnidd,0.5,4,support_weighted__beta0.5,6,33983,0.6752988047808764,0.7018348623853211,0.026536057604444663 +5gnidd,0.5,4,support_weighted__beta0.5,7,3343,0.9150326797385621,0.8705882352941177,-0.0444444444444444 +5gnidd,0.5,4,support_weighted__beta0.5,8,17387,0.987171958429151,0.9890885925632715,0.0019166341341205761 +5gnidd,0.5,4,support_weighted__beta0.5,9,9339,0.9104573073233131,0.9020715630885123,-0.008385744234800763 +5gnidd,0.5,4,support_weighted__beta1,0,13066,0.9896907216494846,0.9056603773584906,-0.08403034429099399 +5gnidd,0.5,4,support_weighted__beta1,1,17365,0.7715385486244764,0.7214141414141414,-0.050124407210334954 +5gnidd,0.5,4,support_weighted__beta1,2,1194,0.9745762711864406,0.9827586206896552,0.008182349503214614 +5gnidd,0.5,4,support_weighted__beta1,3,5360,0.7272727272727272,0.7272727272727272,0.0 +5gnidd,0.5,4,support_weighted__beta1,4,17032,0.9676691729323308,0.7975124378109453,-0.17015673512138552 +5gnidd,0.5,4,support_weighted__beta1,5,3520,0.9864864864864865,0.9864864864864865,0.0 +5gnidd,0.5,4,support_weighted__beta1,6,33983,0.6752988047808764,0.7268041237113402,0.05150531893046373 +5gnidd,0.5,4,support_weighted__beta1,7,3343,0.9150326797385621,0.8686192468619247,-0.04641343287663735 +5gnidd,0.5,4,support_weighted__beta1,8,17387,0.987171958429151,0.9887660607350757,0.0015941023059247161 +5gnidd,0.5,4,support_weighted__beta1,9,9339,0.9104573073233131,0.8576271186440678,-0.05283018867924527 diff --git a/exploration_v2/analysis/confirm_reference_summary.csv b/exploration_v2/analysis/confirm_reference_summary.csv new file mode 100644 index 0000000..ab0403c --- /dev/null +++ b/exploration_v2/analysis/confirm_reference_summary.csv @@ -0,0 +1,3 @@ +config,n_pairs,mean_delta_val_vs_original,sd_delta_val_vs_original,positive_pairs_val,mean_delta_test_vs_original,sd_delta_test_vs_original,positive_pairs_test +support_weighted__beta0.5,4,0.01090766077956501,0.019000659916111093,2,0.012164986913594772,0.01990024144848278,3 +support_weighted__beta1,4,-0.016274987402244034,0.10395541030505347,1,-0.015464250464134077,0.10566381521436177,1 diff --git a/exploration_v2/analysis/confirm_summary.csv b/exploration_v2/analysis/confirm_summary.csv new file mode 100644 index 0000000..9ac2af3 --- /dev/null +++ b/exploration_v2/analysis/confirm_summary.csv @@ -0,0 +1,3 @@ +variant,n_pairs,mean_delta_val_vs_original,sd_delta_val_vs_original,positive_pairs_val_vs_original,mean_delta_test_vs_original,sd_delta_test_vs_original,positive_pairs_test_vs_original,mean_delta_val_vs_beta1,sd_delta_val_vs_beta1,mean_delta_test_vs_beta1,sd_delta_test_vs_beta1,mean_delta_val_vs_beta05,sd_delta_val_vs_beta05,mean_delta_test_vs_beta05,sd_delta_test_vs_beta05,mean_delta_val_vs_current_best,sd_delta_val_vs_current_best,positive_pairs_val_vs_current_best,mean_delta_test_vs_current_best,sd_delta_test_vs_current_best,positive_pairs_test_vs_current_best,improved_vs_original_per_plan,S1_val_deltas,S2_val_deltas +v_shrink__b1__lam0.5,4,0.017994808339324264,0.014072367228704582,4,0.018354318695924132,0.014319068327916913,4,0.0342697957415683,0.10649767725215149,0.03381856916005821,0.1082507145556489,0.007087147559759255,0.023366313613711127,0.00618933178232936,0.024241921772951987,-0.01532128227473148,0.05206218407417258,2,-0.015455669432363206,0.052417393479881856,2,True,0.008456|0.032863,0.026902|0.003758 +v_cap__b1__c4,4,0.00467808081785806,0.02625027663765537,2,0.005868542758943829,0.025844914703010976,2,0.020953068220102095,0.09798314473745867,0.021332793223077906,0.09849148681223491,-0.006229579961706949,0.023343093528822526,-0.006296444154650943,0.023094240221471166,-0.028638009796197683,0.05968831126237434,1,-0.02794144536934351,0.059325448594970964,1,False,0.041031|-0.021430,0.001962|-0.002850 diff --git a/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__characterization.csv b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__characterization.csv new file mode 100644 index 0000000..053e5b2 --- /dev/null +++ b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,1571,112226,33441.5,37695.7725116491,1.127215361501401,71.43602800763844 +1,HTTPFlood,98569,10,0,25,29013,9856.9,9965.921477214235,1.0110604223654736,1160.52 +2,ICMPFlood,808,10,0,1,303,80.8,84.37155918910115,1.044202465211648,303.0 +3,SYNFlood,6805,10,0,77,1688,680.5,557.1650114642878,0.8187582828277558,21.92207792207792 +4,SYNScan,14030,9,1,1,9592,1558.888888888889,2931.2507169179676,1.8803461477021888,9592.0 +5,SlowrateDoS,51187,10,0,28,20803,5118.7,7606.314338100943,1.48598557018402,742.9642857142857 +6,TCPConnectScan,14037,9,1,7,4056,1559.6666666666667,1501.5754689288476,0.962754094205288,579.4285714285714 +7,UDPFlood,320138,10,0,189,107583,32013.8,36126.15487371995,1.1284556932860188,569.2222222222222 +8,UDPScan,11134,10,0,5,8048,1113.4,2354.6204874671416,2.1148019467102044,1609.6 diff --git a/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__manifest_meta.json b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__manifest_meta.json new file mode 100644 index 0000000..1616f60 --- /dev/null +++ b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__manifest_meta.json @@ -0,0 +1,13 @@ +{ + "alpha": 0.5, + "created_utc": "2026-10-31T05:06:50Z", + "dataset": "5gnidd", + "manifest_content_hash": "bf212b0e11ba63d44be64c11215d085658d506065922a26960391eece1143b5b", + "manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/5gnidd__alpha0.5__seed3.json", + "num_classes": 9, + "num_clients": 10, + "partition_hash": "0739f15c51fe4fa2250f717e8f52548706cb7093581447cd43a4b2364f6ec058", + "preprocessing_hash": "60b5682b39184fd7d24e117430a00c160c5c7e95d1bc1fcaa99745108f514d81", + "procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing", + "seed": 3 +} diff --git a/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__rare_unseen.csv b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__rare_unseen.csv new file mode 100644 index 0000000..53badc3 --- /dev/null +++ b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.5,3,0,0,Benign,12367,False,False +5gnidd,0.5,3,0,1,HTTPFlood,24330,False,False +5gnidd,0.5,3,0,2,ICMPFlood,109,True,False +5gnidd,0.5,3,0,3,SYNFlood,732,False,False +5gnidd,0.5,3,0,4,SYNScan,1299,False,False +5gnidd,0.5,3,0,5,SlowrateDoS,2036,False,False +5gnidd,0.5,3,0,6,TCPConnectScan,261,False,False +5gnidd,0.5,3,0,7,UDPFlood,107583,False,False +5gnidd,0.5,3,0,8,UDPScan,129,True,False +5gnidd,0.5,3,1,0,Benign,86013,False,False +5gnidd,0.5,3,1,1,HTTPFlood,9922,False,False +5gnidd,0.5,3,1,2,ICMPFlood,8,True,False +5gnidd,0.5,3,1,3,SYNFlood,104,False,False +5gnidd,0.5,3,1,4,SYNScan,31,False,False +5gnidd,0.5,3,1,5,SlowrateDoS,3166,False,False +5gnidd,0.5,3,1,6,TCPConnectScan,1928,False,False +5gnidd,0.5,3,1,7,UDPFlood,6739,False,False +5gnidd,0.5,3,1,8,UDPScan,5,True,False +5gnidd,0.5,3,2,0,Benign,112226,False,False +5gnidd,0.5,3,2,1,HTTPFlood,6136,False,False +5gnidd,0.5,3,2,2,ICMPFlood,97,True,False +5gnidd,0.5,3,2,3,SYNFlood,152,False,False +5gnidd,0.5,3,2,4,SYNScan,143,False,False +5gnidd,0.5,3,2,5,SlowrateDoS,28,True,False +5gnidd,0.5,3,2,6,TCPConnectScan,966,False,False +5gnidd,0.5,3,2,7,UDPFlood,13880,False,False +5gnidd,0.5,3,2,8,UDPScan,301,False,False +5gnidd,0.5,3,3,0,Benign,7322,False,False +5gnidd,0.5,3,3,1,HTTPFlood,29013,False,False +5gnidd,0.5,3,3,2,ICMPFlood,115,True,False +5gnidd,0.5,3,3,3,SYNFlood,213,False,False +5gnidd,0.5,3,3,4,SYNScan,9592,False,False +5gnidd,0.5,3,3,5,SlowrateDoS,159,True,False +5gnidd,0.5,3,3,6,TCPConnectScan,373,False,False +5gnidd,0.5,3,3,7,UDPFlood,2354,False,False +5gnidd,0.5,3,3,8,UDPScan,503,False,False +5gnidd,0.5,3,4,0,Benign,28752,False,False +5gnidd,0.5,3,4,1,HTTPFlood,497,False,False +5gnidd,0.5,3,4,2,ICMPFlood,15,True,False +5gnidd,0.5,3,4,3,SYNFlood,1688,False,False +5gnidd,0.5,3,4,4,SYNScan,71,False,False +5gnidd,0.5,3,4,5,SlowrateDoS,19630,False,False +5gnidd,0.5,3,4,6,TCPConnectScan,0,False,True +5gnidd,0.5,3,4,7,UDPFlood,607,False,False +5gnidd,0.5,3,4,8,UDPScan,67,True,False +5gnidd,0.5,3,5,0,Benign,11126,False,False +5gnidd,0.5,3,5,1,HTTPFlood,366,False,False +5gnidd,0.5,3,5,2,ICMPFlood,37,True,False +5gnidd,0.5,3,5,3,SYNFlood,786,False,False +5gnidd,0.5,3,5,4,SYNScan,519,False,False +5gnidd,0.5,3,5,5,SlowrateDoS,1272,False,False +5gnidd,0.5,3,5,6,TCPConnectScan,140,True,False +5gnidd,0.5,3,5,7,UDPFlood,84223,False,False +5gnidd,0.5,3,5,8,UDPScan,1613,False,False +5gnidd,0.5,3,6,0,Benign,7958,False,False +5gnidd,0.5,3,6,1,HTTPFlood,249,False,False +5gnidd,0.5,3,6,2,ICMPFlood,40,True,False +5gnidd,0.5,3,6,3,SYNFlood,912,False,False +5gnidd,0.5,3,6,4,SYNScan,93,True,False +5gnidd,0.5,3,6,5,SlowrateDoS,801,False,False +5gnidd,0.5,3,6,6,TCPConnectScan,3851,False,False +5gnidd,0.5,3,6,7,UDPFlood,45015,False,False +5gnidd,0.5,3,6,8,UDPScan,8048,False,False +5gnidd,0.5,3,7,0,Benign,64584,False,False +5gnidd,0.5,3,7,1,HTTPFlood,25,False,False +5gnidd,0.5,3,7,2,ICMPFlood,1,True,False +5gnidd,0.5,3,7,3,SYNFlood,77,False,False +5gnidd,0.5,3,7,4,SYNScan,2281,False,False +5gnidd,0.5,3,7,5,SlowrateDoS,20803,False,False +5gnidd,0.5,3,7,6,TCPConnectScan,7,True,False +5gnidd,0.5,3,7,7,UDPFlood,12615,False,False +5gnidd,0.5,3,7,8,UDPScan,17,False,False +5gnidd,0.5,3,8,0,Benign,2496,False,False +5gnidd,0.5,3,8,1,HTTPFlood,14929,False,False +5gnidd,0.5,3,8,2,ICMPFlood,303,False,False +5gnidd,0.5,3,8,3,SYNFlood,1582,False,False +5gnidd,0.5,3,8,4,SYNScan,1,True,False +5gnidd,0.5,3,8,5,SlowrateDoS,2192,False,False +5gnidd,0.5,3,8,6,TCPConnectScan,2455,False,False +5gnidd,0.5,3,8,7,UDPFlood,189,True,False +5gnidd,0.5,3,8,8,UDPScan,234,False,False +5gnidd,0.5,3,9,0,Benign,1571,False,False +5gnidd,0.5,3,9,1,HTTPFlood,13102,False,False +5gnidd,0.5,3,9,2,ICMPFlood,83,True,False +5gnidd,0.5,3,9,3,SYNFlood,559,False,False +5gnidd,0.5,3,9,4,SYNScan,0,False,True +5gnidd,0.5,3,9,5,SlowrateDoS,1100,False,False +5gnidd,0.5,3,9,6,TCPConnectScan,4056,False,False +5gnidd,0.5,3,9,7,UDPFlood,46933,False,False +5gnidd,0.5,3,9,8,UDPScan,217,True,False diff --git a/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__characterization.csv b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__characterization.csv new file mode 100644 index 0000000..4d8601f --- /dev/null +++ b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,143,159078,33441.5,47131.86755317468,1.409382580122742,1112.4335664335665 +1,HTTPFlood,98569,10,0,4,47186,9856.9,14843.031647544243,1.5058519055224506,11796.5 +2,ICMPFlood,808,9,1,5,318,89.77777777777777,108.34284858497202,1.206789155030629,63.6 +3,SYNFlood,6805,10,0,7,2927,680.5,975.6094761737403,1.4336656519819844,418.14285714285717 +4,SYNScan,14030,10,0,45,8239,1403.0,2313.2844183109,1.6488128427019957,183.0888888888889 +5,SlowrateDoS,51187,10,0,7,22272,5118.7,6802.114951248619,1.3288754862071657,3181.714285714286 +6,TCPConnectScan,14037,10,0,1,5229,1403.7,1552.9292353484752,1.1063113452649962,5229.0 +7,UDPFlood,320138,10,0,6152,111407,32013.8,32088.759828949453,1.002341484889312,18.109070221066318 +8,UDPScan,11134,9,1,15,3055,1237.111111111111,1139.6573992451947,0.9212247703616627,203.66666666666666 diff --git a/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__manifest_meta.json b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__manifest_meta.json new file mode 100644 index 0000000..5055883 --- /dev/null +++ b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__manifest_meta.json @@ -0,0 +1,13 @@ +{ + "alpha": 0.5, + "created_utc": "2026-10-31T05:07:28Z", + "dataset": "5gnidd", + "manifest_content_hash": "80fad04f10a0014a58ea97e87ab901d70cf48f29d99f5803510c0cd44b2618b9", + "manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/5gnidd__alpha0.5__seed4.json", + "num_classes": 9, + "num_clients": 10, + "partition_hash": "21a507d9d94f31154c5fb0c6833344738d1a137c78950f5284dec26d727e983a", + "preprocessing_hash": "ffb7b2a1adb6cf8a292ea6e88dca574b44446da2b3ae18f690ae6fc59699e23d", + "procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing", + "seed": 4 +} diff --git a/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__rare_unseen.csv b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__rare_unseen.csv new file mode 100644 index 0000000..cf0435a --- /dev/null +++ b/exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.5,4,0,0,Benign,159078,False,False +5gnidd,0.5,4,0,1,HTTPFlood,11071,False,False +5gnidd,0.5,4,0,2,ICMPFlood,39,True,False +5gnidd,0.5,4,0,3,SYNFlood,936,False,False +5gnidd,0.5,4,0,4,SYNScan,8239,False,False +5gnidd,0.5,4,0,5,SlowrateDoS,1050,False,False +5gnidd,0.5,4,0,6,TCPConnectScan,33,True,False +5gnidd,0.5,4,0,7,UDPFlood,15599,False,False +5gnidd,0.5,4,0,8,UDPScan,241,False,False +5gnidd,0.5,4,1,0,Benign,15658,False,False +5gnidd,0.5,4,1,1,HTTPFlood,27988,False,False +5gnidd,0.5,4,1,2,ICMPFlood,318,False,False +5gnidd,0.5,4,1,3,SYNFlood,7,True,False +5gnidd,0.5,4,1,4,SYNScan,674,False,False +5gnidd,0.5,4,1,5,SlowrateDoS,7,True,False +5gnidd,0.5,4,1,6,TCPConnectScan,5229,False,False +5gnidd,0.5,4,1,7,UDPFlood,6152,False,False +5gnidd,0.5,4,1,8,UDPScan,93,False,False +5gnidd,0.5,4,2,0,Benign,5874,False,False +5gnidd,0.5,4,2,1,HTTPFlood,2211,False,False +5gnidd,0.5,4,2,2,ICMPFlood,0,False,True +5gnidd,0.5,4,2,3,SYNFlood,2113,False,False +5gnidd,0.5,4,2,4,SYNScan,893,False,False +5gnidd,0.5,4,2,5,SlowrateDoS,1049,False,False +5gnidd,0.5,4,2,6,TCPConnectScan,490,True,False +5gnidd,0.5,4,2,7,UDPFlood,13122,False,False +5gnidd,0.5,4,2,8,UDPScan,131,True,False +5gnidd,0.5,4,3,0,Benign,76962,False,False +5gnidd,0.5,4,3,1,HTTPFlood,851,False,False +5gnidd,0.5,4,3,2,ICMPFlood,10,True,False +5gnidd,0.5,4,3,3,SYNFlood,26,True,False +5gnidd,0.5,4,3,4,SYNScan,1223,False,False +5gnidd,0.5,4,3,5,SlowrateDoS,7546,False,False +5gnidd,0.5,4,3,6,TCPConnectScan,846,False,False +5gnidd,0.5,4,3,7,UDPFlood,57683,False,False +5gnidd,0.5,4,3,8,UDPScan,3055,False,False +5gnidd,0.5,4,4,0,Benign,3588,False,False +5gnidd,0.5,4,4,1,HTTPFlood,47186,False,False +5gnidd,0.5,4,4,2,ICMPFlood,243,True,False +5gnidd,0.5,4,4,3,SYNFlood,26,True,False +5gnidd,0.5,4,4,4,SYNScan,930,False,False +5gnidd,0.5,4,4,5,SlowrateDoS,11733,False,False +5gnidd,0.5,4,4,6,TCPConnectScan,1121,False,False +5gnidd,0.5,4,4,7,UDPFlood,22035,False,False +5gnidd,0.5,4,4,8,UDPScan,1898,False,False +5gnidd,0.5,4,5,0,Benign,26897,False,False +5gnidd,0.5,4,5,1,HTTPFlood,235,False,False +5gnidd,0.5,4,5,2,ICMPFlood,60,True,False +5gnidd,0.5,4,5,3,SYNFlood,2927,False,False +5gnidd,0.5,4,5,4,SYNScan,181,True,False +5gnidd,0.5,4,5,5,SlowrateDoS,263,False,False +5gnidd,0.5,4,5,6,TCPConnectScan,2914,False,False +5gnidd,0.5,4,5,7,UDPFlood,58429,False,False +5gnidd,0.5,4,5,8,UDPScan,2934,False,False +5gnidd,0.5,4,6,0,Benign,143,False,False +5gnidd,0.5,4,6,1,HTTPFlood,5129,False,False +5gnidd,0.5,4,6,2,ICMPFlood,112,False,False +5gnidd,0.5,4,6,3,SYNFlood,237,False,False +5gnidd,0.5,4,6,4,SYNScan,45,True,False +5gnidd,0.5,4,6,5,SlowrateDoS,5427,False,False +5gnidd,0.5,4,6,6,TCPConnectScan,36,True,False +5gnidd,0.5,4,6,7,UDPFlood,7654,False,False +5gnidd,0.5,4,6,8,UDPScan,1444,False,False +5gnidd,0.5,4,7,0,Benign,30083,False,False +5gnidd,0.5,4,7,1,HTTPFlood,3124,False,False +5gnidd,0.5,4,7,2,ICMPFlood,10,True,False +5gnidd,0.5,4,7,3,SYNFlood,8,True,False +5gnidd,0.5,4,7,4,SYNScan,79,False,False +5gnidd,0.5,4,7,5,SlowrateDoS,357,False,False +5gnidd,0.5,4,7,6,TCPConnectScan,1910,False,False +5gnidd,0.5,4,7,7,UDPFlood,111407,False,False +5gnidd,0.5,4,7,8,UDPScan,0,False,True +5gnidd,0.5,4,8,0,Benign,405,False,False +5gnidd,0.5,4,8,1,HTTPFlood,4,True,False +5gnidd,0.5,4,8,2,ICMPFlood,5,False,False +5gnidd,0.5,4,8,3,SYNFlood,413,False,False +5gnidd,0.5,4,8,4,SYNScan,1110,False,False +5gnidd,0.5,4,8,5,SlowrateDoS,1483,False,False +5gnidd,0.5,4,8,6,TCPConnectScan,1,True,False +5gnidd,0.5,4,8,7,UDPFlood,14987,False,False +5gnidd,0.5,4,8,8,UDPScan,1323,False,False +5gnidd,0.5,4,9,0,Benign,15727,False,False +5gnidd,0.5,4,9,1,HTTPFlood,770,False,False +5gnidd,0.5,4,9,2,ICMPFlood,11,True,False +5gnidd,0.5,4,9,3,SYNFlood,112,False,False +5gnidd,0.5,4,9,4,SYNScan,656,False,False +5gnidd,0.5,4,9,5,SlowrateDoS,22272,False,False +5gnidd,0.5,4,9,6,TCPConnectScan,1457,False,False +5gnidd,0.5,4,9,7,UDPFlood,13070,False,False +5gnidd,0.5,4,9,8,UDPScan,15,True,False diff --git a/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__characterization.csv b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__characterization.csv new file mode 100644 index 0000000..3bb9b93 --- /dev/null +++ b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,8,2,4,1191,250.5,373.4360721730026,1.490762763165679,297.75 +1,Exfiltration,15494,10,0,2,5033,1549.4,1886.1375983739892,1.217334192832057,2516.5 +2,Exploitation,793,6,4,7,473,132.16666666666666,167.6011303329691,1.268104390917799,67.57142857142857 +3,Lateral_movement,22117,9,1,2,13685,2457.4444444444443,4284.391882457665,1.7434338717782243,6842.5 +4,Normal,292777,10,0,549,150706,29277.7,51810.80511254385,1.769633718240977,274.5100182149362 +5,RDOS,98883,10,0,9,58544,9888.3,17738.631790811825,1.7939010538527176,6504.888888888889 +6,Reconnaissance,89161,10,0,2,35323,8916.1,10550.735959638077,1.183335310240809,17661.5 +7,Tampering,3586,7,3,2,2013,512.2857142857143,687.5847406030795,1.3421899565592739,1006.5 +8,Weaponization,45094,10,0,31,25866,4509.4,7690.813119560246,1.7055069675700196,834.3870967741935 diff --git a/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__manifest_meta.json b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__manifest_meta.json new file mode 100644 index 0000000..d11cba7 --- /dev/null +++ b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__manifest_meta.json @@ -0,0 +1,13 @@ +{ + "alpha": 0.25, + "created_utc": "2026-10-31T05:05:04Z", + "dataset": "xiiotid", + "manifest_content_hash": "1021d6ce77fdb0d4ddf940988f221fd85c8be4c2162d804e1c0da169ed9d7e2f", + "manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/xiiotid__alpha0.25__seed3.json", + "num_classes": 9, + "num_clients": 10, + "partition_hash": "85a8625b81d3624dec9e5322979e2f1547f330c60d5e822e32c12177195ed602", + "preprocessing_hash": "e22db95e5109bfa02701d8ce450453a7c8f7e408c94e81dfe2b95ef4ab2a441f", + "procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing", + "seed": 3 +} diff --git a/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__rare_unseen.csv b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__rare_unseen.csv new file mode 100644 index 0000000..c9718e6 --- /dev/null +++ b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.25,3,0,0,C&C,21,False,False +xiiotid,0.25,3,0,1,Exfiltration,1003,False,False +xiiotid,0.25,3,0,2,Exploitation,0,False,True +xiiotid,0.25,3,0,3,Lateral_movement,7,True,False +xiiotid,0.25,3,0,4,Normal,2950,False,False +xiiotid,0.25,3,0,5,RDOS,14,True,False +xiiotid,0.25,3,0,6,Reconnaissance,11735,False,False +xiiotid,0.25,3,0,7,Tampering,0,False,True +xiiotid,0.25,3,0,8,Weaponization,342,False,False +xiiotid,0.25,3,1,0,C&C,27,True,False +xiiotid,0.25,3,1,1,Exfiltration,1029,False,False +xiiotid,0.25,3,1,2,Exploitation,7,True,False +xiiotid,0.25,3,1,3,Lateral_movement,199,False,False +xiiotid,0.25,3,1,4,Normal,549,False,False +xiiotid,0.25,3,1,5,RDOS,1426,False,False +xiiotid,0.25,3,1,6,Reconnaissance,8567,False,False +xiiotid,0.25,3,1,7,Tampering,370,False,False +xiiotid,0.25,3,1,8,Weaponization,1278,False,False +xiiotid,0.25,3,2,0,C&C,1191,False,False +xiiotid,0.25,3,2,1,Exfiltration,483,False,False +xiiotid,0.25,3,2,2,Exploitation,23,True,False +xiiotid,0.25,3,2,3,Lateral_movement,5147,False,False +xiiotid,0.25,3,2,4,Normal,17775,False,False +xiiotid,0.25,3,2,5,RDOS,234,False,False +xiiotid,0.25,3,2,6,Reconnaissance,139,False,False +xiiotid,0.25,3,2,7,Tampering,0,False,True +xiiotid,0.25,3,2,8,Weaponization,31,True,False +xiiotid,0.25,3,3,0,C&C,0,False,True +xiiotid,0.25,3,3,1,Exfiltration,2,True,False +xiiotid,0.25,3,3,2,Exploitation,36,False,False +xiiotid,0.25,3,3,3,Lateral_movement,0,False,True +xiiotid,0.25,3,3,4,Normal,3006,False,False +xiiotid,0.25,3,3,5,RDOS,58,False,False +xiiotid,0.25,3,3,6,Reconnaissance,12,True,False +xiiotid,0.25,3,3,7,Tampering,12,False,False +xiiotid,0.25,3,3,8,Weaponization,1420,False,False +xiiotid,0.25,3,4,0,C&C,15,True,False +xiiotid,0.25,3,4,1,Exfiltration,18,True,False +xiiotid,0.25,3,4,2,Exploitation,39,False,False +xiiotid,0.25,3,4,3,Lateral_movement,2326,False,False +xiiotid,0.25,3,4,4,Normal,2692,False,False +xiiotid,0.25,3,4,5,RDOS,58544,False,False +xiiotid,0.25,3,4,6,Reconnaissance,773,False,False +xiiotid,0.25,3,4,7,Tampering,0,False,True +xiiotid,0.25,3,4,8,Weaponization,25866,False,False +xiiotid,0.25,3,5,0,C&C,320,True,False +xiiotid,0.25,3,5,1,Exfiltration,4958,False,False +xiiotid,0.25,3,5,2,Exploitation,0,False,True +xiiotid,0.25,3,5,3,Lateral_movement,31,True,False +xiiotid,0.25,3,5,4,Normal,966,False,False +xiiotid,0.25,3,5,5,RDOS,18218,False,False +xiiotid,0.25,3,5,6,Reconnaissance,3006,False,False +xiiotid,0.25,3,5,7,Tampering,949,False,False +xiiotid,0.25,3,5,8,Weaponization,8009,False,False +xiiotid,0.25,3,6,0,C&C,0,False,True +xiiotid,0.25,3,6,1,Exfiltration,2705,False,False +xiiotid,0.25,3,6,2,Exploitation,215,False,False +xiiotid,0.25,3,6,3,Lateral_movement,167,True,False +xiiotid,0.25,3,6,4,Normal,1157,False,False +xiiotid,0.25,3,6,5,RDOS,1247,False,False +xiiotid,0.25,3,6,6,Reconnaissance,13942,False,False +xiiotid,0.25,3,6,7,Tampering,2,True,False +xiiotid,0.25,3,6,8,Weaponization,347,False,False +xiiotid,0.25,3,7,0,C&C,165,False,False +xiiotid,0.25,3,7,1,Exfiltration,35,True,False +xiiotid,0.25,3,7,2,Exploitation,473,False,False +xiiotid,0.25,3,7,3,Lateral_movement,13685,False,False +xiiotid,0.25,3,7,4,Normal,111159,False,False +xiiotid,0.25,3,7,5,RDOS,19011,False,False +xiiotid,0.25,3,7,6,Reconnaissance,15662,False,False +xiiotid,0.25,3,7,7,Tampering,2013,False,False +xiiotid,0.25,3,7,8,Weaponization,120,True,False +xiiotid,0.25,3,8,0,C&C,4,False,False +xiiotid,0.25,3,8,1,Exfiltration,228,False,False +xiiotid,0.25,3,8,2,Exploitation,0,False,True +xiiotid,0.25,3,8,3,Lateral_movement,2,True,False +xiiotid,0.25,3,8,4,Normal,150706,False,False +xiiotid,0.25,3,8,5,RDOS,122,False,False +xiiotid,0.25,3,8,6,Reconnaissance,35323,False,False +xiiotid,0.25,3,8,7,Tampering,2,True,False +xiiotid,0.25,3,8,8,Weaponization,83,False,False +xiiotid,0.25,3,9,0,C&C,261,False,False +xiiotid,0.25,3,9,1,Exfiltration,5033,False,False +xiiotid,0.25,3,9,2,Exploitation,0,False,True +xiiotid,0.25,3,9,3,Lateral_movement,553,False,False +xiiotid,0.25,3,9,4,Normal,1817,False,False +xiiotid,0.25,3,9,5,RDOS,9,True,False +xiiotid,0.25,3,9,6,Reconnaissance,2,True,False +xiiotid,0.25,3,9,7,Tampering,238,False,False +xiiotid,0.25,3,9,8,Weaponization,7598,False,False diff --git a/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__characterization.csv b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__characterization.csv new file mode 100644 index 0000000..af0aec8 --- /dev/null +++ b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,9,1,1,1110,222.66666666666666,338.2435020316971,1.5190576438549273,1110.0 +1,Exfiltration,15494,10,0,1,7839,1549.4,2701.0935637256257,1.743315840793614,7839.0 +2,Exploitation,793,8,2,3,667,99.125,215.40568556795338,2.1730712289326948,222.33333333333334 +3,Lateral_movement,22117,9,1,11,9367,2457.4444444444443,3120.737203187417,1.2699115987107996,851.5454545454545 +4,Normal,292777,9,1,1,193448,32530.777777777777,58303.312956712805,1.7922508141364084,193448.0 +5,RDOS,98883,9,1,24,36163,10987.0,13856.277750936977,1.2611520661633728,1506.7916666666667 +6,Reconnaissance,89161,10,0,25,32287,8916.1,10687.687022457198,1.1986952840880203,1291.48 +7,Tampering,3586,10,0,3,2959,358.6,870.6838921215897,2.4280086227595916,986.3333333333334 +8,Weaponization,45094,10,0,4,18287,4509.4,6073.340385652693,1.3468178439820582,4571.75 diff --git a/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__manifest_meta.json b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__manifest_meta.json new file mode 100644 index 0000000..3cb4bd6 --- /dev/null +++ b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__manifest_meta.json @@ -0,0 +1,13 @@ +{ + "alpha": 0.25, + "created_utc": "2026-10-31T05:06:13Z", + "dataset": "xiiotid", + "manifest_content_hash": "1ea850f1be9ddda24a69d192ef678cb337e62edb60978985615271a15d46d2c8", + "manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/xiiotid__alpha0.25__seed4.json", + "num_classes": 9, + "num_clients": 10, + "partition_hash": "ff47f504d5b604935333ccbfddd060c18d44d672104e5455f86dcefe48315887", + "preprocessing_hash": "4837f4ab5939fe7bf7f83168d915e67d570086d8534a521a9a9827f10aa41086", + "procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing", + "seed": 4 +} diff --git a/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__rare_unseen.csv b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__rare_unseen.csv new file mode 100644 index 0000000..5f49df1 --- /dev/null +++ b/exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.25,4,0,0,C&C,1,True,False +xiiotid,0.25,4,0,1,Exfiltration,109,False,False +xiiotid,0.25,4,0,2,Exploitation,667,False,False +xiiotid,0.25,4,0,3,Lateral_movement,156,False,False +xiiotid,0.25,4,0,4,Normal,193448,False,False +xiiotid,0.25,4,0,5,RDOS,29952,False,False +xiiotid,0.25,4,0,6,Reconnaissance,1314,False,False +xiiotid,0.25,4,0,7,Tampering,3,True,False +xiiotid,0.25,4,0,8,Weaponization,9802,False,False +xiiotid,0.25,4,1,0,C&C,20,False,False +xiiotid,0.25,4,1,1,Exfiltration,368,False,False +xiiotid,0.25,4,1,2,Exploitation,4,True,False +xiiotid,0.25,4,1,3,Lateral_movement,1852,False,False +xiiotid,0.25,4,1,4,Normal,13384,False,False +xiiotid,0.25,4,1,5,RDOS,262,False,False +xiiotid,0.25,4,1,6,Reconnaissance,1102,False,False +xiiotid,0.25,4,1,7,Tampering,42,False,False +xiiotid,0.25,4,1,8,Weaponization,4,True,False +xiiotid,0.25,4,2,0,C&C,58,False,False +xiiotid,0.25,4,2,1,Exfiltration,336,False,False +xiiotid,0.25,4,2,2,Exploitation,0,False,True +xiiotid,0.25,4,2,3,Lateral_movement,1331,False,False +xiiotid,0.25,4,2,4,Normal,0,False,False +xiiotid,0.25,4,2,5,RDOS,218,False,False +xiiotid,0.25,4,2,6,Reconnaissance,32287,False,False +xiiotid,0.25,4,2,7,Tampering,5,True,False +xiiotid,0.25,4,2,8,Weaponization,27,True,False +xiiotid,0.25,4,3,0,C&C,0,False,True +xiiotid,0.25,4,3,1,Exfiltration,7839,False,False +xiiotid,0.25,4,3,2,Exploitation,10,True,False +xiiotid,0.25,4,3,3,Lateral_movement,11,True,False +xiiotid,0.25,4,3,4,Normal,22325,False,False +xiiotid,0.25,4,3,5,RDOS,619,False,False +xiiotid,0.25,4,3,6,Reconnaissance,12186,False,False +xiiotid,0.25,4,3,7,Tampering,2959,False,False +xiiotid,0.25,4,3,8,Weaponization,7624,False,False +xiiotid,0.25,4,4,0,C&C,413,False,False +xiiotid,0.25,4,4,1,Exfiltration,94,False,False +xiiotid,0.25,4,4,2,Exploitation,4,True,False +xiiotid,0.25,4,4,3,Lateral_movement,9367,False,False +xiiotid,0.25,4,4,4,Normal,13997,False,False +xiiotid,0.25,4,4,5,RDOS,0,False,True +xiiotid,0.25,4,4,6,Reconnaissance,1924,False,False +xiiotid,0.25,4,4,7,Tampering,6,True,False +xiiotid,0.25,4,4,8,Weaponization,47,False,False +xiiotid,0.25,4,5,0,C&C,238,False,False +xiiotid,0.25,4,5,1,Exfiltration,5,True,False +xiiotid,0.25,4,5,2,Exploitation,3,True,False +xiiotid,0.25,4,5,3,Lateral_movement,0,False,True +xiiotid,0.25,4,5,4,Normal,1,False,False +xiiotid,0.25,4,5,5,RDOS,24028,False,False +xiiotid,0.25,4,5,6,Reconnaissance,22974,False,False +xiiotid,0.25,4,5,7,Tampering,62,False,False +xiiotid,0.25,4,5,8,Weaponization,5,False,False +xiiotid,0.25,4,6,0,C&C,4,True,False +xiiotid,0.25,4,6,1,Exfiltration,1,True,False +xiiotid,0.25,4,6,2,Exploitation,50,False,False +xiiotid,0.25,4,6,3,Lateral_movement,121,False,False +xiiotid,0.25,4,6,4,Normal,167,False,False +xiiotid,0.25,4,6,5,RDOS,36163,False,False +xiiotid,0.25,4,6,6,Reconnaissance,14248,False,False +xiiotid,0.25,4,6,7,Tampering,75,False,False +xiiotid,0.25,4,6,8,Weaponization,18287,False,False +xiiotid,0.25,4,7,0,C&C,1110,False,False +xiiotid,0.25,4,7,1,Exfiltration,508,False,False +xiiotid,0.25,4,7,2,Exploitation,47,True,False +xiiotid,0.25,4,7,3,Lateral_movement,6770,False,False +xiiotid,0.25,4,7,4,Normal,61,False,False +xiiotid,0.25,4,7,5,RDOS,2656,False,False +xiiotid,0.25,4,7,6,Reconnaissance,25,True,False +xiiotid,0.25,4,7,7,Tampering,88,False,False +xiiotid,0.25,4,7,8,Weaponization,147,False,False +xiiotid,0.25,4,8,0,C&C,78,False,False +xiiotid,0.25,4,8,1,Exfiltration,328,False,False +xiiotid,0.25,4,8,2,Exploitation,0,False,True +xiiotid,0.25,4,8,3,Lateral_movement,1112,False,False +xiiotid,0.25,4,8,4,Normal,42050,False,False +xiiotid,0.25,4,8,5,RDOS,24,True,False +xiiotid,0.25,4,8,6,Reconnaissance,132,False,False +xiiotid,0.25,4,8,7,Tampering,46,False,False +xiiotid,0.25,4,8,8,Weaponization,10,True,False +xiiotid,0.25,4,9,0,C&C,82,True,False +xiiotid,0.25,4,9,1,Exfiltration,5906,False,False +xiiotid,0.25,4,9,2,Exploitation,8,True,False +xiiotid,0.25,4,9,3,Lateral_movement,1397,False,False +xiiotid,0.25,4,9,4,Normal,7344,False,False +xiiotid,0.25,4,9,5,RDOS,4961,False,False +xiiotid,0.25,4,9,6,Reconnaissance,2969,False,False +xiiotid,0.25,4,9,7,Tampering,300,False,False +xiiotid,0.25,4,9,8,Weaponization,9141,False,False diff --git a/exploration_v2/analysis/reference_consistency.csv b/exploration_v2/analysis/reference_consistency.csv new file mode 100644 index 0000000..50d2979 --- /dev/null +++ b/exploration_v2/analysis/reference_consistency.csv @@ -0,0 +1,3 @@ +condition,dataset,alpha,new_original_val_global,restored_original_val_global,sweep_beta0_val_global,new_minus_restored,new_minus_sweep_beta0 +S1,xiiotid,0.25,0.6380388187784778,0.6415812303524282,0.643275274917486,-0.0035424115739504103,-0.00523645613900825 +S2,5gnidd,0.5,0.8615257707605136,0.8694027416468965,0.8676517373672202,-0.00787697088638284,-0.006125966606706612 diff --git a/exploration_v2/analysis/run_inventory.csv b/exploration_v2/analysis/run_inventory.csv new file mode 100644 index 0000000..f6d0cfa --- /dev/null +++ b/exploration_v2/analysis/run_inventory.csv @@ -0,0 +1,35 @@ +phase,run_id,dataset,alpha,seed,mode,beta,shrink_lambda,cap_factor,min_support,label,status,wall_s,peak_rss_kb,exit_code,notes +screening,xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,0,original_protean,,,,,original_protean,done,10879.00,8788432,0, +screening,5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0,original_protean,,,,,original_protean,done,16807.00,10800928,0, +screening,xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,0,v_beta,1.5,,,,v_beta__b1.5,done,11048.00,8788696,0, +screening,xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,0,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,10965.00,8788104,0, +screening,xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,0,v_cap,1.0,,4,,v_cap__b1__c4,done,11022.00,8789080,0, +screening,xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,0,v_cap,1.0,,2,,v_cap__b1__c2,done,10960.00,8787984,0, +screening,xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,0,v_minsup,0.0,,,10,v_minsup__t10,done,10904.00,8788056,0, +screening,xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,0,v_logweight,0.0,,,,v_logweight,done,10645.00,8787588,0, +screening,5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,0,v_beta,1.5,,,,v_beta__b1.5,done,17571.00,10787996,0, +screening,5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,0,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,17467.00,10791372,0, +screening,5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,0,v_cap,1.0,,4,,v_cap__b1__c4,done,16374.00,10795492,0, +screening,5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,0,v_cap,1.0,,2,,v_cap__b1__c2,done,15875.00,10794444,0, +screening,5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0,v_minsup,0.0,,,10,v_minsup__t10,done,16232.00,10787860,0, +screening,5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0,v_logweight,0.0,,,,v_logweight,done,17342.00,10806700,0, +confirm,xiiotid__alpha0.25__seed3__original_protean,xiiotid,0.25,3,original_protean,,,,,original_protean,done,11102.00,10986852,0, +confirm,xiiotid__alpha0.25__seed3__support_weighted__beta0.5,xiiotid,0.25,3,support_weighted,0.5,,,,support_weighted__beta0.5,done,11105.00,10986972,0, +confirm,xiiotid__alpha0.25__seed3__support_weighted__beta1,xiiotid,0.25,3,support_weighted,1,,,,support_weighted__beta1,done,11247.00,10986812,0, +confirm,xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,xiiotid,0.25,3,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,10678.00,10986572,0, +confirm,xiiotid__alpha0.25__seed3__v_cap__b1__c4,xiiotid,0.25,3,v_cap,1.0,,4,,v_cap__b1__c4,done,10621.00,10986660,0, +confirm,xiiotid__alpha0.25__seed4__original_protean,xiiotid,0.25,4,original_protean,,,,,original_protean,done,11200.00,13373900,0, +confirm,xiiotid__alpha0.25__seed4__support_weighted__beta0.5,xiiotid,0.25,4,support_weighted,0.5,,,,support_weighted__beta0.5,done,11147.00,13376792,0, +confirm,xiiotid__alpha0.25__seed4__support_weighted__beta1,xiiotid,0.25,4,support_weighted,1,,,,support_weighted__beta1,done,11529.00,13377304,0, +confirm,xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,xiiotid,0.25,4,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,11448.00,13377596,0, +confirm,xiiotid__alpha0.25__seed4__v_cap__b1__c4,xiiotid,0.25,4,v_cap,1.0,,4,,v_cap__b1__c4,done,10168.00,13377208,0, +confirm,5gnidd__alpha0.5__seed3__original_protean,5gnidd,0.5,3,original_protean,,,,,original_protean,done,15875.00,9482756,0, +confirm,5gnidd__alpha0.5__seed3__support_weighted__beta0.5,5gnidd,0.5,3,support_weighted,0.5,,,,support_weighted__beta0.5,done,16116.00,9507836,0, +confirm,5gnidd__alpha0.5__seed3__support_weighted__beta1,5gnidd,0.5,3,support_weighted,1,,,,support_weighted__beta1,done,16782.00,9480524,0, +confirm,5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,5gnidd,0.5,3,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,16459.00,9501568,0, +confirm,5gnidd__alpha0.5__seed3__v_cap__b1__c4,5gnidd,0.5,3,v_cap,1.0,,4,,v_cap__b1__c4,done,17023.00,9487356,0, +confirm,5gnidd__alpha0.5__seed4__original_protean,5gnidd,0.5,4,original_protean,,,,,original_protean,done,17324.00,11657012,0, +confirm,5gnidd__alpha0.5__seed4__support_weighted__beta0.5,5gnidd,0.5,4,support_weighted,0.5,,,,support_weighted__beta0.5,done,17181.00,11697428,0, +confirm,5gnidd__alpha0.5__seed4__support_weighted__beta1,5gnidd,0.5,4,support_weighted,1,,,,support_weighted__beta1,done,17582.00,11669892,0, +confirm,5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,5gnidd,0.5,4,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,17234.00,11688404,0, +confirm,5gnidd__alpha0.5__seed4__v_cap__b1__c4,5gnidd,0.5,4,v_cap,1.0,,4,,v_cap__b1__c4,done,16415.00,11679488,0, diff --git a/exploration_v2/analysis/screening_pairing_check.csv b/exploration_v2/analysis/screening_pairing_check.csv new file mode 100644 index 0000000..99e4a30 --- /dev/null +++ b/exploration_v2/analysis/screening_pairing_check.csv @@ -0,0 +1,15 @@ +run_id,dataset,alpha,manifest_content_hash,preprocessing_hash,support_sha256,snapshot0_sha256 +5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef +5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef +5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef +5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef +5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef +5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef +5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef +xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00 +xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00 +xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00 +xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00 +xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00 +xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00 +xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00 diff --git a/exploration_v2/analysis/screening_ranking.csv b/exploration_v2/analysis/screening_ranking.csv new file mode 100644 index 0000000..f397048 --- /dev/null +++ b/exploration_v2/analysis/screening_ranking.csv @@ -0,0 +1,7 @@ +rank,rank_secondary,label,mode,beta,shrink_lambda,cap_factor,min_support,score_mean_delta_val_global,min_condition_delta,secondary_mean_delta_val_per_client,S1_val_global,S1_baseline,S1_delta,S1_val_per_client,S1_delta_per_client,S2_val_global,S2_baseline,S2_delta,S2_val_per_client,S2_delta_per_client +1,1,v_shrink__b1__lam0.5,v_shrink,1.0,0.5,,,0.02666418343657906,0.01418010949105697,0.02904392091826946,0.6771870761605789,0.6380388187784778,0.03914825738210115,0.6394200514807982,0.0314475437388545,0.8757058802515706,0.8615257707605136,0.01418010949105697,0.7519195999208417,0.026640298097684423 +2,2,v_cap__b1__c4,v_cap,1.0,,4.0,,0.006386658832843839,0.0034615578813887993,0.014884565739555988,0.6415003766598666,0.6380388187784778,0.0034615578813887993,0.6043797897480034,-0.003592717993940231,0.8708375305448125,0.8615257707605136,0.009311759784298879,0.7586411512962095,0.03336184947305221 +3,3,v_cap__b1__c2,v_cap,1.0,,2.0,,-0.0024310752295784166,-0.0055917994582352915,-0.013098515805364142,0.6324470193202425,0.6380388187784778,-0.0055917994582352915,0.5746261281359909,-0.03334637960595277,0.8622554197595921,0.8615257707605136,0.0007296489990784583,0.7324286498183817,0.0071493479952244865 +4,5,v_logweight,v_logweight,0.0,,,,-0.013930127799255287,-0.034591638342140296,-0.030092848616604162,0.6447702015221075,0.6380388187784778,0.006731382743629721,0.6126459310493659,0.004673423307422264,0.8269341324183733,0.8615257707605136,-0.034591638342140296,0.6604201812825267,-0.06485912054063059 +5,6,v_minsup__t10,v_minsup,0.0,,,10.0,-0.03017797764203456,-0.04375397523698199,-0.05005934390012601,0.6214368387313907,0.6380388187784778,-0.016601980047087128,0.581752384882787,-0.026220122859156647,0.8177717955235316,0.8615257707605136,-0.04375397523698199,0.6513807368820619,-0.07389856494109537 +6,4,v_beta__b1.5,v_beta,1.5,,,,-0.04850986532623136,-0.09180491842510796,-0.026127080317370543,0.5462339003533698,0.6380388187784778,-0.09180491842510796,0.554598371814126,-0.0533741359278177,0.8563109585331589,0.8615257707605136,-0.005214812227354759,0.7263992771162339,0.0011199752930766138 diff --git a/exploration_v2/analysis/screening_references.csv b/exploration_v2/analysis/screening_references.csv new file mode 100644 index 0000000..c15d662 --- /dev/null +++ b/exploration_v2/analysis/screening_references.csv @@ -0,0 +1,3 @@ +condition,baseline_run,baseline_val_global,best_beta_by_val_global,best_val_global,best_delta_global,best_beta_by_val_per_client,best_val_per_client,best_delta_per_client +S1,xiiotid__alpha0.25__seed0__original_protean,0.6380388187784778,0.75,0.6690666806848056,0.03102786190632778,0.75,0.6564371328240081,0.048464625082064394 +S2,5gnidd__alpha0.5__seed0__original_protean,0.8615257707605136,1.0,0.8762664919755059,0.014740721214992258,0.75,0.7719208198370514,0.046641518013894157 diff --git a/exploration_v2/analysis/walltime.csv b/exploration_v2/analysis/walltime.csv new file mode 100644 index 0000000..d8badf3 --- /dev/null +++ b/exploration_v2/analysis/walltime.csv @@ -0,0 +1,35 @@ +phase,run_id,status,wall_s,wall_h,peak_rss_gib +screening,xiiotid__alpha0.25__seed0__original_protean,done,10879.00,3.022,8.381 +screening,5gnidd__alpha0.5__seed0__original_protean,done,16807.00,4.669,10.301 +screening,xiiotid__alpha0.25__seed0__v_beta__b1.5,done,11048.00,3.069,8.382 +screening,xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,done,10965.00,3.046,8.381 +screening,xiiotid__alpha0.25__seed0__v_cap__b1__c4,done,11022.00,3.062,8.382 +screening,xiiotid__alpha0.25__seed0__v_cap__b1__c2,done,10960.00,3.044,8.381 +screening,xiiotid__alpha0.25__seed0__v_minsup__t10,done,10904.00,3.029,8.381 +screening,xiiotid__alpha0.25__seed0__v_logweight,done,10645.00,2.957,8.38 +screening,5gnidd__alpha0.5__seed0__v_beta__b1.5,done,17571.00,4.881,10.288 +screening,5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,done,17467.00,4.852,10.291 +screening,5gnidd__alpha0.5__seed0__v_cap__b1__c4,done,16374.00,4.548,10.295 +screening,5gnidd__alpha0.5__seed0__v_cap__b1__c2,done,15875.00,4.41,10.294 +screening,5gnidd__alpha0.5__seed0__v_minsup__t10,done,16232.00,4.509,10.288 +screening,5gnidd__alpha0.5__seed0__v_logweight,done,17342.00,4.817,10.306 +confirm,xiiotid__alpha0.25__seed3__original_protean,done,11102.00,3.084,10.478 +confirm,xiiotid__alpha0.25__seed3__support_weighted__beta0.5,done,11105.00,3.085,10.478 +confirm,xiiotid__alpha0.25__seed3__support_weighted__beta1,done,11247.00,3.124,10.478 +confirm,xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,done,10678.00,2.966,10.478 +confirm,xiiotid__alpha0.25__seed3__v_cap__b1__c4,done,10621.00,2.95,10.478 +confirm,xiiotid__alpha0.25__seed4__original_protean,done,11200.00,3.111,12.754 +confirm,xiiotid__alpha0.25__seed4__support_weighted__beta0.5,done,11147.00,3.096,12.757 +confirm,xiiotid__alpha0.25__seed4__support_weighted__beta1,done,11529.00,3.203,12.758 +confirm,xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,done,11448.00,3.18,12.758 +confirm,xiiotid__alpha0.25__seed4__v_cap__b1__c4,done,10168.00,2.824,12.757 +confirm,5gnidd__alpha0.5__seed3__original_protean,done,15875.00,4.41,9.043 +confirm,5gnidd__alpha0.5__seed3__support_weighted__beta0.5,done,16116.00,4.477,9.067 +confirm,5gnidd__alpha0.5__seed3__support_weighted__beta1,done,16782.00,4.662,9.041 +confirm,5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,done,16459.00,4.572,9.061 +confirm,5gnidd__alpha0.5__seed3__v_cap__b1__c4,done,17023.00,4.729,9.048 +confirm,5gnidd__alpha0.5__seed4__original_protean,done,17324.00,4.812,11.117 +confirm,5gnidd__alpha0.5__seed4__support_weighted__beta0.5,done,17181.00,4.772,11.156 +confirm,5gnidd__alpha0.5__seed4__support_weighted__beta1,done,17582.00,4.884,11.129 +confirm,5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,done,17234.00,4.787,11.147 +confirm,5gnidd__alpha0.5__seed4__v_cap__b1__c4,done,16415.00,4.56,11.138 diff --git a/exploration_v2/driver.sh b/exploration_v2/driver.sh new file mode 100755 index 0000000..f86a2e0 --- /dev/null +++ b/exploration_v2/driver.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# exploration_v2 driver entry point. +# +# Holds an exclusive lock so two drivers can never run at once. The actual +# scheduler and heartbeat live in scripts/pool_driver.py. This script is safe +# to re-invoke; resume.sh does exactly that. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/.." && pwd)" +PY="$REPO_ROOT/.venv/bin/python" + +export OMP_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export OPENBLAS_NUM_THREADS=1 +export NUMEXPR_NUM_THREADS=1 +export VECLIB_MAXIMUM_THREADS=1 +export PYTHONDONTWRITEBYTECODE=1 +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=/run/user/$(id -u)/bus}" + +mkdir -p "$HERE/logs" +exec 9>"$HERE/driver.lock" +if ! flock -n 9; then + echo "[exploration_v2] another driver already holds driver.lock; exiting" >&2 + exit 0 +fi + +cd "$REPO_ROOT" || exit 3 +exec "$PY" "$HERE/scripts/pool_driver.py" "$@" diff --git a/exploration_v2/partitions/5gnidd__alpha0.5__seed3.json b/exploration_v2/partitions/5gnidd__alpha0.5__seed3.json new file mode 100644 index 0000000..9612ae2 --- /dev/null +++ b/exploration_v2/partitions/5gnidd__alpha0.5__seed3.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-10-30T04:32:10Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "0739f15c51fe4fa2250f717e8f52548706cb7093581447cd43a4b2364f6ec058", + "preprocessing_hash": "60b5682b39184fd7d24e117430a00c160c5c7e95d1bc1fcaa99745108f514d81", + "python": "3.12.3", + "rng_seed": 2052809991, + "seed": 3, + "split": { + "hash": "178e69cfacc40a1651246d0c33a26d16f76c8e19254d033c8c04cbbfbb86246a", + "seed": 3, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 12367, + 24330, + 109, + 732, + 1299, + 2036, + 261, + 107583, + 129 + ], + [ + 86013, + 9922, + 8, + 104, + 31, + 3166, + 1928, + 6739, + 5 + ], + [ + 112226, + 6136, + 97, + 152, + 143, + 28, + 966, + 13880, + 301 + ], + [ + 7322, + 29013, + 115, + 213, + 9592, + 159, + 373, + 2354, + 503 + ], + [ + 28752, + 497, + 15, + 1688, + 71, + 19630, + 0, + 607, + 67 + ], + [ + 11126, + 366, + 37, + 786, + 519, + 1272, + 140, + 84223, + 1613 + ], + [ + 7958, + 249, + 40, + 912, + 93, + 801, + 3851, + 45015, + 8048 + ], + [ + 64584, + 25, + 1, + 77, + 2281, + 20803, + 7, + 12615, + 17 + ], + [ + 2496, + 14929, + 303, + 1582, + 1, + 2192, + 2455, + 189, + 234 + ], + [ + 1571, + 13102, + 83, + 559, + 0, + 1100, + 4056, + 46933, + 217 + ] + ] +} diff --git a/exploration_v2/partitions/5gnidd__alpha0.5__seed4.json b/exploration_v2/partitions/5gnidd__alpha0.5__seed4.json new file mode 100644 index 0000000..1d617b3 --- /dev/null +++ b/exploration_v2/partitions/5gnidd__alpha0.5__seed4.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-10-30T04:32:41Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "21a507d9d94f31154c5fb0c6833344738d1a137c78950f5284dec26d727e983a", + "preprocessing_hash": "ffb7b2a1adb6cf8a292ea6e88dca574b44446da2b3ae18f690ae6fc59699e23d", + "python": "3.12.3", + "rng_seed": 1792011051, + "seed": 4, + "split": { + "hash": "0b80242a3ac045217b73a89e5a7d03cb61abacefcc050e11b93ac83c831479ce", + "seed": 4, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 159078, + 11071, + 39, + 936, + 8239, + 1050, + 33, + 15599, + 241 + ], + [ + 15658, + 27988, + 318, + 7, + 674, + 7, + 5229, + 6152, + 93 + ], + [ + 5874, + 2211, + 0, + 2113, + 893, + 1049, + 490, + 13122, + 131 + ], + [ + 76962, + 851, + 10, + 26, + 1223, + 7546, + 846, + 57683, + 3055 + ], + [ + 3588, + 47186, + 243, + 26, + 930, + 11733, + 1121, + 22035, + 1898 + ], + [ + 26897, + 235, + 60, + 2927, + 181, + 263, + 2914, + 58429, + 2934 + ], + [ + 143, + 5129, + 112, + 237, + 45, + 5427, + 36, + 7654, + 1444 + ], + [ + 30083, + 3124, + 10, + 8, + 79, + 357, + 1910, + 111407, + 0 + ], + [ + 405, + 4, + 5, + 413, + 1110, + 1483, + 1, + 14987, + 1323 + ], + [ + 15727, + 770, + 11, + 112, + 656, + 22272, + 1457, + 13070, + 15 + ] + ] +} diff --git a/exploration_v2/partitions/xiiotid__alpha0.25__seed3.json b/exploration_v2/partitions/xiiotid__alpha0.25__seed3.json new file mode 100644 index 0000000..ffa7b95 --- /dev/null +++ b/exploration_v2/partitions/xiiotid__alpha0.25__seed3.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-10-30T04:00:15Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "85a8625b81d3624dec9e5322979e2f1547f330c60d5e822e32c12177195ed602", + "preprocessing_hash": "e22db95e5109bfa02701d8ce450453a7c8f7e408c94e81dfe2b95ef4ab2a441f", + "python": "3.12.3", + "rng_seed": 1031130567, + "seed": 3, + "split": { + "hash": "8e66d08e71225567f8da977e5b927c476882c1d1ac768c1b490cdbc8de2945e9", + "seed": 3, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 21, + 1003, + 0, + 7, + 2950, + 14, + 11735, + 0, + 342 + ], + [ + 27, + 1029, + 7, + 199, + 549, + 1426, + 8567, + 370, + 1278 + ], + [ + 1191, + 483, + 23, + 5147, + 17775, + 234, + 139, + 0, + 31 + ], + [ + 0, + 2, + 36, + 0, + 3006, + 58, + 12, + 12, + 1420 + ], + [ + 15, + 18, + 39, + 2326, + 2692, + 58544, + 773, + 0, + 25866 + ], + [ + 320, + 4958, + 0, + 31, + 966, + 18218, + 3006, + 949, + 8009 + ], + [ + 0, + 2705, + 215, + 167, + 1157, + 1247, + 13942, + 2, + 347 + ], + [ + 165, + 35, + 473, + 13685, + 111159, + 19011, + 15662, + 2013, + 120 + ], + [ + 4, + 228, + 0, + 2, + 150706, + 122, + 35323, + 2, + 83 + ], + [ + 261, + 5033, + 0, + 553, + 1817, + 9, + 2, + 238, + 7598 + ] + ] +} diff --git a/exploration_v2/partitions/xiiotid__alpha0.25__seed4.json b/exploration_v2/partitions/xiiotid__alpha0.25__seed4.json new file mode 100644 index 0000000..88385de --- /dev/null +++ b/exploration_v2/partitions/xiiotid__alpha0.25__seed4.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-10-30T04:31:39Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "ff47f504d5b604935333ccbfddd060c18d44d672104e5455f86dcefe48315887", + "preprocessing_hash": "4837f4ab5939fe7bf7f83168d915e67d570086d8534a521a9a9827f10aa41086", + "python": "3.12.3", + "rng_seed": 1148101028, + "seed": 4, + "split": { + "hash": "5bc9100e5cbb4dfccf92e1b0fe64da544a2966c81d82dfab2efa2367173ca807", + "seed": 4, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 1, + 109, + 667, + 156, + 193448, + 29952, + 1314, + 3, + 9802 + ], + [ + 20, + 368, + 4, + 1852, + 13384, + 262, + 1102, + 42, + 4 + ], + [ + 58, + 336, + 0, + 1331, + 0, + 218, + 32287, + 5, + 27 + ], + [ + 0, + 7839, + 10, + 11, + 22325, + 619, + 12186, + 2959, + 7624 + ], + [ + 413, + 94, + 4, + 9367, + 13997, + 0, + 1924, + 6, + 47 + ], + [ + 238, + 5, + 3, + 0, + 1, + 24028, + 22974, + 62, + 5 + ], + [ + 4, + 1, + 50, + 121, + 167, + 36163, + 14248, + 75, + 18287 + ], + [ + 1110, + 508, + 47, + 6770, + 61, + 2656, + 25, + 88, + 147 + ], + [ + 78, + 328, + 0, + 1112, + 42050, + 24, + 132, + 46, + 10 + ], + [ + 82, + 5906, + 8, + 1397, + 7344, + 4961, + 2969, + 300, + 9141 + ] + ] +} diff --git a/exploration_v2/queue_confirm.csv b/exploration_v2/queue_confirm.csv new file mode 100644 index 0000000..d9b2d90 --- /dev/null +++ b/exploration_v2/queue_confirm.csv @@ -0,0 +1,21 @@ +run_id,dataset,alpha,seed,mode,beta,shrink,cap,minsup,part_dir,label,role +xiiotid__alpha0.25__seed3__original_protean,xiiotid,0.25,3,original_protean,,,,,exploration_v2/partitions,original_protean,reference +xiiotid__alpha0.25__seed3__support_weighted__beta0.5,xiiotid,0.25,3,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference +xiiotid__alpha0.25__seed3__support_weighted__beta1,xiiotid,0.25,3,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference +xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,xiiotid,0.25,3,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant +xiiotid__alpha0.25__seed3__v_cap__b1__c4,xiiotid,0.25,3,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant +xiiotid__alpha0.25__seed4__original_protean,xiiotid,0.25,4,original_protean,,,,,exploration_v2/partitions,original_protean,reference +xiiotid__alpha0.25__seed4__support_weighted__beta0.5,xiiotid,0.25,4,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference +xiiotid__alpha0.25__seed4__support_weighted__beta1,xiiotid,0.25,4,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference +xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,xiiotid,0.25,4,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant +xiiotid__alpha0.25__seed4__v_cap__b1__c4,xiiotid,0.25,4,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant +5gnidd__alpha0.5__seed3__original_protean,5gnidd,0.5,3,original_protean,,,,,exploration_v2/partitions,original_protean,reference +5gnidd__alpha0.5__seed3__support_weighted__beta0.5,5gnidd,0.5,3,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference +5gnidd__alpha0.5__seed3__support_weighted__beta1,5gnidd,0.5,3,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference +5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,5gnidd,0.5,3,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant +5gnidd__alpha0.5__seed3__v_cap__b1__c4,5gnidd,0.5,3,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant +5gnidd__alpha0.5__seed4__original_protean,5gnidd,0.5,4,original_protean,,,,,exploration_v2/partitions,original_protean,reference +5gnidd__alpha0.5__seed4__support_weighted__beta0.5,5gnidd,0.5,4,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference +5gnidd__alpha0.5__seed4__support_weighted__beta1,5gnidd,0.5,4,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference +5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,5gnidd,0.5,4,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant +5gnidd__alpha0.5__seed4__v_cap__b1__c4,5gnidd,0.5,4,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant diff --git a/exploration_v2/queue_screen.csv b/exploration_v2/queue_screen.csv new file mode 100644 index 0000000..d375a80 --- /dev/null +++ b/exploration_v2/queue_screen.csv @@ -0,0 +1,15 @@ +run_id,dataset,alpha,seed,mode,beta,shrink,cap,minsup,part_dir,label +xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,0,original_protean,,,,,artifacts/partitions,original_protean +5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0,original_protean,,,,,artifacts/partitions,original_protean +xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,0,v_beta,1.5,,,,artifacts/partitions,v_beta__b1.5 +xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,0,v_shrink,1.0,0.5,,,artifacts/partitions,v_shrink__b1__lam0.5 +xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,0,v_cap,1.0,,4,,artifacts/partitions,v_cap__b1__c4 +xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,0,v_cap,1.0,,2,,artifacts/partitions,v_cap__b1__c2 +xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,0,v_minsup,0.0,,,10,artifacts/partitions,v_minsup__t10 +xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,0,v_logweight,0.0,,,,artifacts/partitions,v_logweight +5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,0,v_beta,1.5,,,,artifacts/partitions,v_beta__b1.5 +5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,0,v_shrink,1.0,0.5,,,artifacts/partitions,v_shrink__b1__lam0.5 +5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,0,v_cap,1.0,,4,,artifacts/partitions,v_cap__b1__c4 +5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,0,v_cap,1.0,,2,,artifacts/partitions,v_cap__b1__c2 +5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0,v_minsup,0.0,,,10,artifacts/partitions,v_minsup__t10 +5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0,v_logweight,0.0,,,,artifacts/partitions,v_logweight diff --git a/exploration_v2/ranking.json b/exploration_v2/ranking.json new file mode 100644 index 0000000..866703d --- /dev/null +++ b/exploration_v2/ranking.json @@ -0,0 +1,513 @@ +{ + "generated_utc": "2026-10-31T05:03:55Z", + "stage": "screening", + "ranking_metric": "mean over S1,S2 of val_global_rare_f1(candidate) - val_global_rare_f1(original)", + "secondary_metric": "same with val_rare_class_macro_f1_per_client_mean", + "conditions": { + "S1": { + "dataset": "xiiotid", + "alpha": 0.25 + }, + "S2": { + "dataset": "5gnidd", + "alpha": 0.5 + } + }, + "screening_seed": 0, + "baseline_source": { + "S1": "in_environment_seed0", + "S2": "in_environment_seed0" + }, + "original": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__original_protean", + "source": "in_environment_seed0", + "val_global_rare_f1": 0.6380388187784778, + "val_global_rare_recall": 0.840600287361644, + "val_global_macro_f1": 0.6716435746566032, + "val_per_client_rare_f1": 0.6079725077419437, + "val_sel_file_present": true + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__original_protean", + "source": "in_environment_seed0", + "val_global_rare_f1": 0.8615257707605136, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8292070866942571, + "val_per_client_rare_f1": 0.7252793018231573, + "val_sel_file_present": true + } + }, + "current_method_reference": { + "S1": { + "original": { + "run_id": "xiiotid__alpha0.25__seed0__original_protean", + "source": "in_environment_seed0", + "val_global_rare_f1": 0.6380388187784778, + "val_global_rare_recall": 0.840600287361644, + "val_global_macro_f1": 0.6716435746566032, + "val_per_client_rare_f1": 0.6079725077419437, + "val_sel_file_present": true + }, + "best_beta_by_val_global": { + "beta": 0.75, + "val_global_rare_f1": 0.6690666806848056, + "delta_vs_original": 0.03102786190632778 + }, + "best_beta_by_val_per_client": { + "beta": 0.75, + "val_per_client_rare_f1": 0.6564371328240081, + "delta_vs_original": 0.048464625082064394 + }, + "all_betas": { + "0": { + "run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.643275274917486, + "val_global_rare_recall": 0.8414156040624469, + "val_global_macro_f1": 0.6762678752194305, + "val_per_client_rare_f1": 0.6151143983374613, + "val_sel_file_present": true + }, + "0.25": { + "run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0.25", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.6474224331223045, + "val_global_rare_recall": 0.8369281236929922, + "val_global_macro_f1": 0.6800358336787912, + "val_per_client_rare_f1": 0.612295413922644, + "val_sel_file_present": true + }, + "0.5": { + "run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0.5", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.6328331530314106, + "val_global_rare_recall": 0.8055422411499216, + "val_global_macro_f1": 0.6670453405769549, + "val_per_client_rare_f1": 0.5909962766172367, + "val_sel_file_present": true + }, + "0.75": { + "run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0.75", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.6690666806848056, + "val_global_rare_recall": 0.8048039361797941, + "val_global_macro_f1": 0.699686253411971, + "val_per_client_rare_f1": 0.6564371328240081, + "val_sel_file_present": true + }, + "1": { + "run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta1", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.5937608319568202, + "val_global_rare_recall": 0.7156752311559107, + "val_global_macro_f1": 0.6331165546843429, + "val_per_client_rare_f1": 0.5797652799748645, + "val_sel_file_present": true + } + } + }, + "S2": { + "original": { + "run_id": "5gnidd__alpha0.5__seed0__original_protean", + "source": "in_environment_seed0", + "val_global_rare_f1": 0.8615257707605136, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8292070866942571, + "val_per_client_rare_f1": 0.7252793018231573, + "val_sel_file_present": true + }, + "best_beta_by_val_global": { + "beta": 1.0, + "val_global_rare_f1": 0.8762664919755059, + "delta_vs_original": 0.014740721214992258 + }, + "best_beta_by_val_per_client": { + "beta": 0.75, + "val_per_client_rare_f1": 0.7719208198370514, + "delta_vs_original": 0.046641518013894157 + }, + "all_betas": { + "0": { + "run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.8676517373672202, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8354514554793465, + "val_per_client_rare_f1": 0.7394716750551001, + "val_sel_file_present": true + }, + "0.25": { + "run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0.25", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.8262070288650597, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.799036115730399, + "val_per_client_rare_f1": 0.6579489943732403, + "val_sel_file_present": true + }, + "0.5": { + "run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0.5", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.8627083270087127, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8363882297413696, + "val_per_client_rare_f1": 0.7316622118992844, + "val_sel_file_present": true + }, + "0.75": { + "run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0.75", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.8751527301478877, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8498739964533271, + "val_per_client_rare_f1": 0.7719208198370514, + "val_sel_file_present": true + }, + "1": { + "run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta1", + "source": "sweep_seed0_1thread", + "val_global_rare_f1": 0.8762664919755059, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8461375258117143, + "val_per_client_rare_f1": 0.7666551346534329, + "val_sel_file_present": true + } + } + } + }, + "candidates": [ + { + "label": "v_shrink__b1__lam0.5", + "mode": "v_shrink", + "params": { + "beta": 1.0, + "shrink_lambda": 0.5, + "cap_factor": null, + "min_support": null + }, + "conditions": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5", + "val_global_rare_f1": 0.6771870761605789, + "val_global_rare_recall": 0.8372489152448566, + "val_global_macro_f1": 0.7085312157748022, + "val_per_client_rare_f1": 0.6394200514807982, + "val_sel_file_present": true + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5", + "val_global_rare_f1": 0.8757058802515706, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.839727653421119, + "val_per_client_rare_f1": 0.7519195999208417, + "val_sel_file_present": true + } + }, + "per_condition": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5", + "val_global_rare_f1": 0.6771870761605789, + "baseline_val_global_rare_f1": 0.6380388187784778, + "delta_vs_original": 0.03914825738210115, + "val_per_client_rare_f1": 0.6394200514807982, + "baseline_val_per_client_rare_f1": 0.6079725077419437, + "delta_per_client_vs_original": 0.0314475437388545 + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5", + "val_global_rare_f1": 0.8757058802515706, + "baseline_val_global_rare_f1": 0.8615257707605136, + "delta_vs_original": 0.01418010949105697, + "val_per_client_rare_f1": 0.7519195999208417, + "baseline_val_per_client_rare_f1": 0.7252793018231573, + "delta_per_client_vs_original": 0.026640298097684423 + } + }, + "score": 0.02666418343657906, + "min_condition_delta": 0.01418010949105697, + "secondary_score_per_client": 0.02904392091826946, + "rank": 1, + "rank_secondary": 1 + }, + { + "label": "v_cap__b1__c4", + "mode": "v_cap", + "params": { + "beta": 1.0, + "shrink_lambda": null, + "cap_factor": 4.0, + "min_support": null + }, + "conditions": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c4", + "val_global_rare_f1": 0.6415003766598666, + "val_global_rare_recall": 0.7994428549956025, + "val_global_macro_f1": 0.6751242688482385, + "val_per_client_rare_f1": 0.6043797897480034, + "val_sel_file_present": true + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c4", + "val_global_rare_f1": 0.8708375305448125, + "val_global_rare_recall": 0.9719002398444734, + "val_global_macro_f1": 0.8324391611800375, + "val_per_client_rare_f1": 0.7586411512962095, + "val_sel_file_present": true + } + }, + "per_condition": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c4", + "val_global_rare_f1": 0.6415003766598666, + "baseline_val_global_rare_f1": 0.6380388187784778, + "delta_vs_original": 0.0034615578813887993, + "val_per_client_rare_f1": 0.6043797897480034, + "baseline_val_per_client_rare_f1": 0.6079725077419437, + "delta_per_client_vs_original": -0.003592717993940231 + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c4", + "val_global_rare_f1": 0.8708375305448125, + "baseline_val_global_rare_f1": 0.8615257707605136, + "delta_vs_original": 0.009311759784298879, + "val_per_client_rare_f1": 0.7586411512962095, + "baseline_val_per_client_rare_f1": 0.7252793018231573, + "delta_per_client_vs_original": 0.03336184947305221 + } + }, + "score": 0.006386658832843839, + "min_condition_delta": 0.0034615578813887993, + "secondary_score_per_client": 0.014884565739555988, + "rank": 2, + "rank_secondary": 2 + }, + { + "label": "v_cap__b1__c2", + "mode": "v_cap", + "params": { + "beta": 1.0, + "shrink_lambda": null, + "cap_factor": 2.0, + "min_support": null + }, + "conditions": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c2", + "val_global_rare_f1": 0.6324470193202425, + "val_global_rare_recall": 0.8098457414207068, + "val_global_macro_f1": 0.6670829111135634, + "val_per_client_rare_f1": 0.5746261281359909, + "val_sel_file_present": true + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c2", + "val_global_rare_f1": 0.8622554197595921, + "val_global_rare_recall": 0.9719002398444734, + "val_global_macro_f1": 0.8250473485396072, + "val_per_client_rare_f1": 0.7324286498183817, + "val_sel_file_present": true + } + }, + "per_condition": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c2", + "val_global_rare_f1": 0.6324470193202425, + "baseline_val_global_rare_f1": 0.6380388187784778, + "delta_vs_original": -0.0055917994582352915, + "val_per_client_rare_f1": 0.5746261281359909, + "baseline_val_per_client_rare_f1": 0.6079725077419437, + "delta_per_client_vs_original": -0.03334637960595277 + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c2", + "val_global_rare_f1": 0.8622554197595921, + "baseline_val_global_rare_f1": 0.8615257707605136, + "delta_vs_original": 0.0007296489990784583, + "val_per_client_rare_f1": 0.7324286498183817, + "baseline_val_per_client_rare_f1": 0.7252793018231573, + "delta_per_client_vs_original": 0.0071493479952244865 + } + }, + "score": -0.0024310752295784166, + "min_condition_delta": -0.0055917994582352915, + "secondary_score_per_client": -0.013098515805364142, + "rank": 3, + "rank_secondary": 3 + }, + { + "label": "v_logweight", + "mode": "v_logweight", + "params": { + "beta": 0.0, + "shrink_lambda": null, + "cap_factor": null, + "min_support": null + }, + "conditions": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_logweight", + "val_global_rare_f1": 0.6447702015221075, + "val_global_rare_recall": 0.8413731264629913, + "val_global_macro_f1": 0.6777965963838619, + "val_per_client_rare_f1": 0.6126459310493659, + "val_sel_file_present": true + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_logweight", + "val_global_rare_f1": 0.8269341324183733, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8025209368110234, + "val_per_client_rare_f1": 0.6604201812825267, + "val_sel_file_present": true + } + }, + "per_condition": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_logweight", + "val_global_rare_f1": 0.6447702015221075, + "baseline_val_global_rare_f1": 0.6380388187784778, + "delta_vs_original": 0.006731382743629721, + "val_per_client_rare_f1": 0.6126459310493659, + "baseline_val_per_client_rare_f1": 0.6079725077419437, + "delta_per_client_vs_original": 0.004673423307422264 + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_logweight", + "val_global_rare_f1": 0.8269341324183733, + "baseline_val_global_rare_f1": 0.8615257707605136, + "delta_vs_original": -0.034591638342140296, + "val_per_client_rare_f1": 0.6604201812825267, + "baseline_val_per_client_rare_f1": 0.7252793018231573, + "delta_per_client_vs_original": -0.06485912054063059 + } + }, + "score": -0.013930127799255287, + "min_condition_delta": -0.034591638342140296, + "secondary_score_per_client": -0.030092848616604162, + "rank": 4, + "rank_secondary": 5 + }, + { + "label": "v_minsup__t10", + "mode": "v_minsup", + "params": { + "beta": 0.0, + "shrink_lambda": null, + "cap_factor": null, + "min_support": 10.0 + }, + "conditions": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_minsup__t10", + "val_global_rare_f1": 0.6214368387313907, + "val_global_rare_recall": 0.8322773169160314, + "val_global_macro_f1": 0.6563272217955932, + "val_per_client_rare_f1": 0.581752384882787, + "val_sel_file_present": true + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_minsup__t10", + "val_global_rare_f1": 0.8177717955235316, + "val_global_rare_recall": 0.9716507803262696, + "val_global_macro_f1": 0.8013311345491211, + "val_per_client_rare_f1": 0.6513807368820619, + "val_sel_file_present": true + } + }, + "per_condition": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_minsup__t10", + "val_global_rare_f1": 0.6214368387313907, + "baseline_val_global_rare_f1": 0.6380388187784778, + "delta_vs_original": -0.016601980047087128, + "val_per_client_rare_f1": 0.581752384882787, + "baseline_val_per_client_rare_f1": 0.6079725077419437, + "delta_per_client_vs_original": -0.026220122859156647 + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_minsup__t10", + "val_global_rare_f1": 0.8177717955235316, + "baseline_val_global_rare_f1": 0.8615257707605136, + "delta_vs_original": -0.04375397523698199, + "val_per_client_rare_f1": 0.6513807368820619, + "baseline_val_per_client_rare_f1": 0.7252793018231573, + "delta_per_client_vs_original": -0.07389856494109537 + } + }, + "score": -0.03017797764203456, + "min_condition_delta": -0.04375397523698199, + "secondary_score_per_client": -0.05005934390012601, + "rank": 5, + "rank_secondary": 6 + }, + { + "label": "v_beta__b1.5", + "mode": "v_beta", + "params": { + "beta": 1.5, + "shrink_lambda": null, + "cap_factor": null, + "min_support": null + }, + "conditions": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_beta__b1.5", + "val_global_rare_f1": 0.5462339003533698, + "val_global_rare_recall": 0.6920487487732283, + "val_global_macro_f1": 0.5912778236542747, + "val_per_client_rare_f1": 0.554598371814126, + "val_sel_file_present": true + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_beta__b1.5", + "val_global_rare_f1": 0.8563109585331589, + "val_global_rare_recall": 0.971733905845804, + "val_global_macro_f1": 0.8386791194607972, + "val_per_client_rare_f1": 0.7263992771162339, + "val_sel_file_present": true + } + }, + "per_condition": { + "S1": { + "run_id": "xiiotid__alpha0.25__seed0__v_beta__b1.5", + "val_global_rare_f1": 0.5462339003533698, + "baseline_val_global_rare_f1": 0.6380388187784778, + "delta_vs_original": -0.09180491842510796, + "val_per_client_rare_f1": 0.554598371814126, + "baseline_val_per_client_rare_f1": 0.6079725077419437, + "delta_per_client_vs_original": -0.0533741359278177 + }, + "S2": { + "run_id": "5gnidd__alpha0.5__seed0__v_beta__b1.5", + "val_global_rare_f1": 0.8563109585331589, + "baseline_val_global_rare_f1": 0.8615257707605136, + "delta_vs_original": -0.005214812227354759, + "val_per_client_rare_f1": 0.7263992771162339, + "baseline_val_per_client_rare_f1": 0.7252793018231573, + "delta_per_client_vs_original": 0.0011199752930766138 + } + }, + "score": -0.04850986532623136, + "min_condition_delta": -0.09180491842510796, + "secondary_score_per_client": -0.026127080317370543, + "rank": 6, + "rank_secondary": 4 + } + ], + "secondary_ranking_order": [ + "v_shrink__b1__lam0.5", + "v_cap__b1__c4", + "v_cap__b1__c2", + "v_beta__b1.5", + "v_logweight", + "v_minsup__t10" + ], + "top2": [ + "v_shrink__b1__lam0.5", + "v_cap__b1__c4" + ], + "selection_margin_top1_minus_top2": 0.02027752460373522, + "note": "Validation only. No test metric of any new run was read before this file was frozen." +} diff --git a/exploration_v2/ranking.sha256 b/exploration_v2/ranking.sha256 new file mode 100644 index 0000000..ca125ca --- /dev/null +++ b/exploration_v2/ranking.sha256 @@ -0,0 +1,2 @@ +f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3 ranking.json +2026-10-31T05:03:55Z diff --git a/exploration_v2/resume.sh b/exploration_v2/resume.sh new file mode 100755 index 0000000..b0e80f4 --- /dev/null +++ b/exploration_v2/resume.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Resume the exploration_v2 driver in its dedicated tmux session. +# +# Safe to call repeatedly. Never kills a run that is already active; the +# scheduler re-attaches to live systemd scopes by name. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/.." && pwd)" +SESSION="explore_v2" + +if [ -f "$HERE/DONE" ]; then + echo "[resume] DONE already exists; nothing to resume." + cat "$HERE/DONE" + exit 0 +fi + +# Is a driver already holding the lock? +if ! flock -n "$HERE/driver.lock" true 2>/dev/null; then + echo "[resume] driver.lock is held: a driver is already running." + exit 0 +fi + +# Also check for a live driver process (belt and braces). +if pgrep -f '[p]ool_driver.py' >/dev/null 2>&1; then + echo "[resume] pool_driver.py is already alive." + exit 0 +fi + +if tmux has-session -t "$SESSION" 2>/dev/null; then + echo "[resume] respawning pane in existing tmux session $SESSION" + tmux respawn-pane -k -t "$SESSION" "bash '$HERE/driver.sh'" +else + echo "[resume] creating tmux session $SESSION" + tmux new-session -d -s "$SESSION" -c "$REPO_ROOT" "bash '$HERE/driver.sh'" +fi +tmux list-sessions 2>/dev/null | grep "^$SESSION" || true diff --git a/exploration_v2/run_one_inner_v2.sh b/exploration_v2/run_one_inner_v2.sh new file mode 100755 index 0000000..b355ea5 --- /dev/null +++ b/exploration_v2/run_one_inner_v2.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Inner command executed inside the systemd MemoryMax scope by run_one_v2.sh. +# Usage: run_one_inner_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL +set -uo pipefail + +if [ "$#" -ne 11 ]; then + echo "usage: run_one_inner_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL" >&2 + exit 2 +fi + +RUN_ID="$1"; DATASET="$2"; ALPHA="$3"; SEED="$4"; MODE="$5"; BETA="$6" +SHRINK="$7"; CAP="$8"; MINSUP="$9"; PART_DIR="${10}"; LABEL="${11}" + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/.." && pwd)" +RUN_DIR="$HERE/results/runs/$RUN_ID" + +cd "$REPO_ROOT" || exit 3 + +EXTRA=(--variant-label "$LABEL") +case "$MODE" in + original_protean) ;; + support_weighted) EXTRA+=(--beta "$BETA") ;; + v_beta) EXTRA+=(--beta "$BETA") ;; + v_shrink) EXTRA+=(--beta "$BETA" --shrink-lambda "$SHRINK") ;; + v_cap) EXTRA+=(--beta "$BETA" --cap-factor "$CAP") ;; + v_minsup) EXTRA+=(--beta "$BETA" --min-support "$MINSUP") ;; + v_logweight) EXTRA+=(--beta "$BETA") ;; + *) echo "unknown mode $MODE" >&2; exit 2 ;; +esac + +exec "$REPO_ROOT/.venv/bin/python" "$HERE/scripts/thread1_launcher.py" \ + "$HERE/scripts/run_variant.py" \ + --dataset "$DATASET" \ + --alpha "$ALPHA" \ + --seed "$SEED" \ + --clients 10 \ + --rounds 10 \ + --local-epochs 3 \ + --batch-size 256 \ + --lr 0.001 \ + --mu 0.1 \ + --lam 1.0 \ + --aggregation-mode "$MODE" \ + "${EXTRA[@]}" \ + --partitions-dir "$PART_DIR" \ + --results-dir "$RUN_DIR/results" \ + --analysis-dir "$RUN_DIR/analysis" \ + --processed-dir "$RUN_DIR/processed" diff --git a/exploration_v2/run_one_v2.sh b/exploration_v2/run_one_v2.sh new file mode 100755 index 0000000..bde09bc --- /dev/null +++ b/exploration_v2/run_one_v2.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# One resumable exploration_v2 run. +# +# run_one_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL +# +# Mirrors results/sweep/run_one.sh: +# * one process, one thread (env pinned here and in thread1_launcher.py) +# * one systemd user scope with MemoryMax=11G (xiiotid) / 15G (5gnidd) +# and MemorySwapMax=0 (the sweep's per-dataset caps) +# * logs under /logs/, COMPLETE only after exit 0 and every primary +# artifact exists and is non-empty +# * summary.tsv upsert through run_record_v2.py +# * .failed marker written on any non-zero exit so the driver can retry +set -uo pipefail + +if [ "$#" -ne 11 ]; then + echo "usage: run_one_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL" >&2 + exit 2 +fi + +RUN_ID="$1"; DATASET="$2"; ALPHA="$3"; SEED="$4"; MODE="$5"; BETA="$6" +SHRINK="$7"; CAP="$8"; MINSUP="$9"; PART_DIR="${10}"; LABEL="${11}" + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/.." && pwd)" +PY="$REPO_ROOT/.venv/bin/python" + +RUN_DIR="$HERE/results/runs/$RUN_ID" +RESULTS_DIR="$RUN_DIR/results" +ANALYSIS_DIR="$RUN_DIR/analysis" +PROCESSED_DIR="$RUN_DIR/processed" +LOG_DIR="$RUN_DIR/logs" + +SUMMARY="$HERE/summary.csv" +LOCK="$HERE/summary.lock" +STOP_FILE="$HERE/STOP" + +# Per-dataset systemd cgroup caps. 5G-NIDD keeps the sweep's 15G. X-IIoTID is +# raised from the sweep's 11G to 13G after three cgroup OOM kills at 11G, while +# two concurrent X-IIoTID processes still stay at 26G <= the 36G budget. +case "$DATASET" in + 5gnidd) MEMORY_MAX="15G" ;; + *) MEMORY_MAX="13G" ;; +esac +RUN_TIMEOUT_S="${EXPLORE_RUN_TIMEOUT_S:-43200}" + +mkdir -p "$RESULTS_DIR" "$ANALYSIS_DIR" "$PROCESSED_DIR" "$LOG_DIR" + +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=/run/user/$(id -u)/bus}" +export PYTHONDONTWRITEBYTECODE=1 + +utc() { date -u +%Y-%m-%dT%H:%M:%SZ; } +log() { echo "[$(utc)] $*" >> "$LOG_DIR/run.log"; } + +summary_row() { + local status="$1" rc="$2" notes="${3:-}" + local time_file="" + [ -s "$LOG_DIR/time.txt" ] && time_file="$LOG_DIR/time.txt" + "$PY" "$HERE/scripts/run_record_v2.py" record \ + --summary "$SUMMARY" --lock "$LOCK" \ + --run-name "$RUN_ID" --dataset "$DATASET" --alpha "$ALPHA" \ + --beta "$BETA" --seed "$SEED" --aggregation-mode "$MODE" \ + --status "$status" --exit-code "$rc" \ + --time-file "$time_file" --started "${STARTED:-}" --ended "$(utc)" \ + --run-dir "$RUN_DIR" --log-dir "$LOG_DIR" --notes "$notes" \ + || log "WARNING: could not update summary.csv" +} + +primary_ok() { + local p + for p in \ + "$RESULTS_DIR/$RUN_ID/metadata.json" \ + "$RESULTS_DIR/$RUN_ID/history.json" \ + "$RESULTS_DIR/$RUN_ID/test_global_metrics.json" \ + "$RESULTS_DIR/$RUN_ID/test_confusion_matrix.json" \ + "$RESULTS_DIR/$RUN_ID/test_per_class.csv" \ + "$RESULTS_DIR/$RUN_ID/val_global_metrics.json" \ + "$RESULTS_DIR/$RUN_ID/val_selection.json" \ + "$RESULTS_DIR/$RUN_ID/per_client.csv" \ + "$RESULTS_DIR/$RUN_ID/val_per_client.csv" \ + "$RESULTS_DIR/$RUN_ID/train_support_Nij.csv" \ + "$RESULTS_DIR/$RUN_ID/characterization.csv" \ + "$RESULTS_DIR/$RUN_ID/rare_unseen.csv" \ + "$ANALYSIS_DIR/${RUN_ID}__prototype_snapshot.json" ; do + [ -s "$p" ] || return 1 + done + return 0 +} + +# ---- resumability ------------------------------------------------------- +if [ -f "$RUN_DIR/COMPLETE" ] && primary_ok; then + log "SKIP: COMPLETE marker and all primary artifacts present" + summary_row "skipped_complete" 0 "resumed: already complete" + exit 0 +fi +if [ -f "$RUN_DIR/COMPLETE" ]; then + mv "$RUN_DIR/COMPLETE" "$RUN_DIR/COMPLETE.invalid.$(date -u +%Y%m%dT%H%M%SZ)" + log "WARNING: moved invalid COMPLETE marker aside" +fi + +if [ -e "$STOP_FILE" ]; then + log "STOP present: refusing to start $RUN_ID" + summary_row "stopped" 0 "not started because exploration_v2/STOP exists" + exit 0 +fi + +# ---- scope unit --------------------------------------------------------- +UNIT="explorev2-$(printf '%s' "$RUN_ID" | sha1sum | cut -c1-12)" +if systemctl --user is-active --quiet "$UNIT.scope" 2>/dev/null; then + log "REFUSE: scope $UNIT already active" + exit 0 +fi +# A failed or OOM-killed transient scope keeps its unit name loaded, so a retry +# with the same deterministic name is rejected by systemd-run. Unload it first +# (and give systemd a moment) so retries can actually start. +systemctl --user reset-failed "$UNIT.scope" >/dev/null 2>&1 || true +sleep 1 + +STARTED="$(utc)" +date -u +%s > "$LOG_DIR/start.epoch" +echo "{\"run_id\": \"$RUN_ID\", \"unit\": \"$UNIT\", \"started_utc\": \"$STARTED\", \"pid\": $$}" > "$RUN_DIR/.running" + +log "START dataset=$DATASET alpha=$ALPHA seed=$SEED mode=$MODE beta=$BETA label=$LABEL" +log "command: systemd-run --user --scope -u $UNIT -p MemoryMax=$MEMORY_MAX -p MemorySwapMax=0 ..." + +systemd-run --user --scope -q --unit="$UNIT" \ + -p "MemoryMax=$MEMORY_MAX" \ + -p "MemorySwapMax=0" \ + --setenv=OMP_NUM_THREADS=1 \ + --setenv=MKL_NUM_THREADS=1 \ + --setenv=OPENBLAS_NUM_THREADS=1 \ + --setenv=NUMEXPR_NUM_THREADS=1 \ + --setenv=VECLIB_MAXIMUM_THREADS=1 \ + --setenv=PYTHONDONTWRITEBYTECODE=1 \ + /usr/bin/time -v -o "$LOG_DIR/time.txt" \ + timeout --signal=TERM --kill-after=60s "$RUN_TIMEOUT_S" \ + bash "$HERE/run_one_inner_v2.sh" "$RUN_ID" "$DATASET" "$ALPHA" "$SEED" "$MODE" "$BETA" "$SHRINK" "$CAP" "$MINSUP" "$PART_DIR" "$LABEL" \ + > "$LOG_DIR/stdout.log" 2> "$LOG_DIR/stderr.log" +RC=$? +ENDED="$(utc)" +date -u +%s > "$LOG_DIR/end.epoch" + +# A systemd cgroup OOM kill can surface as 143 (the timeout process is +# terminated with the rest of the cgroup) rather than 137. Query the transient +# scope's Result so the driver's memory-kill throttle sees it correctly. +SCOPE_RESULT="$(systemctl --user show -p Result --value "$UNIT.scope" 2>/dev/null || true)" +if grep -q "Failed to start transient scope unit" "$LOG_DIR/stderr.log" 2>/dev/null; then + log "scope did not start (unit-name collision); not treating as OOM" +elif [ "$SCOPE_RESULT" = "oom-kill" ]; then + log "scope result=oom-kill (exit was $RC); recording as memory kill 137" + RC=137 +fi +log "END exit_code=$RC" + +rm -f "$RUN_DIR/.running" + +# ---- completion marker / summary --------------------------------------- +if [ "$RC" -eq 0 ] && primary_ok; then + if "$PY" "$HERE/scripts/run_record_v2.py" marker \ + --run-dir "$RUN_DIR" --run-name "$RUN_ID" --dataset "$DATASET" \ + --alpha "$ALPHA" --beta "$BETA" --seed "$SEED" \ + --aggregation-mode "$MODE" --exit-code "$RC" \ + --time-file "$LOG_DIR/time.txt" --finished "$ENDED"; then + summary_row "completed" "$RC" "" + log "COMPLETE marker written" + exit 0 + else + summary_row "failed" "$RC" "artifacts missing; COMPLETE not written" + printf '{"exit_code": %s, "reason": "artifacts_missing", "ended_utc": "%s"}\n' "$RC" "$ENDED" > "$RUN_DIR/.failed" + exit 1 + fi +fi + +note="exit=$RC" +[ "$RC" -eq 124 ] && note="timeout after ${RUN_TIMEOUT_S}s" +[ "$RC" -eq 137 ] && note="killed (137): likely MemoryMax=$MEMORY_MAX or OOM" +[ "$RC" -eq 0 ] && note="exit=0 but one or more primary artifacts are missing/empty" +summary_row "failed" "$RC" "$note" +printf '{"exit_code": %s, "reason": "%s", "ended_utc": "%s"}\n' "$RC" "$note" "$ENDED" > "$RUN_DIR/.failed" +log "ERROR: run not marked complete ($note)" +exit 1 diff --git a/exploration_v2/scripts/analyze_results.py b/exploration_v2/scripts/analyze_results.py new file mode 100644 index 0000000..069c341 --- /dev/null +++ b/exploration_v2/scripts/analyze_results.py @@ -0,0 +1,564 @@ +#!/usr/bin/env python +"""Final analysis for exploration_v2 — all tables and REPORT_DATA.md. + +Runs only after the ranking has been frozen and confirmation has finished (or +the driver stopped). It may read test metrics of confirmation runs; it never +uses them to re-rank or re-select anything. +""" + +from __future__ import annotations + +import csv +import json +import statistics as st +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +EXPL = REPO / "exploration_v2" +RUNS = EXPL / "results" / "runs" +OUT = EXPL / "analysis" +OUT.mkdir(parents=True, exist_ok=True) + + +def read_csv(p): + if not Path(p).is_file(): + return [] + with Path(p).open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def write_csv(name, rows): + p = OUT / name + if not rows: + p.write_text("", encoding="utf-8") + return + with p.open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + + +def jload(p): + p = Path(p) + if not p.is_file(): + return {} + return json.loads(p.read_text(encoding="utf-8")) + + +def resolve(rid): + """Return the directory containing a run's metric files.""" + a = RUNS / rid / "results" / rid + if (a / "val_global_metrics.json").is_file() or (a / "test_global_metrics.json").is_file(): + return a + b = REPO / "results" / rid + if (b / "val_global_metrics.json").is_file() or (b / "test_global_metrics.json").is_file(): + return b + return None + + +def metrics(rid): + d = resolve(rid) + if d is None: + return None + vg = jload(d / "val_global_metrics.json") + vs = jload(d / "val_selection.json") + tg = jload(d / "test_global_metrics.json") + return { + "val_global_rare_f1": vg.get("rare_class_macro_f1"), + "val_global_rare_recall": vg.get("rare_class_recall"), + "val_global_macro_f1": vg.get("macro_f1"), + "val_per_client_rare_f1": vs.get("val_rare_class_macro_f1_per_client_mean"), + "test_rare_f1": tg.get("rare_class_macro_f1"), + "test_rare_recall": tg.get("rare_class_recall"), + "test_macro_f1": tg.get("macro_f1"), + } + + +def summary_rows(): + rows = read_csv(EXPL / "summary.csv") + return {r["config"]: r for r in rows} + + +def pair_stats(diffs): + diffs = [d for d in diffs if d is not None] + if not diffs: + return None, None, 0, 0 + return (st.fmean(diffs), + st.stdev(diffs) if len(diffs) > 1 else None, + sum(1 for d in diffs if d > 0), + len(diffs)) + + +def main() -> int: + OUT.mkdir(parents=True, exist_ok=True) + # Enforce the pre-registration gate: never analyse confirmation test + # metrics unless the ranking was frozen first. + rank_file = EXPL / "ranking.json" + freeze_file = EXPL / "ranking.sha256" + if not rank_file.is_file() or not freeze_file.is_file(): + raise SystemExit("ranking.json/ranking.sha256 missing: refusing to run " + "the post-confirmation analysis") + import hashlib + recorded = freeze_file.read_text(encoding="utf-8").splitlines()[0].split()[0] + actual = hashlib.sha256(rank_file.read_bytes()).hexdigest() + if recorded != actual: + raise SystemExit("ranking.json hash does not match the frozen record; " + "refusing to re-rank or analyse") + ranking = jload(rank_file) + screen_q = read_csv(EXPL / "queue_screen.csv") + confirm_q = read_csv(EXPL / "queue_confirm.csv") + summ = summary_rows() + top2 = ranking.get("top2", []) + + # ---------------- inventory ---------------- + inv = [] + for phase, queue in (("screening", screen_q), ("confirm", confirm_q)): + for r in queue: + rid = r["run_id"] + d = RUNS / rid + if (d / "COMPLETE").is_file(): + status = "done" + elif (d / ".failed").is_file(): + status = "failed" + elif (d / ".running").is_file(): + status = "running" + elif d.is_dir(): + status = "incomplete" + else: + status = "queued" + s = summ.get(rid, {}) + inv.append({ + "phase": phase, "run_id": rid, "dataset": r["dataset"], + "alpha": r["alpha"], "seed": r["seed"], "mode": r["mode"], + "beta": r.get("beta", ""), "shrink_lambda": r.get("shrink", ""), + "cap_factor": r.get("cap", ""), "min_support": r.get("minsup", ""), + "label": r.get("label", ""), "status": status, + "wall_s": s.get("wall_s", ""), "peak_rss_kb": s.get("peak_rss_kb", ""), + "exit_code": s.get("exit_code", ""), "notes": s.get("notes", ""), + }) + write_csv("run_inventory.csv", inv) + + # ---------------- screening ranking ---------------- + screen_rows = [] + for c in ranking.get("candidates", []): + row = { + "rank": c["rank"], "rank_secondary": c.get("rank_secondary", ""), + "label": c["label"], "mode": c["mode"], + "beta": c["params"].get("beta"), + "shrink_lambda": c["params"].get("shrink_lambda"), + "cap_factor": c["params"].get("cap_factor"), + "min_support": c["params"].get("min_support"), + "score_mean_delta_val_global": c["score"], + "min_condition_delta": c["min_condition_delta"], + "secondary_mean_delta_val_per_client": c["secondary_score_per_client"], + } + for cid, per in c["per_condition"].items(): + row[f"{cid}_val_global"] = per["val_global_rare_f1"] + row[f"{cid}_baseline"] = per["baseline_val_global_rare_f1"] + row[f"{cid}_delta"] = per["delta_vs_original"] + row[f"{cid}_val_per_client"] = per["val_per_client_rare_f1"] + row[f"{cid}_delta_per_client"] = per["delta_per_client_vs_original"] + screen_rows.append(row) + write_csv("screening_ranking.csv", screen_rows) + + # reference rows for the same conditions + ref_rows = [] + for cid, ref in ranking.get("current_method_reference", {}).items(): + ref_rows.append({ + "condition": cid, + "baseline_run": ref["original"]["run_id"], + "baseline_val_global": ref["original"]["val_global_rare_f1"], + "best_beta_by_val_global": ref["best_beta_by_val_global"]["beta"], + "best_val_global": ref["best_beta_by_val_global"]["val_global_rare_f1"], + "best_delta_global": ref["best_beta_by_val_global"]["delta_vs_original"], + "best_beta_by_val_per_client": ref["best_beta_by_val_per_client"]["beta"], + "best_val_per_client": ref["best_beta_by_val_per_client"]["val_per_client_rare_f1"], + "best_delta_per_client": ref["best_beta_by_val_per_client"]["delta_vs_original"], + }) + write_csv("screening_references.csv", ref_rows) + + # ---------------- confirmation ---------------- + conditions = {"S1": ("xiiotid", 0.25), "S2": ("5gnidd", 0.5)} + + def rid_for(ds, alpha, seed, label): + return f"{ds}__alpha{alpha:g}__seed{seed}__{label}" + + confirm_rows = [] + for ds, alpha in conditions.values(): + for seed in (3, 4): + base_id = rid_for(ds, alpha, seed, "original_protean") + base = metrics(base_id) + b05 = metrics(rid_for(ds, alpha, seed, "support_weighted__beta0.5")) + b1 = metrics(rid_for(ds, alpha, seed, "support_weighted__beta1")) + if base is None: + continue + cur_candidates = [("original_protean", base), + ("support_weighted__beta0.5", b05), + ("support_weighted__beta1", b1)] + cur_candidates = [(n, m) for n, m in cur_candidates + if m is not None and m["val_global_rare_f1"] is not None] + cur_name, cur_best = max( + cur_candidates, key=lambda nm: nm[1]["val_global_rare_f1"]) + for label in top2 + ["support_weighted__beta0.5", "support_weighted__beta1"]: + rid = rid_for(ds, alpha, seed, label) + m = metrics(rid) + if m is None: + continue + confirm_rows.append({ + "dataset": ds, "alpha": alpha, "seed": seed, + "config": label, "run_id": rid, + "val_global_rare_f1": m["val_global_rare_f1"], + "val_per_client_rare_f1": m["val_per_client_rare_f1"], + "test_rare_f1": m["test_rare_f1"], + "test_rare_recall": m["test_rare_recall"], + "baseline_val_global": base["val_global_rare_f1"], + "baseline_test_rare_f1": base["test_rare_f1"], + "delta_val_vs_original": (m["val_global_rare_f1"] - base["val_global_rare_f1"]), + "delta_test_vs_original": (m["test_rare_f1"] - base["test_rare_f1"]), + "delta_val_vs_beta1": ((m["val_global_rare_f1"] - b1["val_global_rare_f1"]) + if b1 and b1["val_global_rare_f1"] is not None else None), + "delta_test_vs_beta1": ((m["test_rare_f1"] - b1["test_rare_f1"]) + if b1 and b1["test_rare_f1"] is not None else None), + "delta_val_vs_beta05": ((m["val_global_rare_f1"] - b05["val_global_rare_f1"]) + if b05 and b05["val_global_rare_f1"] is not None else None), + "delta_test_vs_beta05": ((m["test_rare_f1"] - b05["test_rare_f1"]) + if b05 and b05["test_rare_f1"] is not None else None), + "current_best_config": cur_name, + "current_best_val_global": cur_best["val_global_rare_f1"], + "delta_val_vs_current_best": (m["val_global_rare_f1"] + - cur_best["val_global_rare_f1"]), + "delta_test_vs_current_best": ((m["test_rare_f1"] - cur_best["test_rare_f1"]) + if m["test_rare_f1"] is not None + and cur_best["test_rare_f1"] is not None + else None), + }) + write_csv("confirm_paired.csv", confirm_rows) + + summary = [] + for label in top2: + rows = [r for r in confirm_rows if r["config"] == label] + dv = [r["delta_val_vs_original"] for r in rows] + dt = [r["delta_test_vs_original"] for r in rows] + d1v = [r["delta_val_vs_beta1"] for r in rows] + d1t = [r["delta_test_vs_beta1"] for r in rows] + d05v = [r["delta_val_vs_beta05"] for r in rows] + d05t = [r["delta_test_vs_beta05"] for r in rows] + dcbv = [r["delta_val_vs_current_best"] for r in rows] + dcbt = [r["delta_test_vs_current_best"] for r in rows] + mcbv, scbv, poscbv, _ = pair_stats(dcbv) + mcbt, scbt, poscbt, _ = pair_stats(dcbt) + mv, sv, pos, n = pair_stats(dv) + mt, stt, post, _ = pair_stats(dt) + m1v, s1v, _, _ = pair_stats(d1v) + m1t, s1t, _, _ = pair_stats(d1t) + m05v, s05v, _, _ = pair_stats(d05v) + m05t, s05t, _, _ = pair_stats(d05t) + per_cond = {} + for cid, (ds, alpha) in conditions.items(): + sub = [r for r in rows if r["dataset"] == ds and r["alpha"] == alpha] + per_cond[cid] = [r["delta_val_vs_original"] for r in sub] + improved = bool(mv is not None and mv > 0 and pos == n and n == 4) + summary.append({ + "variant": label, + "n_pairs": n, + "mean_delta_val_vs_original": mv, "sd_delta_val_vs_original": sv, + "positive_pairs_val_vs_original": pos, + "mean_delta_test_vs_original": mt, "sd_delta_test_vs_original": stt, + "positive_pairs_test_vs_original": post, + "mean_delta_val_vs_beta1": m1v, "sd_delta_val_vs_beta1": s1v, + "mean_delta_test_vs_beta1": m1t, "sd_delta_test_vs_beta1": s1t, + "mean_delta_val_vs_beta05": m05v, "sd_delta_val_vs_beta05": s05v, + "mean_delta_test_vs_beta05": m05t, "sd_delta_test_vs_beta05": s05t, + "mean_delta_val_vs_current_best": mcbv, + "sd_delta_val_vs_current_best": scbv, + "positive_pairs_val_vs_current_best": poscbv, + "mean_delta_test_vs_current_best": mcbt, + "sd_delta_test_vs_current_best": scbt, + "positive_pairs_test_vs_current_best": poscbt, + "improved_vs_original_per_plan": improved, + "S1_val_deltas": "|".join(f"{x:.6f}" for x in per_cond.get("S1", [])), + "S2_val_deltas": "|".join(f"{x:.6f}" for x in per_cond.get("S2", [])), + }) + write_csv("confirm_summary.csv", summary) + + # reference configurations (beta1, beta05) summary over the same seeds + ref_summary = [] + for label in ("support_weighted__beta0.5", "support_weighted__beta1"): + rows = [r for r in confirm_rows if r["config"] == label] + dv = [r["delta_val_vs_original"] for r in rows] + dt = [r["delta_test_vs_original"] for r in rows] + mv, sv, pos, n = pair_stats(dv) + mt, stt, post, _ = pair_stats(dt) + ref_summary.append({"config": label, "n_pairs": n, + "mean_delta_val_vs_original": mv, + "sd_delta_val_vs_original": sv, + "positive_pairs_val": pos, + "mean_delta_test_vs_original": mt, + "sd_delta_test_vs_original": stt, + "positive_pairs_test": post}) + write_csv("confirm_reference_summary.csv", ref_summary) + + # ---------------- reference consistency (environment check) ---------- + ref_cons = [] + for cid, (ds, alpha) in conditions.items(): + base = f"{ds}__alpha{alpha:g}__seed0" + new_id = f"{base}__original_protean" + new_dir = RUNS / new_id / "results" / new_id + new_val = (jload(new_dir / "val_global_metrics.json").get("rare_class_macro_f1") + if (new_dir / "val_global_metrics.json").is_file() + else None) + old_id = new_id + old_val = jload(REPO / "results" / old_id / "val_global_metrics.json").get( + "rare_class_macro_f1") + sweep_id = f"{base}__support_weighted__beta0" + sweep_dir = REPO / "results" / "sweep" / "runs" / sweep_id / "results" / sweep_id + sweep_val = (jload(sweep_dir / "val_global_metrics.json").get("rare_class_macro_f1") + if (sweep_dir / "val_global_metrics.json").is_file() else None) + ref_cons.append({ + "condition": cid, "dataset": ds, "alpha": alpha, + "new_original_val_global": new_val, + "restored_original_val_global": old_val, + "sweep_beta0_val_global": sweep_val, + "new_minus_restored": (new_val - old_val) if new_val is not None and old_val is not None else None, + "new_minus_sweep_beta0": (new_val - sweep_val) if new_val is not None and sweep_val is not None else None, + }) + write_csv("reference_consistency.csv", ref_cons) + + # ---------------- wall time / memory ---------------- + wall = [] + for r in inv: + wall.append({"phase": r["phase"], "run_id": r["run_id"], + "status": r["status"], "wall_s": r["wall_s"], + "wall_h": (round(float(r["wall_s"]) / 3600, 3) + if r["wall_s"] not in ("", None) else None), + "peak_rss_gib": (round(float(r["peak_rss_kb"]) / 1048576, 3) + if r["peak_rss_kb"] not in ("", None) else None)}) + write_csv("walltime.csv", wall) + + # ---------------- REPORT_DATA.md ---------------- + def fmt(x, nd=4): + if x is None or x == "": + return "" + try: + return f"{float(x):.{nd}f}" + except Exception: + return str(x) + + lines = [] + lines.append("# exploration_v2 — report data\n") + from datetime import datetime, timezone + lines.append(f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}\n") + lines.append("This file is generated by `exploration_v2/scripts/analyze_results.py`. " + "All numbers come from files on disk. Missing rows mean a run did not " + "produce the metric.\n") + + lines.append("## Plan and ranking freeze\n") + lines.append("| item | value |\n|---|---|\n") + for name in ("PLAN.sha256", "ranking.sha256"): + p = EXPL / name + if p.is_file(): + content = p.read_text(encoding="utf-8").splitlines() + lines.append(f"| `{name}` | `{content[0] if content else ''}`" + f"{(' at ' + content[1]) if len(content) > 1 else ''} |\n") + lines.append("\n") + + lines.append("## Screening ranking (validation only)\n") + if ranking: + lines.append(f"Top-1 minus top-2 validation score (selection margin): " + f"{fmt(ranking.get('selection_margin_top1_minus_top2'), 6)} " + f"(top2 = {ranking.get('top2')}).\n\n") + lines.append("Score = mean over S1,S2 of the paired validation difference in " + "global rare-class macro-F1 versus the original baseline.\n\n") + lines.append("| rank | variant | β | λ | cap | min N | score | min cond Δ | " + "S1 Δ | S2 Δ | secondary rank |\n|---|---|---|---|---|---|---|---|---|---|---|\n") + for r in screen_rows: + lines.append(f"| {r['rank']} | `{r['label']}` | {fmt(r['beta'],2)} | " + f"{fmt(r['shrink_lambda'],2)} | {fmt(r['cap_factor'],1)} | " + f"{fmt(r['min_support'],0)} | {fmt(r['score_mean_delta_val_global'])} | " + f"{fmt(r['min_condition_delta'])} | {fmt(r.get('S1_delta'))} | " + f"{fmt(r.get('S2_delta'))} | {r['rank_secondary']} |\n") + lines.append("\n") + + lines.append("## Confirmatory results (per seed)\n") + lines.append("| dataset | α | seed | config | val rare-F1 | test rare-F1 | " + "Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 |\n" + "|---|---|---|---|---|---|---|---|---|---|\n") + for r in confirm_rows: + lines.append(f"| {r['dataset']} | {fmt(r['alpha'],2)} | {r['seed']} | " + f"`{r['config']}` | {fmt(r['val_global_rare_f1'])} | " + f"{fmt(r['test_rare_f1'])} | {fmt(r['delta_val_vs_original'])} | " + f"{fmt(r['delta_test_vs_original'])} | {fmt(r['delta_val_vs_beta1'])} | " + f"{fmt(r['delta_test_vs_beta1'])} |\n") + lines.append("\n") + + lines.append("## Confirmatory summary\n") + lines.append("| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | " + "sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | " + "mean Δtest vs current best | improved per plan |\n" + "|---|---|---|---|---|---|---|---|---|\n") + for r in summary: + lines.append(f"| `{r['variant']}` | {fmt(r['mean_delta_val_vs_original'])} | " + f"{fmt(r['sd_delta_val_vs_original'])} | " + f"{r['positive_pairs_val_vs_original']}/{r['n_pairs']} | " + f"{fmt(r['mean_delta_test_vs_original'])} | " + f"{fmt(r['sd_delta_test_vs_original'])} | " + f"{fmt(r['mean_delta_val_vs_beta1'])} | " + f"{fmt(r['mean_delta_test_vs_beta1'])} | " + f"{fmt(r.get('mean_delta_val_vs_current_best'))} | " + f"{fmt(r.get('mean_delta_test_vs_current_best'))} | " + f"{r['improved_vs_original_per_plan']} |\n") + lines.append("\n") + + lines.append("## Reference configurations on the confirmation seeds\n") + lines.append("| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive |\n" + "|---|---|---|---|---|---|---|\n") + for r in ref_summary: + lines.append(f"| `{r['config']}` | {fmt(r['mean_delta_val_vs_original'])} | " + f"{fmt(r['sd_delta_val_vs_original'])} | {r['positive_pairs_val']} | " + f"{fmt(r['mean_delta_test_vs_original'])} | " + f"{fmt(r['sd_delta_test_vs_original'])} | {r['positive_pairs_test']} |\n") + lines.append("\n") + + lines.append("## Reference consistency (environment check)\n") + lines.append("| condition | new original val | restored original val | sweep β=0 val | " + "new − restored | new − sweep β=0 |\n|---|---|---|---|---|---|\n") + for r in ref_cons: + lines.append(f"| {r['condition']} | {fmt(r['new_original_val_global'])} | " + f"{fmt(r['restored_original_val_global'])} | " + f"{fmt(r['sweep_beta0_val_global'])} | " + f"{fmt(r['new_minus_restored'])} | " + f"{fmt(r['new_minus_sweep_beta0'])} |\n") + lines.append("\n") + + lines.append("## Wall time and memory (new runs)\n") + lines.append("| phase | run | status | wall h | peak GiB |\n|---|---|---|---|---|\n") + for r in wall: + lines.append(f"| {r['phase']} | `{r['run_id']}` | {r['status']} | " + f"{fmt(r['wall_h'],2)} | {fmt(r['peak_rss_gib'],2)} |\n") + lines.append("\n") + + lines.append("## Failed or missing runs\n") + bad = [r for r in inv if r["status"] in ("failed", "incomplete", "queued")] + if not bad: + lines.append("None.\n") + else: + lines.append("| phase | run | status | exit | notes |\n|---|---|---|---|---|\n") + for r in bad: + lines.append(f"| {r['phase']} | `{r['run_id']}` | {r['status']} | " + f"{r['exit_code']} | {r['notes']} |\n") + lines.append("\n") + + (EXPL / "REPORT_DATA.md").write_text("".join(lines), encoding="utf-8") + + # ---- inject the same tables into REPORT.md between the markers ------- + def f(x, nd=4): + if x in (None, ""): + return "" + try: + return f"{float(x):.{nd}f}" + except Exception: + return str(x) + + auto = [] + auto.append("## 5. Results of the new runs\n\n") + auto.append("### 5.1 Screening ranking (validation only)\n\n") + if ranking: + auto.append(f"Top-1 minus top-2 validation score (selection margin): " + f"{f(ranking.get('selection_margin_top1_minus_top2'), 6)} " + f"(top2 = {ranking.get('top2')}).\n\n") + auto.append("Score = mean over S1,S2 of the paired validation difference in " + "global rare-class macro-F1 versus the in-environment original " + "baseline. `rank_secondary` uses the deployed per-client-mean " + "metric.\n\n") + auto.append("| rank | variant | β | λ | cap | min N | score | min cond Δ | " + "S1 Δ | S2 Δ | rank_secondary |\n|---|---|---|---|---|---|---|---|---|---|\n") + for r in read_csv(OUT / "screening_ranking.csv"): + auto.append(f"| {r['rank']} | `{r['label']}` | {f(r['beta'],2)} | " + f"{f(r['shrink_lambda'],2)} | {f(r['cap_factor'],1)} | " + f"{f(r['min_support'],0)} | {f(r['score_mean_delta_val_global'])} | " + f"{f(r['min_condition_delta'])} | {f(r.get('S1_delta'))} | " + f"{f(r.get('S2_delta'))} | {r['rank_secondary']} |\n") + auto.append("\n") + auto.append("### 5.2 Confirmatory runs, per seed\n\n") + auto.append("| dataset | α | seed | config | val rare-F1 | test rare-F1 | " + "Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 |\n" + "|---|---|---|---|---|---|---|---|---|---|\n") + for r in read_csv(OUT / "confirm_paired.csv"): + auto.append(f"| {r['dataset']} | {f(r['alpha'],2)} | {r['seed']} | " + f"`{r['config']}` | {f(r['val_global_rare_f1'])} | " + f"{f(r['test_rare_f1'])} | {f(r['delta_val_vs_original'])} | " + f"{f(r['delta_test_vs_original'])} | {f(r['delta_val_vs_beta1'])} | " + f"{f(r['delta_test_vs_beta1'])} |\n") + auto.append("\n") + auto.append("### 5.3 Confirmatory summary\n\n") + auto.append("A variant “improves” under the pre-registered rule only if the " + "mean validation difference over the four pairs is positive **and** " + "all four are positive.\n\n") + auto.append("| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | " + "sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | " + "mean Δtest vs current best | improved |\n" + "|---|---|---|---|---|---|---|---|---|\n") + for r in read_csv(OUT / "confirm_summary.csv"): + auto.append(f"| `{r['variant']}` | {f(r['mean_delta_val_vs_original'])} | " + f"{f(r['sd_delta_val_vs_original'])} | " + f"{r['positive_pairs_val_vs_original']}/{r['n_pairs']} | " + f"{f(r['mean_delta_test_vs_original'])} | " + f"{f(r['sd_delta_test_vs_original'])} | " + f"{f(r['mean_delta_val_vs_beta1'])} | " + f"{f(r['mean_delta_test_vs_beta1'])} | " + f"{f(r.get('mean_delta_val_vs_current_best'))} | " + f"{f(r.get('mean_delta_test_vs_current_best'))} | " + f"{r['improved_vs_original_per_plan']} |\n") + auto.append("\n") + auto.append("### 5.4 Reference configurations on the confirmation seeds\n\n") + auto.append("| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive |\n" + "|---|---|---|---|---|---|---|\n") + for r in read_csv(OUT / "confirm_reference_summary.csv"): + auto.append(f"| `{r['config']}` | {f(r['mean_delta_val_vs_original'])} | " + f"{f(r['sd_delta_val_vs_original'])} | {r['positive_pairs_val']} | " + f"{f(r['mean_delta_test_vs_original'])} | " + f"{f(r['sd_delta_test_vs_original'])} | {r['positive_pairs_test']} |\n") + auto.append("\n") + auto.append("### 5.5 Reference consistency (environment check)\n\n") + auto.append("| condition | new original val | restored original val | sweep β=0 val | " + "new − restored | new − sweep β=0 |\n|---|---|---|---|---|---|\n") + for r in read_csv(OUT / "reference_consistency.csv"): + auto.append(f"| {r['condition']} | {f(r['new_original_val_global'])} | " + f"{f(r['restored_original_val_global'])} | " + f"{f(r['sweep_beta0_val_global'])} | {f(r['new_minus_restored'])} | " + f"{f(r['new_minus_sweep_beta0'])} |\n") + auto.append("\n") + auto.append("### 5.6 Wall time and memory\n\n") + auto.append("| phase | run | status | wall h | peak GiB |\n|---|---|---|---|---|\n") + for r in read_csv(OUT / "walltime.csv"): + auto.append(f"| {r['phase']} | `{r['run_id']}` | {r['status']} | " + f"{f(r['wall_h'],2)} | {f(r['peak_rss_gib'],2)} |\n") + auto.append("\n") + bad = [r for r in inv if r["status"] in ("failed", "incomplete", "queued")] + auto.append("### 5.7 Failed or missing runs\n\n") + if not bad: + auto.append("None.\n") + else: + for r in bad: + auto.append(f"- `{r['run_id']}` ({r['phase']}): {r['status']}, " + f"exit {r['exit_code']}, {r['notes']}\n") + + report = EXPL / "REPORT.md" + if report.is_file(): + text = report.read_text(encoding="utf-8") + start = "" + end = "" + if start in text and end in text: + pre = text.split(start, 1)[0] + start + "\n" + post = "\n" + end + text.split(end, 1)[1] + report.write_text(pre + "".join(auto) + post, encoding="utf-8") + + print(json.dumps({"screened": len(screen_rows), "confirm_rows": len(confirm_rows), + "summary_rows": len(summary), + "failed": [r["run_id"] for r in inv if r["status"] == "failed"]}, + indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/confirm_effects.py b/exploration_v2/scripts/confirm_effects.py new file mode 100644 index 0000000..72a292e --- /dev/null +++ b/exploration_v2/scripts/confirm_effects.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +"""Per-class and per-client effects of the confirmed variants (post-freeze). + +Reads the 20 confirmation runs and the matching original baseline runs. Writes +two CSVs in exploration_v2/analysis/. Descriptive only; no selection. +""" + +from __future__ import annotations + +import csv +import statistics as st +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +RUNS = REPO / "exploration_v2" / "results" / "runs" +OUT = REPO / "exploration_v2" / "analysis" +PAIRS = [("xiiotid", 0.25, 3), ("xiiotid", 0.25, 4), + ("5gnidd", 0.5, 3), ("5gnidd", 0.5, 4)] +VARIANTS = ["v_shrink__b1__lam0.5", "v_cap__b1__c4", + "support_weighted__beta0.5", "support_weighted__beta1"] + + +def rows(p): + with p.open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def base(rid): + return RUNS / rid / "results" / rid + + +def write(name, data): + if not data: + return + with (OUT / name).open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(data[0])) + w.writeheader() + w.writerows(data) + + +def main() -> int: + OUT.mkdir(parents=True, exist_ok=True) + cls_rows, cli_rows = [], [] + for ds, alpha, seed in PAIRS: + b = f"{ds}__alpha{alpha:g}__seed{seed}__original_protean" + bp = {int(r["class_id"]): r for r in rows(base(b) / "test_per_class.csv")} + bm = {int(r["client_id"]): r for r in rows(base(b) / "val_per_client.csv")} + for label in VARIANTS: + c = f"{ds}__alpha{alpha:g}__seed{seed}__{label}" + cp = {int(r["class_id"]): r for r in rows(base(c) / "test_per_class.csv")} + for cid, br in bp.items(): + if float(br["support"]) <= 0: + continue + cr = cp.get(cid) + if cr is None: + continue + cls_rows.append({ + "dataset": ds, "alpha": alpha, "seed": seed, "variant": label, + "class_id": cid, "class_name": br["class_name"], + "test_support": int(float(br["support"])), + "base_f1": br["f1"], "variant_f1": cr["f1"], + "delta_f1": float(cr["f1"]) - float(br["f1"]), + "base_recall": br["recall"], "variant_recall": cr["recall"], + "delta_recall": float(cr["recall"]) - float(br["recall"]), + }) + cm = {int(r["client_id"]): r for r in rows(base(c) / "val_per_client.csv")} + for cid, br in bm.items(): + cr = cm.get(cid) + if cr is None or br["rare_class_macro_f1"] in ("", None): + continue + if cr["rare_class_macro_f1"] in ("", None): + continue + cli_rows.append({ + "dataset": ds, "alpha": alpha, "seed": seed, "variant": label, + "client_id": cid, "n_samples": cr["n_samples"], + "base_val_rare_f1": br["rare_class_macro_f1"], + "variant_val_rare_f1": cr["rare_class_macro_f1"], + "delta_val_rare_f1": (float(cr["rare_class_macro_f1"]) + - float(br["rare_class_macro_f1"])), + }) + write("confirm_per_class_variant.csv", cls_rows) + write("confirm_per_client_variant.csv", cli_rows) + + # compact pooled summary for the two confirmed variants + for label in VARIANTS: + print("VAR", label) + for ds in ("xiiotid", "5gnidd"): + sub = [r for r in cls_rows if r["variant"] == label and r["dataset"] == ds] + by = {} + for r in sub: + by.setdefault(r["class_name"], []).append(r["delta_f1"]) + order = sorted(by, key=lambda k: -st.fmean(by[k])) + print(" ", ds, [(k, round(st.fmean(by[k]), 3)) for k in order]) + cc = [r for r in cli_rows if r["variant"] == label and r["dataset"] == ds] + d = [r["delta_val_rare_f1"] for r in cc] + print(" per-client mean %.4f improve/degrade %d/%d" % ( + st.fmean(d), sum(1 for x in d if x > 0), sum(1 for x in d if x < 0))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/make_confirm_queue.py b/exploration_v2/scripts/make_confirm_queue.py new file mode 100644 index 0000000..4366548 --- /dev/null +++ b/exploration_v2/scripts/make_confirm_queue.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +"""Build exploration_v2/queue_confirm.csv from the frozen ranking.json. + +Confirmation design (PLAN.md §6): top-2 variants plus three reference configs +(original, support_weighted beta=0.5, support_weighted beta=1.0) on the two +screening conditions and the two new seeds 3 and 4. +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +EXPL = REPO / "exploration_v2" + + +def main() -> int: + ranking = json.loads((EXPL / "ranking.json").read_text(encoding="utf-8")) + top2 = ranking["top2"] + if len(top2) != 2: + raise SystemExit(f"ranking.json does not contain 2 candidates: {top2}") + + specs = {} + with (EXPL / "queue_screen.csv").open(newline="", encoding="utf-8") as fh: + for row in csv.DictReader(fh): + specs[row["label"]] = row + for label in top2: + if label not in specs: + raise SystemExit(f"top candidate {label} has no screening spec") + + rows = [] + for ds, alpha in (("xiiotid", 0.25), ("5gnidd", 0.5)): + for seed in (3, 4): + base = f"{ds}__alpha{alpha:g}__seed{seed}" + rows.append(dict(run_id=f"{base}__original_protean", dataset=ds, + alpha=alpha, seed=seed, mode="original_protean", + beta="", shrink="", cap="", minsup="", + part_dir="exploration_v2/partitions", + label="original_protean", role="reference")) + for b in ("0.5", "1"): + rows.append(dict(run_id=f"{base}__support_weighted__beta{b}", + dataset=ds, alpha=alpha, seed=seed, + mode="support_weighted", beta=b, shrink="", cap="", + minsup="", part_dir="exploration_v2/partitions", + label=f"support_weighted__beta{b}", + role="reference")) + for label in top2: + s = specs[label] + rows.append(dict(run_id=f"{base}__{label}", dataset=ds, + alpha=alpha, seed=seed, mode=s["mode"], + beta=s["beta"], shrink=s["shrink"], cap=s["cap"], + minsup=s["minsup"], + part_dir="exploration_v2/partitions", + label=label, role="confirm_variant")) + + out = EXPL / "queue_confirm.csv" + with out.open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + print(f"wrote {out} ({len(rows)} rows) top2={top2}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/make_manifest_v2.py b/exploration_v2/scripts/make_manifest_v2.py new file mode 100644 index 0000000..dacc4e1 --- /dev/null +++ b/exploration_v2/scripts/make_manifest_v2.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +"""Create one new-seed partition manifest inside exploration_v2/. + +This reuses the frozen project partition procedure +(``protean_research.data.pipeline.create_partition`` -> +``per_class_dirichlet_multinomial``) and the frozen split/preprocessing code. +It never writes into ``artifacts/partitions``. + +Usage: + python exploration_v2/scripts/make_manifest_v2.py --dataset xiiotid --alpha 0.25 --seed 3 +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO / "src")) + +from protean_research.data import manifest as manifest_mod # noqa: E402 +from protean_research.data import rare_unseen # noqa: E402 +from protean_research.data.characterization import characterize_support # noqa: E402 +from protean_research.data.pipeline import ( # noqa: E402 + create_partition, + prepare_dataset, + preprocessing_content_hash, +) +from protean_research.utils.config import load_datasets_config # noqa: E402 + + +def write_csv(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8") + return + with path.open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--dataset", required=True) + ap.add_argument("--alpha", type=float, required=True) + ap.add_argument("--seed", type=int, required=True) + ap.add_argument("--clients", type=int, default=10) + ap.add_argument("--partitions-dir", default=None) + ap.add_argument("--analysis-dir", default=None) + args = ap.parse_args(argv) + + partitions_dir = Path(args.partitions_dir or REPO / "exploration_v2" / "partitions") + analysis_dir = Path(args.analysis_dir or REPO / "exploration_v2" / "analysis" / "partitions") + partitions_dir.mkdir(parents=True, exist_ok=True) + analysis_dir.mkdir(parents=True, exist_ok=True) + + datasets_cfg = load_datasets_config() + bundle = prepare_dataset(args.dataset, seed=args.seed, datasets_cfg=datasets_cfg) + pre_hash = preprocessing_content_hash(bundle.preprocessor) + path = manifest_mod.manifest_path(partitions_dir, args.dataset, args.alpha, args.seed) + if path.exists(): + manifest = manifest_mod.load_manifest(path) + print(f"exists: {path}") + else: + manifest, assignments, support = create_partition( + bundle, + alpha=args.alpha, + seed=args.seed, + num_clients=args.clients, + partition_dir=partitions_dir, + preprocessing_hash=pre_hash, + ) + print(f"created: {path}") + + # Exact reload check against the stored recipe. + split, assignments, support = manifest_mod.reproduce_train_assignments( + manifest, bundle.labels) + tag = f"{args.dataset}__alpha{args.alpha:g}__seed{args.seed}" + write_csv(analysis_dir / f"{tag}__characterization.csv", + characterize_support(support, bundle.class_names)) + write_csv(analysis_dir / f"{tag}__rare_unseen.csv", + rare_unseen.rare_unseen_rows( + support, bundle.class_names, bundle.cleaned.benign_index, + args.dataset, args.alpha, args.seed)) + meta = { + "dataset": args.dataset, + "alpha": args.alpha, + "seed": args.seed, + "manifest_path": str(path), + "partition_hash": manifest["partition_hash"], + "manifest_content_hash": manifest_mod.manifest_content_hash(manifest), + "preprocessing_hash": pre_hash, + "num_clients": args.clients, + "num_classes": len(bundle.class_names), + "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "procedure": "protean_research.data.pipeline.create_partition " + "(per_class_dirichlet_multinomial), frozen split/preprocessing", + } + (analysis_dir / f"{tag}__manifest_meta.json").write_text( + json.dumps(meta, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"manifest": str(path), "partition_hash": meta["partition_hash"]})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/phase2_diagnose.py b/exploration_v2/scripts/phase2_diagnose.py new file mode 100644 index 0000000..059146f --- /dev/null +++ b/exploration_v2/scripts/phase2_diagnose.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python +"""Phase 2 read-only diagnosis of the existing 108-run grid. + +Reads only already-restored result directories (results//) plus the +committed summary tables under artifacts/analysis/. Writes only inside +exploration_v2/results/. Does not import project code, does not train. +""" + +from __future__ import annotations + +import csv +import json +import math +import statistics as st +from itertools import product +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +OUT = REPO / "exploration_v2" / "results" +OUT.mkdir(parents=True, exist_ok=True) + +DATASETS = ["xiiotid", "5gnidd"] +ALPHAS = [0.25, 0.5, 0.75] +SEEDS = [0, 1, 2] +BETAS = [0.0, 0.25, 0.5, 0.75, 1.0] + + +def read_json(path): + with path.open(encoding="utf-8") as fh: + return json.load(fh) + + +def read_csv(path): + with path.open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def fnum(v): + if v is None or v == "": + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + +def run_dir(dataset, alpha, seed, tag): + return REPO / "results" / f"{dataset}__alpha{alpha:g}__seed{seed}__{tag}" + + +def tag_for(beta): + if beta is None: + return "original_protean" + return f"support_weighted__beta{beta:g}" + + +def load_run(dataset, alpha, seed, beta): + d = run_dir(dataset, alpha, seed, tag_for(beta)) + if not d.is_dir(): + return None + rec = { + "dataset": dataset, "alpha": alpha, "seed": seed, + "beta": beta, "mode": "original" if beta is None else "support_weighted", + "dir": str(d.relative_to(REPO)), + } + for name in ["metadata", "val_selection", "val_global_metrics", + "test_global_metrics", "history"]: + p = d / f"{name}.json" + rec[name] = read_json(p) if p.exists() else None + for name in ["val_per_client", "per_client", "val_per_class", "test_per_class", + "train_support_Nij", "rare_unseen"]: + p = d / f"{name}.csv" + rec[name] = read_csv(p) if p.exists() else None + return rec + + +def support_stats(rec, class_names): + rows = rec.get("train_support_Nij") or [] + if not rows: + return {} + support = [] + for r in rows: + support.append([int(float(r[c])) for c in class_names]) + n_clients = len(support) + n_classes = len(class_names) + per_class = [] + for j in range(n_classes): + vals = [support[i][j] for i in range(n_clients)] + pos = [v for v in vals if v > 0] + total = sum(vals) + if not pos: + per_class.append({"class_id": j, "n_contrib": 0, "cv": None, + "max_share": 0.0, "min_pos": 0, "max_pos": 0}) + continue + mean = sum(pos) / len(pos) + sd = st.pstdev(pos) if len(pos) > 1 else 0.0 + beta1 = [v ** 1.0 for v in pos] + per_class.append({ + "class_id": j, + "n_contrib": len(pos), + "cv": (sd / mean) if mean else None, + "max_share": max(beta1) / sum(beta1) if sum(beta1) else None, + "min_pos": min(pos), + "max_pos": max(pos), + "max_over_min": (max(pos) / min(pos)) if min(pos) else None, + }) + cvs = [p["cv"] for p in per_class if p["cv"] is not None] + return { + "client_sizes": [sum(row) for row in support], + "mean_support_cv": st.fmean(cvs) if cvs else None, + "max_support_cv": max(cvs) if cvs else None, + "n_classes_lt_M": sum(1 for p in per_class if p["n_contrib"] < n_clients), + "n_classes_singleton": sum(1 for p in per_class if p["n_contrib"] == 1), + "max_weight_share_beta1": max( + (p["max_share"] for p in per_class if p["max_share"] is not None), default=None), + "mean_weight_share_beta1": st.fmean( + [p["max_share"] for p in per_class if p["max_share"] is not None]) + if any(p["max_share"] is not None for p in per_class) else None, + "per_class": per_class, + } + + +def main() -> int: + runs = {} + for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS): + ref = load_run(dataset, alpha, seed, None) + if ref is None: + print(f"MISSING original {dataset} {alpha} {seed}") + continue + class_names = ref["metadata"]["class_names"] + runs[(dataset, alpha, seed, None)] = ref + for beta in BETAS: + r = load_run(dataset, alpha, seed, beta) + if r is None: + print(f"MISSING beta {dataset} {alpha} {seed} {beta}") + continue + runs[(dataset, alpha, seed, beta)] = r + + # ---------- per-run metric table ---------- + rows = [] + for (dataset, alpha, seed, beta), rec in runs.items(): + vs = rec.get("val_selection") or {} + vg = rec.get("val_global_metrics") or {} + tg = rec.get("test_global_metrics") or {} + md = rec.get("metadata") or {} + class_names = md.get("class_names") or [] + ss = support_stats(rec, class_names) + rows.append({ + "dataset": dataset, "alpha": alpha, "seed": seed, + "mode": rec["mode"], "beta": "" if beta is None else beta, + "val_sel_per_client_rare_f1": vs.get("val_rare_class_macro_f1_per_client_mean"), + "val_rare_f1_global": vg.get("rare_class_macro_f1"), + "val_rare_recall_global": vg.get("rare_class_recall"), + "val_macro_f1": vg.get("macro_f1"), + "val_accuracy": vg.get("accuracy"), + "val_weighted_f1": vg.get("weighted_f1"), + "test_rare_f1_global": tg.get("rare_class_macro_f1"), + "test_rare_recall_global": tg.get("rare_class_recall"), + "test_macro_f1": tg.get("macro_f1"), + "test_accuracy": tg.get("accuracy"), + "test_weighted_f1": tg.get("weighted_f1"), + "rare_support_test": tg.get("rare_support"), + "val_rare_support": vg.get("rare_support"), + "mean_support_cv": ss.get("mean_support_cv"), + "max_support_cv": ss.get("max_support_cv"), + "n_classes_lt_M": ss.get("n_classes_lt_M"), + "n_classes_singleton": ss.get("n_classes_singleton"), + "max_weight_share_beta1": ss.get("max_weight_share_beta1"), + "mean_weight_share_beta1": ss.get("mean_weight_share_beta1"), + "client_size_ratio_mean": ( + st.fmean(ss["client_sizes"]) / (sum(ss["client_sizes"]) / len(ss["client_sizes"])) + if ss.get("client_sizes") else None), + }) + # client size ratio proper: mean client size / smallest? Use max/mean computed below. + for r in rows: + rec = runs[(r["dataset"], r["alpha"], r["seed"], + None if r["mode"] == "original" else r["beta"])] + ss = support_stats(rec, (rec["metadata"] or {}).get("class_names") or []) + sizes = ss.get("client_sizes") + r["client_size_ratio_mean"] = (max(sizes) / st.fmean(sizes)) if sizes else None + + field_order = list(rows[0].keys()) + with (OUT / "phase2_per_run.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=field_order) + w.writeheader() + w.writerows(rows) + + # ---------- paired deltas vs original and vs beta=0 ---------- + by_key = {(r["dataset"], r["alpha"], r["seed"], r["mode"], r["beta"]): r for r in rows} + deltas = [] + for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS): + base = by_key.get((dataset, alpha, seed, "original", "")) + b0 = by_key.get((dataset, alpha, seed, "support_weighted", 0.0)) + if not base: + continue + for beta in BETAS: + cur = by_key.get((dataset, alpha, seed, "support_weighted", beta)) + if not cur: + continue + for refname, ref in (("vs_original", base), ("vs_beta0", b0)): + if not ref: + continue + deltas.append({ + "dataset": dataset, "alpha": alpha, "seed": seed, "beta": beta, + "reference": refname, + "d_val_sel": (cur["val_sel_per_client_rare_f1"] or 0) - (ref["val_sel_per_client_rare_f1"] or 0), + "d_val_rare_f1_global": (cur["val_rare_f1_global"] or 0) - (ref["val_rare_f1_global"] or 0), + "d_test_rare_f1": (cur["test_rare_f1_global"] or 0) - (ref["test_rare_f1_global"] or 0), + "d_val_rare_recall": (cur["val_rare_recall_global"] or 0) - (ref["val_rare_recall_global"] or 0), + "d_test_rare_recall": (cur["test_rare_recall_global"] or 0) - (ref["test_rare_recall_global"] or 0), + "d_val_macro_f1": (cur["val_macro_f1"] or 0) - (ref["val_macro_f1"] or 0), + "d_test_macro_f1": (cur["test_macro_f1"] or 0) - (ref["test_macro_f1"] or 0), + "d_val_accuracy": (cur["val_accuracy"] or 0) - (ref["val_accuracy"] or 0), + "d_test_accuracy": (cur["test_accuracy"] or 0) - (ref["test_accuracy"] or 0), + "val_test_gap_cur": (cur["test_rare_f1_global"] or 0) - (cur["val_rare_f1_global"] or 0), + "val_test_gap_ref": (ref["test_rare_f1_global"] or 0) - (ref["val_rare_f1_global"] or 0), + }) + with (OUT / "phase2_paired_deltas.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(deltas[0].keys())) + w.writeheader() + w.writerows(deltas) + + # ---------- pooled per condition x beta ---------- + pooled = [] + for dataset, alpha in product(DATASETS, ALPHAS): + for beta in [None] + BETAS: + vals = [] + for seed in SEEDS: + r = by_key.get((dataset, alpha, seed, + "original" if beta is None else "support_weighted", + "" if beta is None else beta)) + if r: + vals.append(r) + if not vals: + continue + def mean(key): + xs = [v[key] for v in vals if v[key] is not None] + return st.fmean(xs) if xs else None + def sd(key): + xs = [v[key] for v in vals if v[key] is not None] + return st.stdev(xs) if len(xs) > 1 else None + pooled.append({ + "dataset": dataset, "alpha": alpha, + "beta": "" if beta is None else beta, + "n_seeds": len(vals), + "val_sel_mean": mean("val_sel_per_client_rare_f1"), + "val_sel_sd": sd("val_sel_per_client_rare_f1"), + "val_rare_f1_global_mean": mean("val_rare_f1_global"), + "val_rare_f1_global_sd": sd("val_rare_f1_global"), + "test_rare_f1_mean": mean("test_rare_f1_global"), + "test_rare_f1_sd": sd("test_rare_f1_global"), + "val_rare_recall_mean": mean("val_rare_recall_global"), + "test_rare_recall_mean": mean("test_rare_recall_global"), + "val_macro_f1_mean": mean("val_macro_f1"), + "test_macro_f1_mean": mean("test_macro_f1"), + "val_accuracy_mean": mean("val_accuracy"), + "test_accuracy_mean": mean("test_accuracy"), + "mean_support_cv": mean("mean_support_cv"), + "mean_weight_share_beta1": mean("mean_weight_share_beta1"), + "n_classes_lt_M": mean("n_classes_lt_M"), + }) + with (OUT / "phase2_pooled_by_beta.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(pooled[0].keys())) + w.writeheader() + w.writerows(pooled) + + # ---------- adjacent beta-step deltas (pooled over seeds) ---------- + steps = [] + for dataset, alpha in product(DATASETS, ALPHAS): + seq = [0.0, 0.25, 0.5, 0.75, 1.0] + for lo, hi in zip(seq, seq[1:]): + dv, dt = [], [] + for seed in SEEDS: + a = by_key.get((dataset, alpha, seed, "support_weighted", lo)) + b = by_key.get((dataset, alpha, seed, "support_weighted", hi)) + if a and b: + dv.append((b["val_sel_per_client_rare_f1"] or 0) - (a["val_sel_per_client_rare_f1"] or 0)) + dt.append((b["test_rare_f1_global"] or 0) - (a["test_rare_f1_global"] or 0)) + steps.append({ + "dataset": dataset, "alpha": alpha, "step": f"{lo}->{hi}", + "mean_d_val_sel": st.fmean(dv) if dv else None, + "mean_d_test_rare_f1": st.fmean(dt) if dt else None, + "per_seed_d_val_sel": "|".join(f"{x:.6f}" for x in dv), + "per_seed_d_test_rare_f1": "|".join(f"{x:.6f}" for x in dt), + "n": len(dv), + }) + with (OUT / "phase2_beta_steps.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(steps[0].keys())) + w.writeheader() + w.writerows(steps) + + # ---------- per-class test/val F1 deltas at beta=1 and selected beta ---------- + # Selected beta per condition = argmax pooled val selection metric. + selected = {} + for dataset, alpha in product(DATASETS, ALPHAS): + cand = [p for p in pooled if p["dataset"] == dataset and p["alpha"] == alpha + and p["beta"] != ""] + if cand: + best = max(cand, key=lambda p: p["val_sel_mean"]) + selected[(dataset, alpha)] = float(best["beta"]) + class_rows = [] + for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS): + base = runs.get((dataset, alpha, seed, None)) + if not base: + continue + base_class = {int(r["class_id"]): r for r in (base.get("test_per_class") or [])} + base_vclass = {int(r["class_id"]): r for r in (base.get("val_per_class") or [])} + for beta in [1.0, selected.get((dataset, alpha))]: + cur = runs.get((dataset, alpha, seed, beta)) + if not cur: + continue + cur_class = {int(r["class_id"]): r for r in (cur.get("test_per_class") or [])} + cur_vclass = {int(r["class_id"]): r for r in (cur.get("val_per_class") or [])} + for cid in sorted(base_class): + b = base_class[cid] + c = cur_class.get(cid) + if not c: + continue + bv = base_vclass.get(cid, {}) + cv = cur_vclass.get(cid, {}) + class_rows.append({ + "dataset": dataset, "alpha": alpha, "seed": seed, "beta": beta, + "class_id": cid, "class_name": b.get("class_name"), + "support": b.get("support"), + "base_f1": fnum(b.get("f1")), "cur_f1": fnum(c.get("f1")), + "delta_f1": (fnum(c.get("f1")) or 0) - (fnum(b.get("f1")) or 0), + "base_recall": fnum(b.get("recall")), "cur_recall": fnum(c.get("recall")), + "delta_recall": (fnum(c.get("recall")) or 0) - (fnum(b.get("recall")) or 0), + "base_val_f1": fnum(bv.get("f1")), "cur_val_f1": fnum(cv.get("f1")), + "delta_val_f1": ((fnum(cv.get("f1")) or 0) - (fnum(bv.get("f1")) or 0)) + if bv and cv else None, + }) + with (OUT / "phase2_per_class_deltas.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(class_rows[0].keys())) + w.writeheader() + w.writerows(class_rows) + + # pooled per-class delta + class_pooled = [] + keys = sorted({(r["dataset"], r["alpha"], r["beta"], r["class_id"]) for r in class_rows}) + for dataset, alpha, beta, cid in keys: + sub = [r for r in class_rows if r["dataset"] == dataset and r["alpha"] == alpha + and r["beta"] == beta and r["class_id"] == cid] + class_pooled.append({ + "dataset": dataset, "alpha": alpha, "beta": beta, "class_id": cid, + "class_name": sub[0]["class_name"], "n_seeds": len(sub), + "mean_delta_test_f1": st.fmean([r["delta_f1"] for r in sub]), + "mean_delta_test_recall": st.fmean([r["delta_recall"] for r in sub]), + "mean_delta_val_f1": st.fmean([r["delta_val_f1"] for r in sub + if r["delta_val_f1"] is not None]) + if any(r["delta_val_f1"] is not None for r in sub) else None, + "mean_test_support": st.fmean([float(r["support"]) for r in sub]), + }) + with (OUT / "phase2_per_class_pooled.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(class_pooled[0].keys())) + w.writeheader() + w.writerows(class_pooled) + + # ---------- per-client deltas (validation) at beta=1 ---------- + client_rows = [] + for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS): + base = runs.get((dataset, alpha, seed, None)) + cur = runs.get((dataset, alpha, seed, 1.0)) + if not base or not cur: + continue + bmap = {int(r["client_id"]): r for r in (base.get("val_per_client") or [])} + for r in cur.get("val_per_client") or []: + cid = int(r["client_id"]) + b = bmap.get(cid) + if not b: + continue + bv = fnum(b.get("rare_class_macro_f1")) + cv = fnum(r.get("rare_class_macro_f1")) + client_rows.append({ + "dataset": dataset, "alpha": alpha, "seed": seed, "client_id": cid, + "n_samples": r.get("n_samples"), + "base_val_rare_f1": bv, "cur_val_rare_f1": cv, + "delta_val_rare_f1": (cv - bv) if bv is not None and cv is not None else None, + "base_val_rare_recall": fnum(b.get("rare_class_recall")), + "cur_val_rare_recall": fnum(r.get("rare_class_recall")), + }) + with (OUT / "phase2_per_client_delta_beta1.csv").open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(client_rows[0].keys())) + w.writeheader() + w.writerows(client_rows) + + # ---------- displacement table ---------- + disp = {} + disp_path = REPO / "artifacts" / "analysis" / "final__prototype_displacement_by_beta.csv" + if disp_path.exists(): + for r in read_csv(disp_path): + disp[(r["dataset"], float(r["alpha"]), float(r["beta"]))] = float(r["mean_displacement"]) + + # ---------- summary JSON ---------- + def cond_mean(sel, key): + xs = [r[key] for r in rows if sel(r) and r[key] is not None] + return st.fmean(xs) if xs else None + + worst = sorted( + [d for d in deltas if d["reference"] == "vs_original"], + key=lambda d: d["d_test_rare_f1"])[:10] + summary = { + "n_runs_loaded": len(runs), + "selected_beta_by_condition": {f"{k[0]}__a{k[1]}": v for k, v in selected.items()}, + "pooled_pooled": pooled, + "largest_test_losses_vs_original": worst, + "displacement_beta1": {f"{k[0]}__a{k[1]}": v for k, v in disp.items() if k[2] == 1.0}, + } + (OUT / "phase2_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + + print(json.dumps({ + "runs": len(runs), + "rows": len(rows), + "deltas": len(deltas), + "selected": summary["selected_beta_by_condition"], + }, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/pool_driver.py b/exploration_v2/scripts/pool_driver.py new file mode 100644 index 0000000..a7c02ef --- /dev/null +++ b/exploration_v2/scripts/pool_driver.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +"""Unattended driver for exploration_v2. + +Phases: smoke -> screening -> rank (freeze) -> confirm -> analysis -> DONE. + +Safety properties +----------------- +* one driver at a time (flock held by driver.sh) +* resumable from filesystem markers (COMPLETE / .failed / .running / scopes) +* retries a failed run once, from a clean directory +* concurrency <= 3 processes and <= 36 GiB of MemoryMax caps; <= 2 5G-NIDD +* heartbeat written every loop (target: < 1 minute old) +* STOP, failure-rate, budget and repeated-memory-kill stop rules from PLAN.md +* never kills a process it did not start; on resume it only observes scopes +""" + +from __future__ import annotations + +import csv +import fcntl +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +EXPL = Path(__file__).resolve().parents[1] +RUNS = EXPL / "results" / "runs" +ATTEMPTS = EXPL / "results" / "attempts" +LOGS = EXPL / "logs" +STATE = EXPL / "state.json" +HEARTBEAT = EXPL / "HEARTBEAT" +STOP = EXPL / "STOP" +DONE = EXPL / "DONE" +STOP_REASON = EXPL / "STOP_REASON" +ISSUES = EXPL / "ISSUES.md" +BUDGET_START = EXPL / "BUDGET_START" +RUN_ONE = EXPL / "run_one_v2.sh" +PY = REPO / ".venv" / "bin" / "python" + +MEM_CAP = {"xiiotid": 13, "5gnidd": 15} # GiB; X-IIoTID raised 11->13 after OOMs +MEM_TOTAL = 36 +MAX_PROCS = 2 +MAX_5GNIDD = 2 +MIN_AVAILABLE_GIB = 12.0 # host-level guard before launching +BUDGET_HOURS = 96 + + +def utc(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def now_epoch(): + return time.time() + + +def unit_for(run_id: str) -> str: + h = hashlib.sha1(run_id.encode()).hexdigest()[:12] + return f"explorev2-{h}" + + +def scope_active(unit: str) -> bool: + try: + r = subprocess.run(["systemctl", "--user", "is-active", "--quiet", f"{unit}.scope"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20) + return r.returncode == 0 + except Exception: + return False + + +def read_csv(path: Path): + if not path.is_file(): + return [] + with path.open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def load_state(): + if STATE.is_file(): + try: + return json.loads(STATE.read_text(encoding="utf-8")) + except Exception: + pass + return {"phase": "smoke", "attempts": {}, "terminal_failed": [], + "max_procs": MAX_PROCS, "notes": [], "budget_start_utc": None, + "screening_started": False, "ranking_frozen": False} + + +def save_state(st): + tmp = STATE.with_suffix(".tmp") + tmp.write_text(json.dumps(st, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(tmp, STATE) + + +def append_issue(msg: str): + with ISSUES.open("a", encoding="utf-8") as fh: + fh.write(f"\n## {utc()} — driver\n\n{msg}\n") + + +def run_dir(run_id: str) -> Path: + return RUNS / run_id + + +def primary_ok(rid: str) -> bool: + d = run_dir(rid) + res = d / "results" / rid + ana = d / "analysis" / f"{rid}__prototype_snapshot.json" + need = [ + res / "metadata.json", res / "history.json", + res / "test_global_metrics.json", res / "test_confusion_matrix.json", + res / "test_per_class.csv", res / "val_global_metrics.json", + res / "val_selection.json", res / "per_client.csv", + res / "val_per_client.csv", res / "train_support_Nij.csv", + res / "characterization.csv", res / "rare_unseen.csv", ana, + ] + return all(p.is_file() and p.stat().st_size > 0 for p in need) + + +def status_of(rid: str): + d = run_dir(rid) + if (d / "COMPLETE").is_file() and primary_ok(rid): + return "done" + if (d / ".failed").is_file(): + return "failed" + if (d / ".running").is_file() or scope_active(unit_for(rid)): + return "running" + if d.is_dir(): + return "incomplete" + return "queued" + + +def archive_attempt(rid: str, n: int): + src = run_dir(rid) + if not src.exists(): + return + ATTEMPTS.mkdir(parents=True, exist_ok=True) + k = n + dst = ATTEMPTS / f"{rid}__attempt{k}" + while dst.exists(): + k += 1 + dst = ATTEMPTS / f"{rid}__attempt{k}" + shutil.move(str(src), str(dst)) + + +def terminal(st: dict, rid: str) -> bool: + return status_of(rid) == "done" or rid in st["terminal_failed"] + + +def any_memory_kill_recent(st: dict, window: int = 2) -> bool: + recent = st.get("recent_exits", [])[-window:] + return len(recent) == window and all(x == 137 for x in recent) + + +def write_heartbeat(st: dict, running: dict, phase: str, note: str = ""): + HEARTBEAT.write_text(json.dumps({ + "utc": utc(), + "epoch": round(now_epoch(), 1), + "pid": os.getpid(), + "phase": phase, + "running": sorted(running), + "running_count": len(running), + "max_procs": st.get("max_procs", MAX_PROCS), + "note": note, + }, indent=2) + "\n", encoding="utf-8") + + +def launch(row: dict) -> subprocess.Popen: + cmd = [ + "bash", str(RUN_ONE), + row["run_id"], row["dataset"], str(row["alpha"]), str(row["seed"]), + row["mode"], str(row["beta"] or "0"), + str(row["shrink"] or "0"), str(row["cap"] or "0"), + str(row["minsup"] or "0"), row["part_dir"], row["label"], + ] + LOGS.mkdir(parents=True, exist_ok=True) + logf = LOGS / f"{row['run_id']}.driver.log" + with logf.open("a") as fh: + fh.write(f"[{utc()}] launch: {' '.join(cmd)}\n") + fh.flush() + return subprocess.Popen(cmd, stdout=fh, stderr=subprocess.STDOUT, + cwd=str(REPO), start_new_session=True) + + +def mem_available_gib() -> float: + try: + with open("/proc/meminfo", encoding="ascii") as fh: + for line in fh: + if line.startswith("MemAvailable:"): + return int(line.split()[1]) / (1024 * 1024) + except Exception: + return 0.0 + return 0.0 + + +def can_launch(st: dict, running: dict, row: dict) -> bool: + if mem_available_gib() < MIN_AVAILABLE_GIB: + return False + if len(running) >= int(st.get("max_procs", MAX_PROCS)): + return False + ds = row["dataset"] + used = sum(MEM_CAP.get(r["dataset"], 11) for r in running.values()) + if used + MEM_CAP.get(ds, 11) > MEM_TOTAL: + return False + if ds == "5gnidd": + if sum(1 for r in running.values() if r["dataset"] == "5gnidd") >= MAX_5GNIDD: + return False + return True + + +def prepare_retry(st: dict, rid: str) -> bool: + """Archive a failed/incomplete attempt. Return True if a retry is allowed.""" + n = int(st["attempts"].get(rid, 0)) + if n >= 1: + if rid not in st["terminal_failed"]: + st["terminal_failed"].append(rid) + append_issue(f"run `{rid}` failed after {n + 1} attempt(s); marked terminal failed.") + return False + st["attempts"][rid] = n + 1 + archive_attempt(rid, n + 1) + save_state(st) + return True + + +def run_script(name: str, *args) -> int: + cmd = [str(PY), str(EXPL / "scripts" / name), *args] + logf = LOGS / f"{name}.log" + with logf.open("a") as fh: + fh.write(f"\n[{utc()}] {' '.join(cmd)}\n") + fh.flush() + r = subprocess.run(cmd, stdout=fh, stderr=subprocess.STDOUT, cwd=str(REPO)) + return r.returncode + + +def main() -> int: + EXPL.mkdir(parents=True, exist_ok=True) + st = load_state() + if STOP.is_file(): + write_heartbeat(st, {}, "stopped", "STOP exists at startup") + append_issue("STOP exists at driver startup; no run launched.") + return 0 + + # smoke (re-run after the plan hash; non-scientific) + if not (EXPL / "smoke" / "SMOKE.json").is_file() or st.get("phase") == "smoke": + rc = run_script("smoke_test.py") + if rc != 0: + append_issue(f"smoke test failed (rc={rc}); see exploration_v2/smoke/SMOKE.json. Aborting.") + STOP_REASON.write_text(f"smoke failure at {utc()}\n", encoding="utf-8") + return 1 + st["phase"] = "screening" + save_state(st) + + running: dict[str, dict] = {} + procs: dict[str, subprocess.Popen] = {} + last_hb = 0.0 + stall_loops = 0 + + while True: + if time.time() - last_hb >= 15: + write_heartbeat(st, running, st["phase"]) + last_hb = time.time() + + # observe finished children + for rid in list(procs): + p = procs[rid] + rc = p.poll() + if rc is not None: + del procs[rid] + running.pop(rid, None) + st.setdefault("recent_exits", []).append(rc) + st["recent_exits"] = st["recent_exits"][-4:] + save_state(st) + if rc != 0: + append_issue(f"run `{rid}` wrapper exit {rc}; see its logs.") + if rc == 137: + if any_memory_kill_recent(st): + old = int(st.get("max_procs", MAX_PROCS)) + st["max_procs"] = max(1, old - 1) + append_issue(f"two consecutive exit-137 kills; lowering max_procs " + f"from {old} to {st['max_procs']} and continuing.") + save_state(st) + + # observe resumable scopes (driver may have been restarted) + if st["phase"] in ("screening", "confirm"): + queue = (read_csv(EXPL / "queue_screen.csv") if st["phase"] == "screening" + else read_csv(EXPL / "queue_confirm.csv")) + for row in queue: + rid = row["run_id"] + if rid in procs: + continue + if scope_active(unit_for(rid)) and rid not in running: + running[rid] = row + # drop external runs whose scope has ended + for rid in list(running): + if rid in procs: + continue + if not scope_active(unit_for(rid)): + running.pop(rid, None) + + # stop rules + stop_now = False + reason = "" + if STOP.is_file(): + stop_now, reason = True, "STOP file exists" + elif st["phase"] in ("screening", "confirm"): + queue = (read_csv(EXPL / "queue_screen.csv") if st["phase"] == "screening" + else read_csv(EXPL / "queue_confirm.csv")) + finished = [r["run_id"] for r in queue if terminal(st, r["run_id"])] + failed = [r for r in finished if r in st["terminal_failed"]] + if len(finished) >= 5 and len(failed) / len(finished) > 0.20: + if any_memory_kill_recent(st): + append_issue("failure rate would stop the driver, but the last two " + "exits were memory kills; applying the memory throttle " + "instead of a full stop.") + else: + stop_now, reason = True, (f"failure rate {len(failed)}/{len(finished)} " + f"exceeds 20%") + if st.get("budget_start_utc"): + started = datetime.strptime(st["budget_start_utc"], "%Y-%m-%dT%H:%M:%SZ") + elapsed = (datetime.now(timezone.utc) - started.replace(tzinfo=timezone.utc)).total_seconds() + if elapsed > BUDGET_HOURS * 3600: + stop_now, reason = True, f"96 h budget exhausted ({elapsed/3600:.1f} h)" + if stop_now: + append_issue(f"stopping launches: {reason}") + STOP_REASON.write_text(f"{reason} at {utc()}\n", encoding="utf-8") + st["phase"] = "stopped" + save_state(st) + + # phase transitions only when nothing is running + if not running and not procs: + if st["phase"] == "screening": + queue = read_csv(EXPL / "queue_screen.csv") + if all(terminal(st, r["run_id"]) for r in queue): + ready = [r for r in queue if status_of(r["run_id"]) == "done"] + labels = sorted({r["label"] for r in ready + if r["mode"] != "original_protean"}) + if len(labels) < 3: + append_issue(f"only {len(labels)} screening variants completed; " + f"PLAN requires at least 3. Ranking not run.") + st["phase"] = "stopped" + STOP_REASON.write_text(f"fewer than 3 variants at {utc()}\n", + encoding="utf-8") + save_state(st) + continue + rc = run_script("rank_screening.py") + if rc != 0: + append_issue(f"rank_screening.py failed rc={rc}; aborting.") + st["phase"] = "stopped" + save_state(st) + continue + ranking = EXPL / "ranking.json" + h = hashlib.sha256(ranking.read_bytes()).hexdigest() + (EXPL / "ranking.sha256").write_text( + f"{h} ranking.json\n{utc()}\n", encoding="utf-8") + st["ranking_frozen"] = True + rc = run_script("make_confirm_queue.py") + if rc != 0: + append_issue(f"make_confirm_queue.py failed rc={rc}; aborting.") + st["phase"] = "stopped" + save_state(st) + continue + # new manifests for seeds 3,4 + for ds, alpha in (("xiiotid", 0.25), ("5gnidd", 0.5)): + for seed in (3, 4): + rc = run_script("make_manifest_v2.py", "--dataset", ds, + "--alpha", str(alpha), "--seed", str(seed)) + if rc != 0: + append_issue(f"manifest generation failed for {ds} " + f"alpha={alpha} seed={seed} rc={rc}") + st["phase"] = "stopped" + save_state(st) + break + st["phase"] = "confirm" + save_state(st) + continue + elif st["phase"] == "confirm": + queue = read_csv(EXPL / "queue_confirm.csv") + if all(terminal(st, r["run_id"]) for r in queue): + rc = run_script("analyze_results.py") + if rc != 0: + append_issue(f"analyze_results.py failed rc={rc}") + DONE.write_text(json.dumps({ + "status": "complete" if rc == 0 else "analysis_failed", + "utc": utc(), "runs_done": sum(1 for r in queue if status_of(r["run_id"]) == "done"), + "runs_failed": len([r for r in queue if r["run_id"] in st["terminal_failed"]]), + }, indent=2) + "\n", encoding="utf-8") + st["phase"] = "done" + save_state(st) + write_heartbeat(st, running, "done") + return 0 if rc == 0 else 1 + elif st["phase"] in ("stopped", "done"): + if st["phase"] == "stopped": + rc = run_script("analyze_results.py") + DONE.write_text(json.dumps({ + "status": "stopped", "utc": utc(), "analysis_rc": rc, + "reason": (STOP_REASON.read_text(encoding="utf-8").strip() + if STOP_REASON.is_file() else "unknown"), + }, indent=2) + "\n", encoding="utf-8") + st["phase"] = "done" + save_state(st) + write_heartbeat(st, running, "done") + return 0 + return 0 + + # launch queued runs + if st["phase"] in ("screening", "confirm") and not STOP.is_file(): + queue = (read_csv(EXPL / "queue_screen.csv") if st["phase"] == "screening" + else read_csv(EXPL / "queue_confirm.csv")) + launched_any = False + for row in queue: + rid = row["run_id"] + if terminal(st, rid) or rid in running or rid in procs: + continue + s = status_of(rid) + if s == "failed" or s == "incomplete": + if not prepare_retry(st, rid): + continue + if not can_launch(st, running, row): + continue + if not st.get("budget_start_utc"): + st["budget_start_utc"] = utc() + BUDGET_START.write_text(st["budget_start_utc"] + "\n", encoding="utf-8") + save_state(st) + procs[rid] = launch(row) + running[rid] = row + launched_any = True + if len(running) >= int(st.get("max_procs", MAX_PROCS)): + break + if launched_any: + stall_loops = 0 + elif not running: + stall_loops += 1 + if stall_loops >= 6: + append_issue("driver found no launchable and no running runs for 60 s; " + "recording stall and exiting for resume.sh.") + write_heartbeat(st, running, st["phase"], "stalled") + return 2 + time.sleep(10) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/rank_screening.py b/exploration_v2/scripts/rank_screening.py new file mode 100644 index 0000000..519344a --- /dev/null +++ b/exploration_v2/scripts/rank_screening.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python +"""Validation-only screening ranking for exploration_v2. + +Reads ONLY ``val_*`` files. Every opened file is appended to +``exploration_v2/ACCESS_LOG.txt`` with a UTC timestamp. + +Writes ``exploration_v2/ranking.json``. This file is frozen (sha256 + time in +``ranking.sha256``) before any confirmatory run starts. + +Ranking metric (pre-registered in PLAN.md §5): + score(V) = mean over the two screening conditions of + [val_global_rare_f1(V) - val_global_rare_f1(original)] + where val_global_rare_f1 = rare_class_macro_f1 in val_global_metrics.json. +Secondary (sensitivity): the same difference using +val_rare_class_macro_f1_per_client_mean from val_selection.json. +""" + +from __future__ import annotations + +import csv +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +EXPL = REPO / "exploration_v2" +RUNS = EXPL / "results" / "runs" +ACCESS = EXPL / "ACCESS_LOG.txt" +SCREEN = EXPL / "queue_screen.csv" +OUT = EXPL / "ranking.json" + +CONDITIONS = { + "S1": ("xiiotid", 0.25), + "S2": ("5gnidd", 0.5), +} +BETAS = ["0", "0.25", "0.5", "0.75", "1"] + + +def utc(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def log_access(path: Path): + with ACCESS.open("a", encoding="utf-8") as fh: + fh.write(f"{utc()}\t{path}\n") + + +def read_json_val(path: Path): + if not path.is_file(): + return None + log_access(path) + with path.open(encoding="utf-8") as fh: + return json.load(fh) + + +def val_files(base: Path): + """Return the two validation file paths for a run base directory.""" + return (base / "val_global_metrics.json", base / "val_selection.json") + + +def metrics_for(val_global_path: Path, val_selection_path: Path): + vg = read_json_val(val_global_path) + vs = read_json_val(val_selection_path) + if vg is None and vs is None: + return None + return { + "val_global_rare_f1": (vg or {}).get("rare_class_macro_f1"), + "val_global_rare_recall": (vg or {}).get("rare_class_recall"), + "val_global_macro_f1": (vg or {}).get("macro_f1"), + "val_per_client_rare_f1": (vs or {}).get("val_rare_class_macro_f1_per_client_mean"), + "val_sel_file_present": vs is not None, + } + + +def main() -> int: + ACCESS.parent.mkdir(parents=True, exist_ok=True) + with ACCESS.open("a", encoding="utf-8") as fh: + fh.write(f"# ranking.py run at {utc()}\n") + + # ---- original references --------------------------------------------- + # Primary: an original_protean run in the current environment/partition + # (added as amendment A2). Fallback: the restored grid. + original = {} + beta_refs = {} + baseline_source = {} + for cid, (ds, alpha) in CONDITIONS.items(): + base_id = f"{ds}__alpha{alpha:g}__seed0__original_protean" + new_paths = val_files(RUNS / base_id / "results" / base_id) + old_paths = val_files(REPO / "results" / base_id) + if new_paths[0].is_file(): + vg, vs = new_paths + source = "in_environment_seed0" + else: + vg, vs = old_paths + source = "restored_grid_seed0" + m = metrics_for(vg, vs) + if m is None or m["val_global_rare_f1"] is None: + raise SystemExit(f"missing original validation metrics for {cid} ({base_id})") + original[cid] = {"run_id": base_id, "source": source, **m} + baseline_source[cid] = source + beta_refs[cid] = {} + for b in BETAS: + rid = f"{ds}__alpha{alpha:g}__seed0__support_weighted__beta{b}" + # Prefer the current-environment 1-thread sweep (same host and + # library versions as the new runs) when it exists. + sweep_paths = val_files(REPO / "results" / "sweep" / "runs" / rid / "results" / rid) + grid_paths = val_files(REPO / "results" / rid) + if sweep_paths[0].is_file(): + vg, vs = sweep_paths + src = "sweep_seed0_1thread" + else: + vg, vs = grid_paths + src = "restored_grid_seed0" + mm = metrics_for(vg, vs) + if mm is not None: + beta_refs[cid][b] = {"run_id": rid, "source": src, **mm} + + # ---- screening candidates ------------------------------------------- + log_access(SCREEN) + with SCREEN.open(newline="", encoding="utf-8") as fh: + queue = list(csv.DictReader(fh)) + + candidates = {} + for row in queue: + if row["mode"] == "original_protean": + continue + rid = row["run_id"] + vg, vs = val_files(RUNS / rid / "results" / rid) + m = metrics_for(vg, vs) + if m is None or m["val_global_rare_f1"] is None: + raise SystemExit(f"missing validation metrics for screening run {rid}") + candidates.setdefault(row["label"], { + "label": row["label"], "mode": row["mode"], + "params": {"beta": float(row["beta"]), + "shrink_lambda": float(row["shrink"]) if row["shrink"] else None, + "cap_factor": float(row["cap"]) if row["cap"] else None, + "min_support": float(row["minsup"]) if row["minsup"] else None}, + "conditions": {}, + }) + candidates[row["label"]]["conditions"][ + "S1" if row["dataset"] == "xiiotid" else "S2"] = {"run_id": rid, **m} + + ranked = [] + for label, item in candidates.items(): + per = {} + deltas_global, deltas_client = [], [] + for cid in CONDITIONS: + cond = item["conditions"].get(cid) + if cond is None: + raise SystemExit(f"{label}: missing condition {cid}") + base = original[cid] + dg = cond["val_global_rare_f1"] - base["val_global_rare_f1"] + dc = cond["val_per_client_rare_f1"] - base["val_per_client_rare_f1"] + per[cid] = { + "run_id": cond["run_id"], + "val_global_rare_f1": cond["val_global_rare_f1"], + "baseline_val_global_rare_f1": base["val_global_rare_f1"], + "delta_vs_original": dg, + "val_per_client_rare_f1": cond["val_per_client_rare_f1"], + "baseline_val_per_client_rare_f1": base["val_per_client_rare_f1"], + "delta_per_client_vs_original": dc, + } + deltas_global.append(dg) + deltas_client.append(dc) + item["per_condition"] = per + item["score"] = sum(deltas_global) / len(deltas_global) + item["min_condition_delta"] = min(deltas_global) + item["secondary_score_per_client"] = sum(deltas_client) / len(deltas_client) + ranked.append(item) + + ranked.sort(key=lambda x: (-x["score"], -x["min_condition_delta"], x["label"])) + secondary = sorted(ranked, key=lambda x: (-x["secondary_score_per_client"], + x["label"])) + for i, item in enumerate(ranked, 1): + item["rank"] = i + for i, item in enumerate(secondary, 1): + item["rank_secondary"] = i + + # ---- current-method reference per condition ------------------------ + current_ref = {} + for cid in CONDITIONS: + by_global = max(beta_refs[cid].items(), + key=lambda kv: kv[1]["val_global_rare_f1"]) + by_client = max(beta_refs[cid].items(), + key=lambda kv: kv[1]["val_per_client_rare_f1"]) + current_ref[cid] = { + "original": original[cid], + "best_beta_by_val_global": { + "beta": float(by_global[0]), + "val_global_rare_f1": by_global[1]["val_global_rare_f1"], + "delta_vs_original": (by_global[1]["val_global_rare_f1"] + - original[cid]["val_global_rare_f1"]), + }, + "best_beta_by_val_per_client": { + "beta": float(by_client[0]), + "val_per_client_rare_f1": by_client[1]["val_per_client_rare_f1"], + "delta_vs_original": (by_client[1]["val_per_client_rare_f1"] + - original[cid]["val_per_client_rare_f1"]), + }, + "all_betas": beta_refs[cid], + } + + top2 = [ranked[0]["label"], ranked[1]["label"]] + payload = { + "generated_utc": utc(), + "stage": "screening", + "ranking_metric": "mean over S1,S2 of val_global_rare_f1(candidate) - " + "val_global_rare_f1(original)", + "secondary_metric": "same with val_rare_class_macro_f1_per_client_mean", + "conditions": {k: {"dataset": v[0], "alpha": v[1]} for k, v in CONDITIONS.items()}, + "screening_seed": 0, + "baseline_source": baseline_source, + "original": original, + "current_method_reference": current_ref, + "candidates": ranked, + "secondary_ranking_order": [x["label"] for x in secondary], + "top2": top2, + "selection_margin_top1_minus_top2": ranked[0]["score"] - ranked[1]["score"], + "note": "Validation only. No test metric of any new run was read before " + "this file was frozen.", + } + OUT.write_text(json.dumps(payload, indent=2, sort_keys=False) + "\n", + encoding="utf-8") + print(json.dumps({"top2": top2, + "ranking": [(x["label"], round(x["score"], 6)) for x in ranked], + "margin": payload["selection_margin_top1_minus_top2"]}, + indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/run_record_v2.py b/exploration_v2/scripts/run_record_v2.py new file mode 100755 index 0000000..b62d6e3 --- /dev/null +++ b/exploration_v2/scripts/run_record_v2.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Record one sweep run in summary.csv and write/validate COMPLETE markers. + +Two subcommands: + + record -- upsert one row in results/sweep/summary.csv (flock-protected) + marker -- validate the primary artifacts of a finished run and write COMPLETE + +The summary is keyed by run config, so re-running a failed config replaces its +row instead of appending a duplicate. A completed row is never downgraded to +"skipped_complete" or "stopped". +""" + +from __future__ import annotations + +import argparse +import csv +import fcntl +import hashlib +import json +import os +import re +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +HEADER = [ + "config", + "dataset", + "alpha", + "beta", + "seed", + "aggregation_mode", + "status", + "wall_s", + "cpu_user_s", + "cpu_sys_s", + "percent_cpu", + "peak_rss_kb", + "exit_code", + "started_utc", + "ended_utc", + "run_dir", + "log_dir", + "notes", +] + + +def utcnow() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _elapsed_to_seconds(value: str) -> float | None: + value = value.strip() + parts = value.split(":") + try: + if len(parts) == 3: + return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) + if len(parts) == 2: + return int(parts[0]) * 60 + float(parts[1]) + if len(parts) == 1: + return float(parts[0]) + except ValueError: + return None + return None + + +def parse_time_file(path: str | None) -> dict: + out = { + "wall_s": "", + "cpu_user_s": "", + "cpu_sys_s": "", + "percent_cpu": "", + "peak_rss_kb": "", + } + if not path: + return out + p = Path(path) + if not p.is_file() or p.stat().st_size == 0: + return out + text = p.read_text(encoding="utf-8", errors="replace") + + patterns = { + "cpu_user_s": r"User time \(seconds\):\s*([0-9.]+)", + "cpu_sys_s": r"System time \(seconds\):\s*([0-9.]+)", + "percent_cpu": r"Percent of CPU this job got:\s*([0-9.]+)%", + "peak_rss_kb": r"Maximum resident set size \(kbytes\):\s*([0-9]+)", + } + for key, pat in patterns.items(): + m = re.search(pat, text) + if m: + out[key] = m.group(1) + + m = re.search(r"Elapsed \(wall clock\) time \(h:mm:ss or m:ss\):\s*(\S+)", text) + if m: + wall = _elapsed_to_seconds(m.group(1)) + if wall is not None: + out["wall_s"] = f"{wall:.2f}" + return out + + +def _read_rows(path: Path) -> list[dict]: + if not path.is_file() or path.stat().st_size == 0: + return [] + with path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + for row in rows: + for key in HEADER: + row.setdefault(key, "") + return rows + + +def _write_rows(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=str(path.parent), prefix=".summary.", suffix=".csv", text=True + ) + try: + with os.fdopen(fd, "w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=HEADER) + writer.writeheader() + for row in rows: + writer.writerow({key: row.get(key, "") for key in HEADER}) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def upsert(summary: Path, lock: Path, row: dict) -> None: + summary.parent.mkdir(parents=True, exist_ok=True) + with lock.open("a+", encoding="utf-8") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + rows = _read_rows(summary) + index = next( + (i for i, item in enumerate(rows) if item.get("config") == row["config"]), + None, + ) + if index is None: + rows.append({key: row.get(key, "") for key in HEADER}) + else: + old = rows[index] + if old.get("status") == "completed" and row.get("status") in ( + "skipped_complete", + "stopped", + ): + pass # never downgrade a completed run + elif row.get("status") == "completed": + # A fresh completion replaces the whole row, clearing stale + # failure notes from an earlier attempt. + rows[index] = {key: row.get(key, "") for key in HEADER} + else: + for key in HEADER: + value = row.get(key, "") + if value not in ("", None) or key in ("status", "exit_code"): + old[key] = value + rows[index] = old + _write_rows(summary, rows) + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def required_files(run_dir: Path, run_name: str) -> list[Path]: + results = run_dir / "results" / run_name + analysis = run_dir / "analysis" + return [ + results / "metadata.json", + results / "history.json", + results / "test_global_metrics.json", + results / "test_confusion_matrix.json", + results / "test_per_class.csv", + results / "val_global_metrics.json", + results / "val_selection.json", + results / "val_per_class.csv", + results / "per_client.csv", + results / "val_per_client.csv", + results / "train_support_Nij.csv", + results / "characterization.csv", + results / "rare_unseen.csv", + analysis / f"{run_name}__prototype_snapshot.json", + ] + + +def cmd_record(args: argparse.Namespace) -> int: + timing = parse_time_file(args.time_file) + row = { + "config": args.run_name, + "dataset": args.dataset, + "alpha": args.alpha, + "beta": args.beta, + "seed": str(args.seed), + "aggregation_mode": args.aggregation_mode, + "status": args.status, + "wall_s": timing["wall_s"], + "cpu_user_s": timing["cpu_user_s"], + "cpu_sys_s": timing["cpu_sys_s"], + "percent_cpu": timing["percent_cpu"], + "peak_rss_kb": timing["peak_rss_kb"], + "exit_code": str(args.exit_code), + "started_utc": args.started or "", + "ended_utc": args.ended or utcnow(), + "run_dir": str(Path(args.run_dir)), + "log_dir": str(Path(args.log_dir)), + "notes": args.notes or "", + } + upsert(Path(args.summary), Path(args.lock), row) + return 0 + + +def cmd_marker(args: argparse.Namespace) -> int: + run_dir = Path(args.run_dir) + missing = [ + str(path) + for path in required_files(run_dir, args.run_name) + if not path.is_file() or path.stat().st_size == 0 + ] + if missing: + print("refusing to write COMPLETE; missing/empty artifacts:", file=sys.stderr) + for path in missing: + print(f" {path}", file=sys.stderr) + return 2 + + timing = parse_time_file(args.time_file) + primary = {} + for path in required_files(run_dir, args.run_name): + primary[str(path.relative_to(run_dir))] = sha256_file(path) + + payload = { + "run_name": args.run_name, + "dataset": args.dataset, + "alpha": args.alpha, + "beta": args.beta, + "seed": args.seed, + "aggregation_mode": args.aggregation_mode, + "exit_code": args.exit_code, + "finished_utc": args.finished or utcnow(), + "wall_s": timing["wall_s"], + "cpu_user_s": timing["cpu_user_s"], + "cpu_sys_s": timing["cpu_sys_s"], + "percent_cpu": timing["percent_cpu"], + "peak_rss_kb": timing["peak_rss_kb"], + "required_files_sha256": primary, + } + marker = run_dir / "COMPLETE" + marker.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"wrote {marker}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + rec = sub.add_parser("record") + rec.add_argument("--summary", required=True) + rec.add_argument("--lock", required=True) + rec.add_argument("--run-name", required=True) + rec.add_argument("--dataset", required=True) + rec.add_argument("--alpha", required=True) + rec.add_argument("--beta", required=True) + rec.add_argument("--seed", type=int, default=0) + rec.add_argument("--aggregation-mode", default="unknown") + rec.add_argument("--status", required=True) + rec.add_argument("--exit-code", type=int, required=True) + rec.add_argument("--time-file", default=None) + rec.add_argument("--started", default=None) + rec.add_argument("--ended", default=None) + rec.add_argument("--run-dir", required=True) + rec.add_argument("--log-dir", required=True) + rec.add_argument("--notes", default=None) + rec.set_defaults(func=cmd_record) + + mk = sub.add_parser("marker") + mk.add_argument("--run-dir", required=True) + mk.add_argument("--run-name", required=True) + mk.add_argument("--dataset", required=True) + mk.add_argument("--alpha", required=True) + mk.add_argument("--beta", required=True) + mk.add_argument("--seed", type=int, default=0) + mk.add_argument("--aggregation-mode", default="unknown") + mk.add_argument("--exit-code", type=int, required=True) + mk.add_argument("--time-file", default=None) + mk.add_argument("--finished", default=None) + mk.set_defaults(func=cmd_marker) + + return parser + + +def main(argv=None) -> int: + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/run_variant.py b/exploration_v2/scripts/run_variant.py new file mode 100644 index 0000000..9a033d7 --- /dev/null +++ b/exploration_v2/scripts/run_variant.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python +"""exploration_v2 copy of scripts/run_baseline.py with aggregation variants. + +This file is a *copy*; the original ``scripts/run_baseline.py`` is untouched. +Only three things differ: + +1. the repository root is resolved two levels up (this copy lives in + ``exploration_v2/scripts/``); +2. ``--aggregation-mode`` also accepts the exploration variants defined in + ``exploration_v2/scripts/variants.py``, and their hyperparameters are + explicit CLI flags; +3. the server prototype aggregator is monkey-patched to the variant dispatcher + for the duration of this process. The baseline and ``support_weighted`` + modes are delegated to the unmodified project implementation. + +Every default output directory points inside ``exploration_v2/`` so that a +missing flag can never write into the tracked ``results/`` or +``artifacts/analysis/`` trees. + +Usage (smoke):: + + python exploration_v2/scripts/run_variant.py --dataset xiiotid --alpha 0.5 \ + --seed 0 --max-rows 4000 --clients 4 --rounds 2 --local-epochs 1 +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from protean_research.data import manifest as manifest_mod # noqa: E402 +from protean_research.data import partition as partition_mod # noqa: E402 +from protean_research.data import rare_unseen # noqa: E402 +from protean_research.aggregation import validate_beta # noqa: E402 +from protean_research.data.characterization import characterize_support # noqa: E402 +from protean_research.data.pipeline import ( # noqa: E402 + create_partition, + prepare_dataset, + preprocessing_content_hash, +) +from protean_research.evaluation.metrics import ( # noqa: E402 + classification_metrics, + rare_class_metrics, +) +from protean_research.federated import TrainingConfig, run_federated # noqa: E402 +from protean_research.models.protean_cnn import extract_embeddings # noqa: E402 +from protean_research.models.prototypes import nearest_prototype_predict # noqa: E402 +from protean_research.utils.config import ( # noqa: E402 + expand_path, + load_datasets_config, + load_yaml, +) + +import variants as variants_mod # noqa: E402 + +MODE_CHOICES = list(variants_mod.ALL_MODES) + + +def parse_args(argv=None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset", required=True) + parser.add_argument("--alpha", type=float, required=True) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--clients", type=int, default=10) + parser.add_argument("--rounds", type=int, default=10) + parser.add_argument("--local-epochs", type=int, default=3) + parser.add_argument("--batch-size", type=int, default=256) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--mu", type=float, default=0.1) + parser.add_argument("--lam", type=float, default=1.0) + parser.add_argument( + "--aggregation-mode", + choices=MODE_CHOICES, + default="original_protean", + help="server-side prototype aggregation mode (default: frozen baseline).", + ) + parser.add_argument( + "--beta", + type=float, + default=None, + help="support-weighting exponent; required for every weighted mode.", + ) + parser.add_argument("--shrink-lambda", type=float, default=0.5, + help="v_shrink: weight on the support-weighted vector (rest goes to equal).") + parser.add_argument("--cap-factor", type=float, default=4.0, + help="v_cap: cap each raw weight at cap_factor * median raw weight.") + parser.add_argument("--min-support", type=float, default=10.0, + help="v_minsup: drop contributors with N_ij below this threshold.") + parser.add_argument("--variant-label", default=None, + help="short label used in the run directory name.") + parser.add_argument( + "--max-rows", + type=int, + default=None, + help="SMOKE ONLY: limit how many raw rows are read; never use for science.", + ) + parser.add_argument("--sample-from", type=int, default=200000, + help="SMOKE ONLY: rows to read before random subsampling.") + parser.add_argument("--partitions-dir", default=None) + parser.add_argument("--processed-dir", default=None) + parser.add_argument("--results-dir", default=None) + parser.add_argument("--analysis-dir", default=None) + return parser.parse_args(argv) + + +def write_json(path: Path, payload) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def write_csv(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8") + return + fieldnames = list(rows[0].keys()) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def main(argv=None) -> int: + args = parse_args(argv) + + if args.aggregation_mode == "original_protean": + if args.beta is not None: + raise SystemExit( + "--beta is only valid with a weighted aggregation mode" + ) + beta = None + else: + if args.beta is None: + raise SystemExit( + "--beta is required with a weighted aggregation mode" + ) + raw_beta = float(args.beta) + if not (raw_beta == raw_beta) or raw_beta in (float("inf"), float("-inf")): + raise SystemExit(f"--beta must be finite, got {args.beta!r}") + if args.aggregation_mode == "v_beta": + if raw_beta < 0.0 or raw_beta > 2.0: + raise SystemExit("v_beta requires 0 <= beta <= 2") + beta = raw_beta + else: + beta = validate_beta(raw_beta) + + if args.shrink_lambda < 0.0 or args.shrink_lambda > 1.0: + raise SystemExit("--shrink-lambda must lie in [0, 1]") + if args.cap_factor < 1.0: + raise SystemExit("--cap-factor must be >= 1") + + # Patch the server aggregator for this process only. The project modules on + # disk are not modified. Baseline and support_weighted delegate inside + # variants.aggregate_variant to the unmodified project implementation. + from protean_research.federated import server as server_mod + + def _variant_aggregator(client_prototypes, client_supports, num_classes, + mode=None, beta=None): + return variants_mod.aggregate_from_client_dicts( + client_prototypes, + client_supports, + num_classes, + mode=mode, + beta=beta, + shrink_lambda=args.shrink_lambda, + cap_factor=args.cap_factor, + min_support=args.min_support, + ) + + server_mod.aggregate_prototypes = _variant_aggregator + + datasets_cfg = load_datasets_config() + protocol = load_yaml(REPO_ROOT / "configs" / "experiments" / "experiment_protocol.yaml") + + artifacts_cfg = protocol.get("artifacts", {}) + # Safe defaults: never write into the tracked trees. Callers pass explicit + # directories for scientific runs. + partitions_dir = expand_path( + args.partitions_dir or REPO_ROOT / "exploration_v2" / "partitions") + processed_dir = expand_path( + args.processed_dir or REPO_ROOT / "exploration_v2" / "processed") + analysis_dir = expand_path( + args.analysis_dir or REPO_ROOT / "exploration_v2" / "analysis") + results_root = expand_path( + args.results_dir or REPO_ROOT / "exploration_v2" / "results") + + # Smoke runs must never pollute the tracked partition/analysis directories. + if args.max_rows is not None: + if args.partitions_dir is None: + partitions_dir = processed_dir / "smoke" / "partitions" + if args.analysis_dir is None: + analysis_dir = processed_dir / "smoke" / "analysis" + if args.results_dir is None: + results_root = results_root / "smoke" + + if args.max_rows is not None: + from protean_research.data import loading as loading_mod + import numpy as np + + spec = loading_mod.load_spec(args.dataset, datasets_cfg) + pool = loading_mod.read_raw(spec, nrows=args.sample_from) + rng = np.random.default_rng(args.seed) + take = min(args.max_rows, len(pool)) + chosen = np.sort(rng.choice(len(pool), size=take, replace=False)) + pool = pool.iloc[chosen].reset_index(drop=True) + bundle = prepare_dataset( + args.dataset, seed=args.seed, datasets_cfg=datasets_cfg, raw_frame=pool + ) + else: + bundle = prepare_dataset(args.dataset, seed=args.seed, datasets_cfg=datasets_cfg) + + # ---- partition (create or reload) ------------------------------------- + pre_hash = preprocessing_content_hash(bundle.preprocessor) + manifest_file = manifest_mod.manifest_path(partitions_dir, args.dataset, args.alpha, args.seed) + if manifest_file.exists(): + manifest = manifest_mod.load_manifest(manifest_file) + split, assignments, support = manifest_mod.reproduce_train_assignments(manifest, bundle.labels) + # Reuse the manifest's split for pairing; verify it matches the bundle. + bundle.split = split + else: + manifest, assignments, support = create_partition( + bundle, alpha=args.alpha, seed=args.seed, num_clients=args.clients, + partition_dir=partitions_dir, preprocessing_hash=pre_hash, + ) + manifest_reload_ok = True + try: + manifest_mod.reproduce_train_assignments(manifest, bundle.labels) + except Exception: # pragma: no cover - surfaced in metadata + manifest_reload_ok = False + + labels = bundle.labels + train_features = bundle.features[bundle.split.train] + train_labels = labels[bundle.split.train] + val_features = bundle.features[bundle.split.validation] + val_labels = labels[bundle.split.validation] + test_features = bundle.features[bundle.split.test] + test_labels = labels[bundle.split.test] + + benign_index = bundle.cleaned.benign_index + per_client_rare = rare_unseen.rare_class_ids(support, benign_index) + union_rare = sorted({c for row in per_client_rare for c in row}) + + cfg = TrainingConfig( + rounds=args.rounds, + local_epochs=args.local_epochs, + batch_size=args.batch_size, + learning_rate=args.lr, + mu=args.mu, + lam=args.lam, + num_clients=args.clients, + aggregation_mode=args.aggregation_mode, + beta=beta, + ) + result = run_federated( + train_features=train_features, + train_labels=train_labels, + assignments=assignments, + val_features=val_features, + val_labels=val_labels, + num_classes=bundle.num_classes, + input_dim=bundle.features.shape[1], + config=cfg, + init_seed_parts=(args.dataset, f"seed={args.seed}"), + rare_class_ids=union_rare, + ) + + model = result["model"] + global_prototypes = result["global_prototypes"] + + # ---- global test evaluation ------------------------------------------ + test_embeddings = extract_embeddings(model, test_features) + test_pred = nearest_prototype_predict(test_embeddings, global_prototypes) + test_metrics = classification_metrics(test_labels, test_pred, bundle.num_classes) + test_rare = rare_class_metrics(test_labels, test_pred, bundle.num_classes, union_rare) + + # ---- per-client evaluation ------------------------------------------- + eval_assignments, eval_rng = partition_mod.partition_eval_split( + test_labels, args.clients, args.alpha, args.dataset, args.seed, "test" + ) + per_client_rows = [] + for client_id in range(args.clients): + idx = eval_assignments[client_id] + if len(idx) == 0: + per_client_rows.append({ + "client_id": client_id, "n_samples": 0, "accuracy": None, + "macro_f1": None, "rare_class_macro_f1": None, "rare_class_recall": None, + }) + continue + emb = extract_embeddings(model, test_features[idx]) + pred = nearest_prototype_predict(emb, global_prototypes) + metrics = classification_metrics(test_labels[idx], pred, bundle.num_classes) + rare_ids = per_client_rare[client_id] + if rare_ids: + rare = rare_class_metrics(test_labels[idx], pred, bundle.num_classes, rare_ids) + else: + rare = {"rare_class_macro_f1": None, "rare_class_recall": None} + per_client_rows.append({ + "client_id": client_id, + "n_samples": int(len(idx)), + "accuracy": metrics["accuracy"], + "macro_f1": metrics["macro_f1"], + "rare_class_macro_f1": rare["rare_class_macro_f1"], + "rare_class_recall": rare["rare_class_recall"], + }) + + # ---- global validation evaluation (SELECTION ONLY) ------------------- + # This block never influences training and is never used to tune any + # hyperparameter except the prototype-aggregation beta, which is selected + # on validation metrics only. The test split is untouched here. + val_embeddings = extract_embeddings(model, val_features) + val_pred = nearest_prototype_predict(val_embeddings, global_prototypes) + val_metrics = classification_metrics(val_labels, val_pred, bundle.num_classes) + val_rare = rare_class_metrics(val_labels, val_pred, bundle.num_classes, union_rare) + + val_eval_assignments, _ = partition_mod.partition_eval_split( + val_labels, args.clients, args.alpha, args.dataset, args.seed, "validation" + ) + val_per_client_rows = [] + for client_id in range(args.clients): + idx = val_eval_assignments[client_id] + if len(idx) == 0: + val_per_client_rows.append({ + "client_id": client_id, "n_samples": 0, "accuracy": None, + "macro_f1": None, "rare_class_macro_f1": None, "rare_class_recall": None, + }) + continue + emb = extract_embeddings(model, val_features[idx]) + pred = nearest_prototype_predict(emb, global_prototypes) + metrics = classification_metrics(val_labels[idx], pred, bundle.num_classes) + rare_ids = per_client_rare[client_id] + if rare_ids: + rare = rare_class_metrics(val_labels[idx], pred, bundle.num_classes, rare_ids) + else: + rare = {"rare_class_macro_f1": None, "rare_class_recall": None} + val_per_client_rows.append({ + "client_id": client_id, + "n_samples": int(len(idx)), + "accuracy": metrics["accuracy"], + "macro_f1": metrics["macro_f1"], + "rare_class_macro_f1": rare["rare_class_macro_f1"], + "rare_class_recall": rare["rare_class_recall"], + }) + + def _mean_over_present(rows, key): + values = [float(row[key]) for row in rows if row.get(key) is not None] + return float(sum(values) / len(values)) if values else None + + val_selection = { + "val_rare_class_macro_f1_global": val_rare["rare_class_macro_f1"], + "val_rare_class_recall_global": val_rare["rare_class_recall"], + "val_macro_f1_global": val_metrics["macro_f1"], + "val_weighted_f1_global": val_metrics["weighted_f1"], + "val_accuracy_global": val_metrics["accuracy"], + "val_rare_class_macro_f1_per_client_mean": _mean_over_present( + val_per_client_rows, "rare_class_macro_f1" + ), + "val_rare_class_recall_per_client_mean": _mean_over_present( + val_per_client_rows, "rare_class_recall" + ), + "val_n_eval_clients": sum(1 for row in val_per_client_rows if row["n_samples"] > 0), + "selection_metric": "rare_class_macro_f1_per_client_mean", + "selection_split": "validation", + "test_used_for_selection": False, + } + + # ---- structured artifacts -------------------------------------------- + mode_tag = args.aggregation_mode + if args.variant_label: + mode_tag = args.variant_label + elif args.aggregation_mode == "support_weighted": + mode_tag = f"support_weighted__beta{beta:g}" + elif args.aggregation_mode != "original_protean": + if args.variant_label: + mode_tag = args.variant_label + elif args.aggregation_mode == "v_beta": + mode_tag = f"v_beta__b{beta:g}" + elif args.aggregation_mode == "v_shrink": + mode_tag = f"v_shrink__b{beta:g}__lam{args.shrink_lambda:g}" + elif args.aggregation_mode == "v_cap": + mode_tag = f"v_cap__b{beta:g}__c{args.cap_factor:g}" + elif args.aggregation_mode == "v_minsup": + mode_tag = f"v_minsup__t{args.min_support:g}" + elif args.aggregation_mode == "v_logweight": + mode_tag = "v_logweight" + else: # pragma: no cover + mode_tag = args.aggregation_mode + run_name = f"{args.dataset}__alpha{args.alpha:g}__seed{args.seed}__{mode_tag}" + run_dir = results_root / run_name + run_dir.mkdir(parents=True, exist_ok=True) + + metadata = { + "run_name": run_name, + "dataset": args.dataset, + "alpha": args.alpha, + "seed": args.seed, + "num_clients": args.clients, + "num_classes": bundle.num_classes, + "class_names": bundle.class_names, + "benign_class": bundle.spec.benign_label, + "aggregation_mode": args.aggregation_mode, + "beta": beta, + "variant": { + "label": mode_tag, + "shrink_lambda": args.shrink_lambda + if args.aggregation_mode == "v_shrink" else None, + "cap_factor": args.cap_factor + if args.aggregation_mode == "v_cap" else None, + "min_support": args.min_support + if args.aggregation_mode == "v_minsup" else None, + }, + "training": { + "rounds": args.rounds, + "local_epochs": args.local_epochs, + "batch_size": args.batch_size, + "learning_rate": args.lr, + "mu": args.mu, + "lambda": args.lam, + }, + "smoke": args.max_rows is not None, + "max_rows": args.max_rows, + "manifest_path": str(manifest_file), + "manifest_content_hash": manifest_mod.manifest_content_hash(manifest), + "preprocessing_hash": pre_hash, + "manifest_reload_ok": manifest_reload_ok, + "repo_root": str(REPO_ROOT), + } + write_json(run_dir / "metadata.json", metadata) + write_json(run_dir / "history.json", result["history"]) + write_json(run_dir / "test_global_metrics.json", { + "accuracy": test_metrics["accuracy"], + "macro_f1": test_metrics["macro_f1"], + "weighted_f1": test_metrics["weighted_f1"], + **test_rare, + }) + write_json(run_dir / "test_confusion_matrix.json", test_metrics["confusion_matrix"]) + write_csv(run_dir / "test_per_class.csv", [ + {**row, "class_name": bundle.class_names[row["class_id"]]} + for row in test_metrics["per_class"] + ]) + write_csv(run_dir / "per_client.csv", per_client_rows) + write_json(run_dir / "val_global_metrics.json", { + "accuracy": val_metrics["accuracy"], + "macro_f1": val_metrics["macro_f1"], + "weighted_f1": val_metrics["weighted_f1"], + **val_rare, + }) + write_json(run_dir / "val_confusion_matrix.json", val_metrics["confusion_matrix"]) + write_csv(run_dir / "val_per_class.csv", [ + {**row, "class_name": bundle.class_names[row["class_id"]]} + for row in val_metrics["per_class"] + ]) + write_csv(run_dir / "val_per_client.csv", val_per_client_rows) + write_json(run_dir / "val_selection.json", val_selection) + write_csv(run_dir / "train_support_Nij.csv", [ + {"client_id": i, **{bundle.class_names[c]: int(support[i, c]) for c in range(bundle.num_classes)}} + for i in range(args.clients) + ]) + write_csv(run_dir / "characterization.csv", characterize_support(support, bundle.class_names)) + write_csv(run_dir / "rare_unseen.csv", rare_unseen.rare_unseen_rows( + support, bundle.class_names, benign_index, args.dataset, args.alpha, args.seed + )) + + processed_dir.mkdir(parents=True, exist_ok=True) + bundle.preprocessor.save(processed_dir / f"{args.dataset}__seed{args.seed}__preprocessor.json") + + snapshot = result["snapshot"] + if snapshot is not None: + write_json(analysis_dir / f"{run_name}__prototype_snapshot.json", snapshot) + + summary = { + "run": run_name, + "test_accuracy": test_metrics["accuracy"], + "test_macro_f1": test_metrics["macro_f1"], + "test_rare_class_macro_f1": test_rare["rare_class_macro_f1"], + "n_global_prototypes": len(global_prototypes), + "history_last": result["history"][-1] if result["history"] else None, + "results_dir": str(run_dir), + } + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/smoke_test.py b/exploration_v2/scripts/smoke_test.py new file mode 100644 index 0000000..eafeb7f --- /dev/null +++ b/exploration_v2/scripts/smoke_test.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +"""Very short smoke slice of every aggregation variant. + +Checks, in order: +1. the exploration unit tests pass (``exploration_v2/tests``); +2. the baseline run completes and writes validation artifacts; +3. ``support_weighted`` beta=0 and ``v_beta`` beta=0 reproduce the baseline + validation metric within a documented smoke tolerance; +4. every candidate variant completes. + +This is a runtime sanity check, not a scientific comparison. It writes only +inside ``exploration_v2/smoke``. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +HERE = REPO / "exploration_v2" +SMOKE = HERE / "smoke" +PY = REPO / ".venv" / "bin" / "python" +RUNNER = HERE / "scripts" / "run_variant.py" + +TOL = 1e-3 + +VARIANTS = [ + # id, mode, beta, extra args, label + ("base", "original_protean", None, [], "smoke_base"), + ("beta0", "support_weighted", 0.0, [], "smoke_beta0"), + ("beta1", "support_weighted", 1.0, [], "smoke_beta1"), + ("v_beta1.5", "v_beta", 1.5, [], "smoke_v_beta1.5"), + ("v_shrink", "v_shrink", 1.0, ["--shrink-lambda", "0.5"], "smoke_v_shrink"), + ("v_cap4", "v_cap", 1.0, ["--cap-factor", "4"], "smoke_v_cap4"), + ("v_cap2", "v_cap", 1.0, ["--cap-factor", "2"], "smoke_v_cap2"), + ("v_minsup", "v_minsup", 0.0, ["--min-support", "10"], "smoke_v_minsup"), + ("v_logweight", "v_logweight", 0.0, [], "smoke_v_logweight"), +] + + +def utc(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def run_one(name, mode, beta, extra, label): + cmd = [ + str(PY), str(RUNNER), + "--dataset", "xiiotid", "--alpha", "0.25", "--seed", "0", + "--max-rows", "3000", "--sample-from", "30000", + "--clients", "4", "--rounds", "2", "--local-epochs", "1", + "--aggregation-mode", mode, "--variant-label", label, + "--partitions-dir", str(SMOKE / "partitions"), + "--processed-dir", str(SMOKE / "processed"), + "--analysis-dir", str(SMOKE / "analysis"), + "--results-dir", str(SMOKE / "results"), + ] + if beta is not None: + cmd[cmd.index("--aggregation-mode"):cmd.index("--aggregation-mode")] = ["--beta", str(beta)] + cmd += extra + env = dict(os.environ) + for k in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", + "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"): + env[k] = "1" + env["PYTHONDONTWRITEBYTECODE"] = "1" + log = SMOKE / "logs" + log.mkdir(parents=True, exist_ok=True) + out = log / f"{name}.stdout.log" + err = log / f"{name}.stderr.log" + with out.open("w") as fo, err.open("w") as fe: + proc = subprocess.run(cmd, stdout=fo, stderr=fe, env=env, timeout=1800) + return proc.returncode, cmd + + +def val_metric(result_dir: Path): + p = result_dir / "val_global_metrics.json" + if not p.exists(): + return None + return float(json.loads(p.read_text())["rare_class_macro_f1"]) + + +def main() -> int: + SMOKE.mkdir(parents=True, exist_ok=True) + report = {"started_utc": utc(), "tolerance": TOL, "runs": {}} + + # 1. unit tests + unittest = subprocess.run( + [str(PY), "-m", "pytest", str(HERE / "tests"), "-q", "-p", "no:cacheprovider"], + cwd=str(REPO), capture_output=True, text=True, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, timeout=600) + report["pytest_returncode"] = unittest.returncode + report["pytest_tail"] = (unittest.stdout + unittest.stderr).strip().splitlines()[-3:] + if unittest.returncode != 0: + report["verdict"] = "FAIL_UNIT_TESTS" + (SMOKE / "SMOKE.json").write_text(json.dumps(report, indent=2)) + return 1 + + metrics = {} + for name, mode, beta, extra, label in VARIANTS: + rc, cmd = run_one(name, mode, beta, extra, label) + rdir = SMOKE / "results" / f"xiiotid__alpha0.25__seed0__{label}" + metric = val_metric(rdir) + metrics[name] = metric + report["runs"][name] = { + "mode": mode, "beta": beta, "extra": extra, + "returncode": rc, "val_rare_macro_f1": metric, + "results_dir": str(rdir.relative_to(REPO)), + } + if rc != 0 or metric is None: + report["verdict"] = f"FAIL_RUN_{name}" + report["finished_utc"] = utc() + (SMOKE / "SMOKE.json").write_text(json.dumps(report, indent=2)) + return 1 + + base = metrics["base"] + checks = {} + for name in ("beta0", "v_beta1.5"): + # v_beta1.5 is not expected to equal baseline; only beta0 equivalence is checked. + if name == "beta0": + checks[name] = abs(metrics[name] - base) + # v_beta at beta=0 is exercised by the unit tests; here we use beta1.5. + checks["beta0_within_tolerance"] = checks["beta0"] <= TOL + report["equivalence_checks"] = checks + if not checks["beta0_within_tolerance"]: + report["verdict"] = "FAIL_BETA0_EQUIVALENCE" + else: + report["verdict"] = "PASS" + report["finished_utc"] = utc() + (SMOKE / "SMOKE.json").write_text(json.dumps(report, indent=2)) + print(json.dumps({"verdict": report["verdict"], + "beta0_abs_diff": checks["beta0"]}, indent=2)) + return 0 if report["verdict"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/status_table.py b/exploration_v2/scripts/status_table.py new file mode 100644 index 0000000..bd8c391 --- /dev/null +++ b/exploration_v2/scripts/status_table.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +"""Fast status table for exploration_v2 (returns immediately).""" + +from __future__ import annotations + +import csv +import json +from datetime import datetime, timezone +from pathlib import Path + +EXPL = Path(__file__).resolve().parents[1] +RUNS = EXPL / "results" / "runs" + + +def read_csv(p): + if not p.is_file(): + return [] + with p.open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def status(rid): + d = RUNS / rid + if (d / "COMPLETE").is_file(): + return "done" + if (d / ".failed").is_file(): + return "failed" + if (d / ".running").is_file(): + return "running" + if d.is_dir(): + return "incomplete" + return "queued" + + +def main(): + st = {} + if (EXPL / "state.json").is_file(): + st = json.loads((EXPL / "state.json").read_text()) + hb = "" + if (EXPL / "HEARTBEAT").is_file(): + try: + hb = json.loads((EXPL / "HEARTBEAT").read_text()) + except Exception: + hb = {} + queue = read_csv(EXPL / "queue_screen.csv") + phase = st.get("phase", "?") + if phase in ("confirm", "done", "stopped"): + queue = read_csv(EXPL / "queue_confirm.csv") or queue + rows = [] + for r in queue: + rows.append((r["run_id"], r["dataset"], r.get("label", ""), status(r["run_id"]))) + counts = {} + for _, _, _, s in rows: + counts[s] = counts.get(s, 0) + 1 + print(f"phase={phase} budget_start={st.get('budget_start_utc')} " + f"max_procs={st.get('max_procs')}") + print(f"heartbeat={(hb or {}).get('utc', 'MISSING')} " + f"running={(hb or {}).get('running_count', 0)} " + f"note={(hb or {}).get('note', '')}") + print(f"terminal_failed={len(st.get('terminal_failed', []))} " + f"DONE={str((EXPL / 'DONE').is_file()).lower()} " + f"STOP={str((EXPL / 'STOP').is_file()).lower()}") + print("counts: " + " ".join(f"{k}={v}" for k, v in sorted(counts.items()))) + print() + print(f"{'run_id':<62} {'dataset':<8} {'label':<28} status") + for rid, ds, label, s in rows: + print(f"{rid:<62} {ds:<8} {label:<28} {s}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/scripts/thread1_launcher.py b/exploration_v2/scripts/thread1_launcher.py new file mode 100755 index 0000000..94a5250 --- /dev/null +++ b/exploration_v2/scripts/thread1_launcher.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Force a single compute thread, then execute scripts/run_baseline.py. + +The sweep design is one process = one thread. Environment variables +(OMP_NUM_THREADS etc.) are set by the caller, but PyTorch's intra-op and +inter-op pools are also pinned explicitly here so that no hidden pool can +create additional worker threads. + +Usage: + python thread1_launcher.py /path/to/scripts/run_baseline.py [args...] +""" + +from __future__ import annotations + +import runpy +import sys + +import torch + +torch.set_num_threads(1) +try: + torch.set_num_interop_threads(1) +except RuntimeError: + # Already initialised (should not happen before any parallel work). + pass + +if len(sys.argv) < 2: + raise SystemExit("usage: thread1_launcher.py SCRIPT [args...]") + +script = sys.argv[1] +sys.argv = [script] + sys.argv[2:] +runpy.run_path(script, run_name="__main__") diff --git a/exploration_v2/scripts/variants.py b/exploration_v2/scripts/variants.py new file mode 100644 index 0000000..c3dcd2f --- /dev/null +++ b/exploration_v2/scripts/variants.py @@ -0,0 +1,161 @@ +"""Candidate server-side prototype aggregation variants (exploration_v2 only). + +Every variant operates on the same contributor set semantics as the frozen +project code: + + S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } + +except ``v_minsup`` which deliberately changes the participant rule and is +labelled as such. Missing classes are never fabricated. + +The baseline (``original_protean``) and the current proposed method +(``support_weighted``) are *delegated* to the unmodified project implementation +so that their behaviour cannot drift. Only genuinely new modes are implemented +here. + +Mode catalogue +-------------- +original_protean equal mean over S_j (delegated, unchanged) +support_weighted w_i proportional to N_ij**beta in [0, 1] (delegated) +v_beta w_i proportional to N_ij**beta, beta allowed in [0, 2] + (extended-grid candidate; beta > 1 is outside the frozen grid) +v_shrink p_i = N_ij**beta / sum(N_ij**beta); + w_i = (1 - lam) / |S_j| + lam * p_i + (shrinkage of the weight vector toward equal weighting) +v_cap w_i proportional to min(N_ij**beta, c * median(N_ij**beta)) + (clipping/capping of the weight vector) +v_minsup drop contributors with N_ij < t, then equal mean over the + remaining contributors (participant-rule change) +v_logweight w_i proportional to log1p(N_ij), normalised + (a different, concave weight function) +""" + +from __future__ import annotations + +import math +from typing import Mapping, Sequence + +import numpy as np + +from protean_research.aggregation import aggregate_class_prototypes + +BASELINE_MODES = ("original_protean", "support_weighted") +VARIANT_MODES = ("v_beta", "v_shrink", "v_cap", "v_minsup", "v_logweight") +ALL_MODES = BASELINE_MODES + VARIANT_MODES + + +def _collect(contributions): + """Return class_id -> (list[vector ndarray], list[support float]).""" + out = {} + for class_id in sorted(contributions, key=lambda c: int(c)): + protos, supports = [], [] + for proto, support in contributions[class_id]: + if proto is None: + continue + if support is None or float(support) <= 0.0: + continue + protos.append(np.asarray(proto, dtype=np.float64)) + supports.append(float(support)) + if protos: + out[int(class_id)] = (protos, supports) + return out + + +def _weighted_mean(protos, weights): + w = np.asarray(weights, dtype=np.float64) + total = float(w.sum()) + if not math.isfinite(total) or total <= 0.0: + raise ValueError("weights must sum to a positive finite value") + w = w / total + mat = np.stack(protos) + return (w[:, None] * mat).sum(axis=0) + + +def _normalised_power(supports, beta): + n = np.asarray(supports, dtype=np.float64) + if np.any(~np.isfinite(n)) or np.any(n <= 0): + raise ValueError("supports must be strictly positive and finite") + w = n ** float(beta) + return w / w.sum() + + +def aggregate_variant( + contributions: Mapping[int, Sequence], + mode, + *, + beta=None, + shrink_lambda: float = 0.5, + cap_factor: float = 4.0, + min_support: float = 10.0, +): + """Aggregate one round's per-class contributions under *mode*. + + ``contributions`` maps ``class_id -> [(C_ij, N_ij), ...]`` exactly like + :func:`protean_research.aggregation.aggregate_class_prototypes`. + """ + if mode in BASELINE_MODES: + # Unchanged project behaviour; beta semantics are also the project's. + return aggregate_class_prototypes(contributions, mode, beta=beta) + + if mode not in VARIANT_MODES: + raise ValueError(f"unknown aggregation mode {mode!r}") + if beta is None: + raise ValueError(f"{mode} requires beta") + beta = float(beta) + if not math.isfinite(beta) or beta < 0.0: + raise ValueError(f"beta must be finite and >= 0, got {beta}") + + collected = _collect(contributions) + result = {} + for class_id, (protos, supports) in collected.items(): + n = np.asarray(supports, dtype=np.float64) + if mode == "v_beta": + if beta > 2.0: + raise ValueError("v_beta beta must be <= 2.0") + w = n ** beta + elif mode == "v_shrink": + if not 0.0 <= shrink_lambda <= 1.0: + raise ValueError("shrink_lambda must lie in [0, 1]") + p = _normalised_power(n, beta) + w = (1.0 - shrink_lambda) / len(protos) + shrink_lambda * p + elif mode == "v_cap": + if cap_factor < 1.0: + raise ValueError("cap_factor must be >= 1") + raw = n ** beta + med = float(np.median(raw)) + w = np.minimum(raw, cap_factor * med) + elif mode == "v_minsup": + keep = n >= float(min_support) + if not np.any(keep): + # Documented fallback: never emit an empty class. If every + # contributor is below the threshold, keep all of them and + # fall back to equal weighting. + keep = np.ones_like(n, dtype=bool) + sub = [p for p, k in zip(protos, keep) if k] + w = np.ones(len(sub), dtype=np.float64) + protos = sub + elif mode == "v_logweight": + w = np.log1p(n) + else: # pragma: no cover + raise AssertionError(mode) + if not np.any(w > 0): + w = np.ones(len(protos), dtype=np.float64) + result[int(class_id)] = _weighted_mean(protos, w).astype(np.float64).tolist() + return result + + +def aggregate_from_client_dicts( + client_prototypes, + client_supports, + num_classes: int, + mode, + **kwargs, +): + """Adapter with the same signature as the project server aggregator.""" + contributions = {c: [] for c in range(num_classes)} + for prototypes, supports in zip(client_prototypes, client_supports): + for class_id, proto in prototypes.items(): + contributions[int(class_id)].append( + (proto, supports.get(int(class_id), 0)) + ) + return aggregate_variant(contributions, mode, **kwargs) diff --git a/exploration_v2/scripts/verify_setup.py b/exploration_v2/scripts/verify_setup.py new file mode 100644 index 0000000..b1530d3 --- /dev/null +++ b/exploration_v2/scripts/verify_setup.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +"""Read-only pre-flight/verification checks for exploration_v2.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +EXPL = REPO / "exploration_v2" + + +def sh(cmd): + r = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True) + return r.returncode, r.stdout.strip(), r.stderr.strip() + + +def main() -> int: + checks = [] + + def add(name, ok, detail=""): + checks.append({"check": name, "ok": bool(ok), "detail": detail}) + + plan = (EXPL / "PLAN.md").read_bytes() + h = hashlib.sha256(plan).hexdigest() + rec = (EXPL / "PLAN.sha256").read_text().splitlines() + add("plan_hash_matches", rec and rec[0].startswith(h), f"{h}") + + rc, out, err = sh(["git", "status", "--porcelain"]) + tracked = [line for line in out.splitlines() + if not line.startswith("?? exploration_v2/") + and "RESTORE_VERIFICATION_REPORT.md" not in line] + add("no_tracked_modifications", rc == 0 and not tracked, "|".join(tracked[:5])) + + rc, out, err = sh(["git", "status", "--porcelain", "planningawal/"]) + add("planningawal_unchanged", out == "", out) + + raw = list(REPO.glob("**/5G-NIDD.csv")) + list(REPO.glob("**/X-IIoTID-dataset.csv")) + raw = [p for p in raw if ".venv" not in str(p)] + add("no_raw_dataset_in_repo", not raw, "|".join(str(p) for p in raw[:3])) + + rc, out, err = sh([str(REPO / ".venv/bin/python"), "-m", "pytest", "-q", + "-p", "no:cacheprovider"]) + add("original_tests_pass", rc == 0, out.splitlines()[-1] if out else err[-200:]) + + rc, out, err = sh([str(REPO / ".venv/bin/python"), "-m", "pytest", + str(EXPL / "tests"), "-q", "-p", "no:cacheprovider"]) + add("variant_tests_pass", rc == 0, out.splitlines()[-1] if out else err[-200:]) + + seeds = set() + if (REPO / "results").is_dir(): + for p in (REPO / "results").glob("*seed*"): + name = p.name + for part in name.split("__"): + if part.startswith("seed"): + try: + seeds.add(int(part[4:])) + except ValueError: + pass + add("confirm_seeds_new", not ({3, 4} & seeds), f"restored seeds={sorted(seeds)}") + + for p in (EXPL / "partitions").glob("*.json"): + m = json.loads(p.read_text()) + if m.get("seed") in (3, 4): + add(f"manifest_ok_{p.name}", m.get("num_clients") == 10 and + len(m.get("support", [])) == 10, f"seed={m.get('seed')}") + + ok = all(c["ok"] for c in checks) + lines = ["# exploration_v2 setup check\n\n", + f"Generated: {__import__('datetime').datetime.utcnow().isoformat()}Z\n\n", + "| check | result | detail |\n|---|---|---|\n"] + for c in checks: + lines.append(f"| {c['check']} | {'PASS' if c['ok'] else 'FAIL'} | {c['detail']} |\n") + lines.append(f"\nOverall: {'PASS' if ok else 'FAIL'}\n") + (EXPL / "SETUP_CHECK.md").write_text("".join(lines), encoding="utf-8") + print(json.dumps({"overall": "PASS" if ok else "FAIL", "checks": checks}, indent=2)) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploration_v2/smoke/SMOKE.json b/exploration_v2/smoke/SMOKE.json new file mode 100644 index 0000000..ccdb681 --- /dev/null +++ b/exploration_v2/smoke/SMOKE.json @@ -0,0 +1,100 @@ +{ + "started_utc": "2026-10-30T04:10:46Z", + "tolerance": 0.001, + "runs": { + "base": { + "mode": "original_protean", + "beta": null, + "extra": [], + "returncode": 0, + "val_rare_macro_f1": 0.3506859881859882, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_base" + }, + "beta0": { + "mode": "support_weighted", + "beta": 0.0, + "extra": [], + "returncode": 0, + "val_rare_macro_f1": 0.3506859881859882, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_beta0" + }, + "beta1": { + "mode": "support_weighted", + "beta": 1.0, + "extra": [], + "returncode": 0, + "val_rare_macro_f1": 0.10617431010522135, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_beta1" + }, + "v_beta1.5": { + "mode": "v_beta", + "beta": 1.5, + "extra": [], + "returncode": 0, + "val_rare_macro_f1": 0.0014492753623188406, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_beta1.5" + }, + "v_shrink": { + "mode": "v_shrink", + "beta": 1.0, + "extra": [ + "--shrink-lambda", + "0.5" + ], + "returncode": 0, + "val_rare_macro_f1": 0.07774294670846396, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_shrink" + }, + "v_cap4": { + "mode": "v_cap", + "beta": 1.0, + "extra": [ + "--cap-factor", + "4" + ], + "returncode": 0, + "val_rare_macro_f1": 0.09021243192856791, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_cap4" + }, + "v_cap2": { + "mode": "v_cap", + "beta": 1.0, + "extra": [ + "--cap-factor", + "2" + ], + "returncode": 0, + "val_rare_macro_f1": 0.11728240313145973, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_cap2" + }, + "v_minsup": { + "mode": "v_minsup", + "beta": 0.0, + "extra": [ + "--min-support", + "10" + ], + "returncode": 0, + "val_rare_macro_f1": 0.09491525423728812, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_minsup" + }, + "v_logweight": { + "mode": "v_logweight", + "beta": 0.0, + "extra": [], + "returncode": 0, + "val_rare_macro_f1": 0.18652998776009794, + "results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_logweight" + } + }, + "pytest_returncode": 0, + "pytest_tail": [ + "........... [100%]" + ], + "equivalence_checks": { + "beta0": 0.0, + "beta0_within_tolerance": true + }, + "verdict": "PASS", + "finished_utc": "2026-10-30T04:11:54Z" +} \ No newline at end of file diff --git a/exploration_v2/smoke/partitions/xiiotid__alpha0.25__seed0.json b/exploration_v2/smoke/partitions/xiiotid__alpha0.25__seed0.json new file mode 100644 index 0000000..74d8d32 --- /dev/null +++ b/exploration_v2/smoke/partitions/xiiotid__alpha0.25__seed0.json @@ -0,0 +1,88 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-10-30T04:01:46Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 4, + "num_train": 2098, + "partition_hash": "9ac08797480badf66493b2ea557a1f8fd3adfcd190217e1d8fe6ce4accedb3e6", + "preprocessing_hash": "9adc542272436f89b3dfd3285bd6b73427c3f2c1a66caf7da95cd1962cc6ec7b", + "python": "3.12.3", + "rng_seed": 1617371214, + "seed": 0, + "split": { + "hash": "79e21dfb1bb8fa9076e8c24ea14cef409ee702be85c3ab86fc4ccb24fe0e6cb7", + "seed": 0, + "sizes": { + "test": 599, + "train": 2098, + "validation": 300 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 4, + 34, + 3, + 1, + 381, + 0, + 0, + 12, + 3 + ], + [ + 0, + 0, + 0, + 0, + 0, + 121, + 0, + 0, + 38 + ], + [ + 0, + 4, + 0, + 89, + 450, + 7, + 327, + 0, + 151 + ], + [ + 3, + 11, + 0, + 4, + 236, + 216, + 0, + 0, + 3 + ] + ] +} diff --git a/exploration_v2/smoke/partitions/xiiotid__alpha0.5__seed0.json b/exploration_v2/smoke/partitions/xiiotid__alpha0.5__seed0.json new file mode 100644 index 0000000..5ca564f --- /dev/null +++ b/exploration_v2/smoke/partitions/xiiotid__alpha0.5__seed0.json @@ -0,0 +1,88 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-10-30T03:57:20Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 4, + "num_train": 2798, + "partition_hash": "81600a44982bd77e32c0897aa9c367835ef5c8a3c651eb30f2b3e2862cc06a47", + "preprocessing_hash": "83d50d3adb93dd6b6f82e9c9395d9d4b28c30c7a783034508496988f6eddf077", + "python": "3.12.3", + "rng_seed": 2009062324, + "seed": 0, + "split": { + "hash": "f5370ac857a1b4463eb26b0f06a446a6235c008d0fa772019cf11a8d32f2eaf0", + "seed": 0, + "sizes": { + "test": 800, + "train": 2798, + "validation": 399 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 7, + 0, + 1, + 2, + 65, + 15, + 0, + 0, + 138 + ], + [ + 1, + 47, + 1, + 1, + 132, + 330, + 292, + 10, + 68 + ], + [ + 2, + 14, + 1, + 63, + 13, + 56, + 55, + 6, + 22 + ], + [ + 0, + 18, + 0, + 33, + 1246, + 101, + 47, + 5, + 6 + ] + ] +} diff --git a/exploration_v2/state.json b/exploration_v2/state.json new file mode 100644 index 0000000..e9a6947 --- /dev/null +++ b/exploration_v2/state.json @@ -0,0 +1,22 @@ +{ + "attempts": { + "xiiotid__alpha0.25__seed3__v_cap__b1__c4": 1, + "xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5": 1 + }, + "budget_start_utc": "2026-10-30T04:11:55Z", + "max_procs": 2, + "notes": [ + "2026-10-31T08:35Z recovery: 2 OOM-killed seed4 reference runs requeued at max_procs=2", + "2026-10-31T14:12Z recovery A5: unit-name collision on retries fixed; seed4 refs requeued" + ], + "phase": "done", + "ranking_frozen": true, + "recent_exits": [ + 0, + 0, + 0, + 0 + ], + "screening_started": false, + "terminal_failed": [] +} diff --git a/exploration_v2/status.sh b/exploration_v2/status.sh new file mode 100755 index 0000000..150ed90 --- /dev/null +++ b/exploration_v2/status.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Quick exploration_v2 status (returns in well under 60 s). +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/.." && pwd)" + +"$REPO_ROOT/.venv/bin/python" "$HERE/scripts/status_table.py" +echo +if [ -f "$HERE/HEARTBEAT" ]; then + echo "heartbeat file age: $(( $(date +%s) - $(stat -c %Y "$HERE/HEARTBEAT") )) s" +fi +echo "driver scope:" +systemctl --user list-units --type=scope --no-legend 2>/dev/null | grep -i explorev2 || true +echo "queue sizes: screen=$(($(wc -l < "$HERE/queue_screen.csv" 2>/dev/null || echo 1) - 1)) confirm=$( [ -f "$HERE/queue_confirm.csv" ] && echo $(( $(wc -l < "$HERE/queue_confirm.csv") - 1 )) || echo 0 )" diff --git a/exploration_v2/summary.csv b/exploration_v2/summary.csv new file mode 100644 index 0000000..74ab934 --- /dev/null +++ b/exploration_v2/summary.csv @@ -0,0 +1,35 @@ +config,dataset,alpha,beta,seed,aggregation_mode,status,wall_s,cpu_user_s,cpu_sys_s,percent_cpu,peak_rss_kb,exit_code,started_utc,ended_utc,run_dir,log_dir,notes +xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,1.0,0,v_shrink,completed,10965.00,10331.62,629.68,99,8788104,0,2026-10-30T04:11:55Z,2026-10-30T07:14:40Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/logs, +xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,1.0,0,v_cap,completed,11022.00,10392.67,626.13,99,8789080,0,2026-10-30T04:11:55Z,2026-10-30T07:15:37Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4/logs, +xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,1.5,0,v_beta,completed,11048.00,10424.25,620.24,99,8788696,0,2026-10-30T04:11:55Z,2026-10-30T07:16:03Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5/logs, +xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,0,0,original_protean,completed,10879.00,10141.92,628.62,98,8788432,0,2026-10-30T07:14:43Z,2026-10-30T10:16:03Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean/logs, +xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,0.0,0,v_minsup,completed,10904.00,10156.27,630.54,98,8788056,0,2026-10-30T07:16:06Z,2026-10-30T10:17:50Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10/logs, +xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,1.0,0,v_cap,completed,10960.00,10217.02,625.13,98,8787984,0,2026-10-30T07:15:45Z,2026-10-30T10:18:26Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2/logs, +xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,0.0,0,v_logweight,completed,10645.00,10013.81,627.86,99,8787588,0,2026-10-30T10:16:12Z,2026-10-30T13:13:38Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight/logs, +5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0,0,original_protean,completed,16807.00,15709.29,1094.58,99,10800928,0,2026-10-30T10:18:28Z,2026-10-30T14:58:36Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean/logs, +5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,1.5,0,v_beta,completed,17571.00,16433.69,1133.44,99,10787996,0,2026-10-30T13:13:46Z,2026-10-30T18:06:38Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5/logs, +5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,1.0,0,v_shrink,completed,17467.00,16353.62,1108.69,99,10791372,0,2026-10-30T14:58:45Z,2026-10-30T19:49:52Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/logs, +5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,1.0,0,v_cap,completed,16374.00,15305.03,1064.96,99,10795492,0,2026-10-30T18:06:47Z,2026-10-30T22:39:42Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4/logs, +5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,1.0,0,v_cap,completed,15875.00,14852.73,1018.36,99,10794444,0,2026-10-30T19:50:02Z,2026-10-31T00:14:37Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2/logs, +5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0.0,0,v_minsup,completed,16232.00,15203.43,1021.85,99,10787860,0,2026-10-30T22:39:52Z,2026-10-31T03:10:24Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10/logs, +5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0.0,0,v_logweight,completed,17342.00,16248.83,1085.02,99,10806700,0,2026-10-31T00:14:45Z,2026-10-31T05:03:48Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight/logs, +xiiotid__alpha0.25__seed3__original_protean,xiiotid,0.25,0,3,original_protean,completed,11102.00,10505.16,593.52,99,10986852,0,2026-10-31T05:07:30Z,2026-10-31T08:12:32Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__original_protean/logs, +xiiotid__alpha0.25__seed3__support_weighted__beta0.5,xiiotid,0.25,0.5,3,support_weighted,completed,11105.00,10503.16,599.06,99,10986972,0,2026-10-31T05:07:30Z,2026-10-31T08:12:36Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta0.5/logs, +xiiotid__alpha0.25__seed3__support_weighted__beta1,xiiotid,0.25,1,3,support_weighted,completed,11247.00,10628.09,615.51,99,10986812,0,2026-10-31T05:07:30Z,2026-10-31T08:14:57Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta1/logs, +xiiotid__alpha0.25__seed4__original_protean,xiiotid,0.25,0,4,original_protean,completed,11200.00,10544.02,652.44,99,13373900,0,2026-10-31T14:09:18Z,2026-10-31T17:15:59Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__original_protean/logs, +xiiotid__alpha0.25__seed4__support_weighted__beta0.5,xiiotid,0.25,0.5,4,support_weighted,completed,11147.00,10490.48,653.41,99,13376792,0,2026-10-31T14:09:18Z,2026-10-31T17:15:06Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta0.5/logs, +xiiotid__alpha0.25__seed4__support_weighted__beta1,xiiotid,0.25,1,4,support_weighted,completed,11529.00,10850.72,675.30,99,13377304,0,2026-10-31T17:15:10Z,2026-10-31T20:27:20Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta1/logs, +xiiotid__alpha0.25__seed3__v_cap__b1__c4,xiiotid,0.25,1.0,3,v_cap,completed,10621.00,10020.19,598.36,99,10986660,0,2026-10-31T11:09:05Z,2026-10-31T14:06:08Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_cap__b1__c4/logs, +xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,xiiotid,0.25,1.0,3,v_shrink,completed,10678.00,10077.25,599.26,99,10986572,0,2026-10-31T11:09:48Z,2026-10-31T14:07:47Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5/logs, +xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,xiiotid,0.25,1.0,4,v_shrink,completed,11448.00,10782.09,663.27,99,13377596,0,2026-10-31T17:16:03Z,2026-10-31T20:26:52Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5/logs, +xiiotid__alpha0.25__seed4__v_cap__b1__c4,xiiotid,0.25,1.0,4,v_cap,completed,10168.00,9580.67,584.50,99,13377208,0,2026-10-31T20:27:02Z,2026-10-31T23:16:30Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_cap__b1__c4/logs, +5gnidd__alpha0.5__seed3__original_protean,5gnidd,0.5,0,3,original_protean,completed,15875.00,14911.46,960.26,99,9482756,0,2026-10-31T20:27:23Z,2026-11-01T00:51:58Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__original_protean/logs, +5gnidd__alpha0.5__seed3__support_weighted__beta0.5,5gnidd,0.5,0.5,3,support_weighted,completed,16116.00,15128.78,983.96,99,9507836,0,2026-10-31T23:16:40Z,2026-11-01T03:45:16Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta0.5/logs, +5gnidd__alpha0.5__seed3__support_weighted__beta1,5gnidd,0.5,1,3,support_weighted,completed,16782.00,15774.09,1004.24,99,9480524,0,2026-11-01T00:52:06Z,2026-11-01T05:31:49Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta1/logs, +5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,5gnidd,0.5,1.0,3,v_shrink,completed,16459.00,15453.75,1000.89,99,9501568,0,2026-11-01T03:45:24Z,2026-11-01T08:19:43Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5/logs, +5gnidd__alpha0.5__seed3__v_cap__b1__c4,5gnidd,0.5,1.0,3,v_cap,completed,17023.00,15980.81,1039.03,99,9487356,0,2026-11-01T05:31:54Z,2026-11-01T10:15:38Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_cap__b1__c4/logs, +5gnidd__alpha0.5__seed4__original_protean,5gnidd,0.5,0,4,original_protean,completed,17324.00,16219.34,1100.77,99,11657012,0,2026-11-01T08:19:51Z,2026-11-01T13:08:35Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__original_protean/logs, +5gnidd__alpha0.5__seed4__support_weighted__beta0.5,5gnidd,0.5,0.5,4,support_weighted,completed,17181.00,16103.55,1073.60,99,11697428,0,2026-11-01T10:15:40Z,2026-11-01T15:02:03Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta0.5/logs, +5gnidd__alpha0.5__seed4__support_weighted__beta1,5gnidd,0.5,1,4,support_weighted,completed,17582.00,16447.80,1131.04,99,11669892,0,2026-11-01T13:08:37Z,2026-11-01T18:01:41Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta1/logs, +5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,5gnidd,0.5,1.0,4,v_shrink,completed,17234.00,16097.43,1133.25,99,11688404,0,2026-11-01T15:02:11Z,2026-11-01T19:49:27Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5/logs, +5gnidd__alpha0.5__seed4__v_cap__b1__c4,5gnidd,0.5,1.0,4,v_cap,completed,16415.00,15378.21,1031.38,99,11679488,0,2026-11-01T18:01:52Z,2026-11-01T22:35:27Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_cap__b1__c4/logs, diff --git a/exploration_v2/tests/test_pipeline_scripts.py b/exploration_v2/tests/test_pipeline_scripts.py new file mode 100644 index 0000000..238cd93 --- /dev/null +++ b/exploration_v2/tests/test_pipeline_scripts.py @@ -0,0 +1,124 @@ +"""Fixture tests for the screening ranking and post-confirmation analysis. + +These use a temporary directory tree and never read or write the live +``exploration_v2/results`` or any restored result. The references for the +fixture come from the real restored grid, read-only. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO / "exploration_v2" / "scripts")) + +import rank_screening # noqa: E402 +import analyze_results # noqa: E402 + + +def _write(path: Path, payload): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _screen_fixture(tmp: Path): + base = {"S1": 0.60, "S2": 0.85} + labels = [("v_beta__b1.5", 0.01), ("v_shrink__b1__lam0.5", 0.02), + ("v_cap__b1__c4", 0.015), ("v_cap__b1__c2", -0.01), + ("v_minsup__t10", -0.02), ("v_logweight", 0.005)] + rows = [] + for ds, alpha, cid in (("xiiotid", 0.25, "S1"), ("5gnidd", 0.5, "S2")): + for label, off in labels: + rid = f"{ds}__alpha{alpha:g}__seed0__{label}" + d = tmp / "runs" / rid / "results" / rid + _write(d / "val_global_metrics.json", + {"rare_class_macro_f1": base[cid] + off}) + _write(d / "val_selection.json", + {"val_rare_class_macro_f1_per_client_mean": 0.5 + off}) + # A decoy test file: the ranking must never open it. + _write(d / "test_global_metrics.json", {"rare_class_macro_f1": 999.0}) + rows.append(dict(run_id=rid, dataset=ds, alpha=str(alpha), seed="0", + mode="v_beta", beta="1", shrink="", cap="", + minsup="", part_dir="x", label=label)) + q = tmp / "queue_screen.csv" + with q.open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + return q + + +def test_rank_uses_validation_only(tmp_path, monkeypatch): + q = _screen_fixture(tmp_path) + monkeypatch.setattr(rank_screening, "RUNS", tmp_path / "runs") + monkeypatch.setattr(rank_screening, "SCREEN", q) + monkeypatch.setattr(rank_screening, "ACCESS", tmp_path / "ACCESS_LOG.txt") + monkeypatch.setattr(rank_screening, "OUT", tmp_path / "ranking.json") + assert rank_screening.main() == 0 + payload = json.loads((tmp_path / "ranking.json").read_text()) + assert payload["top2"] == ["v_shrink__b1__lam0.5", "v_cap__b1__c4"] + log = (tmp_path / "ACCESS_LOG.txt").read_text() + assert "test_global_metrics" not in log + assert "per_client.csv" not in log + + +def test_analysis_refuses_without_freeze(tmp_path, monkeypatch): + monkeypatch.setattr(analyze_results, "EXPL", tmp_path) + monkeypatch.setattr(analyze_results, "RUNS", tmp_path / "runs") + monkeypatch.setattr(analyze_results, "OUT", tmp_path / "analysis") + with pytest.raises(SystemExit): + analyze_results.main() + + +def _confirm_fixture(tmp: Path, top2): + rows = [] + for ds, alpha in (("xiiotid", 0.25), ("5gnidd", 0.5)): + for seed in (3, 4): + for label, val in [("original_protean", 0.60), + ("support_weighted__beta0.5", 0.61), + ("support_weighted__beta1", 0.605), + (top2[0], 0.62), (top2[1], 0.615)]: + rid = f"{ds}__alpha{alpha:g}__seed{seed}__{label}" + d = tmp / "runs" / rid / "results" / rid + _write(d / "val_global_metrics.json", + {"rare_class_macro_f1": val, "rare_class_recall": 0.8, + "macro_f1": 0.7}) + _write(d / "val_selection.json", + {"val_rare_class_macro_f1_per_client_mean": val - 0.05}) + _write(d / "test_global_metrics.json", + {"rare_class_macro_f1": val + 0.01, + "rare_class_recall": 0.8, "macro_f1": 0.7}) + rows.append(dict(run_id=rid, dataset=ds, alpha=str(alpha), + seed=str(seed), mode="x", beta="", shrink="", + cap="", minsup="", part_dir="x", label=label)) + q = tmp / "queue_confirm.csv" + with q.open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + (tmp / "queue_screen.csv").write_text( + "run_id,dataset,alpha,seed,mode,beta,shrink,cap,minsup,part_dir,label\n") + return top2 + + +def test_analysis_writes_tables(tmp_path, monkeypatch): + top2 = ["v_shrink__b1__lam0.5", "v_cap__b1__c4"] + _confirm_fixture(tmp_path, top2) + ranking = {"top2": top2, "candidates": [], "current_method_reference": {}} + (tmp_path / "ranking.json").write_text(json.dumps(ranking)) + h = hashlib.sha256((tmp_path / "ranking.json").read_bytes()).hexdigest() + (tmp_path / "ranking.sha256").write_text(f"{h} ranking.json\n2026-01-01T00:00:00Z\n") + monkeypatch.setattr(analyze_results, "EXPL", tmp_path) + monkeypatch.setattr(analyze_results, "RUNS", tmp_path / "runs") + monkeypatch.setattr(analyze_results, "OUT", tmp_path / "analysis") + assert analyze_results.main() == 0 + summary = list(csv.DictReader(open(tmp_path / "analysis" / "confirm_summary.csv"))) + assert len(summary) == 2 + assert summary[0]["positive_pairs_val_vs_original"] == "4" + assert (tmp_path / "REPORT_DATA.md").is_file() diff --git a/exploration_v2/tests/test_variants.py b/exploration_v2/tests/test_variants.py new file mode 100644 index 0000000..fe848ef --- /dev/null +++ b/exploration_v2/tests/test_variants.py @@ -0,0 +1,160 @@ +"""Unit tests for the exploration_v2 aggregation variants. + +The critical invariants required before any expensive run: + +* ``original_protean`` is bit-for-bit the project implementation; +* ``support_weighted`` (including beta = 0) delegates to the project code; +* ``v_beta`` at beta = 0 reproduces the equal mean, and at beta = 1 reproduces + ``support_weighted`` beta = 1; +* a single contributor, equal supports and missing classes behave as documented; +* every variant returns finite, deterministic vectors. + +Run with: + .venv/bin/python -m pytest exploration_v2/tests -q +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO / "src")) +sys.path.insert(0, str(REPO / "exploration_v2" / "scripts")) + +from protean_research.aggregation import aggregate_class_prototypes # noqa: E402 +import variants as V # noqa: E402 + + +def _fixture(): + rng = np.random.default_rng(7) + # class 0: 4 contributors; class 1: 2 contributors; class 2 absent + return { + 0: [(rng.normal(size=8), n) for n in (5, 50, 500, 5000)], + 1: [(rng.normal(size=8), 3), (rng.normal(size=8), 300)], + }, rng + + +def _mean(protos): + return np.mean(np.stack([np.asarray(p, float) for p in protos]), axis=0) + + +def test_original_is_project_implementation(): + contributions, _ = _fixture() + got = V.aggregate_variant(contributions, "original_protean") + ref = aggregate_class_prototypes(contributions, "original_protean") + assert set(got) == set(ref) == {0, 1} + for c in ref: + assert np.array_equal(np.asarray(got[c]), np.asarray(ref[c])) + + +def test_support_weighted_delegates_exactly(): + contributions, _ = _fixture() + for beta in (0.0, 0.25, 0.5, 0.75, 1.0): + got = V.aggregate_variant(contributions, "support_weighted", beta=beta) + ref = aggregate_class_prototypes(contributions, "support_weighted", beta=beta) + for c in ref: + assert np.array_equal(np.asarray(got[c]), np.asarray(ref[c])) + + +def test_beta0_is_equal_mean(): + contributions, _ = _fixture() + base = V.aggregate_variant(contributions, "original_protean") + for mode in ("support_weighted", "v_beta"): + got = V.aggregate_variant(contributions, mode, beta=0.0) + for c in base: + np.testing.assert_allclose(got[c], base[c], rtol=0, atol=1e-12) + # and v_beta beta1 == support_weighted beta1 + a = V.aggregate_variant(contributions, "v_beta", beta=1.0) + b = V.aggregate_variant(contributions, "support_weighted", beta=1.0) + for c in a: + np.testing.assert_allclose(a[c], b[c], rtol=0, atol=1e-12) + + +def test_single_contributor_invariance(): + proto = np.arange(6, dtype=float) + contributions = {0: [(proto, 17)]} + for mode in V.VARIANT_MODES: + for beta in (0.0, 0.5, 1.0): + got = V.aggregate_variant(contributions, mode, beta=beta) + np.testing.assert_allclose(got[0], proto, rtol=0, atol=0) + + +def test_equal_support_invariance(): + rng = np.random.default_rng(11) + contributions = {0: [(rng.normal(size=5), 100) for _ in range(4)]} + means = [np.asarray(V.aggregate_variant(contributions, "support_weighted", beta=b)[0]) + for b in (0.0, 0.25, 0.5, 0.75, 1.0)] + for m in means[1:]: + np.testing.assert_allclose(m, means[0], rtol=0, atol=1e-12) + + +def test_missing_class_never_fabricated(): + contributions, _ = _fixture() + for mode in V.ALL_MODES: + kwargs = {"beta": 0.0} if mode != "original_protean" else {} + got = V.aggregate_variant(contributions, mode, **kwargs) + assert 2 not in got + for vec in got.values(): + assert np.all(np.isfinite(vec)) + assert np.any(np.asarray(vec) != 0.0) + + +def test_shrink_endpoints(): + contributions, _ = _fixture() + equal = V.aggregate_variant(contributions, "original_protean") + sw = V.aggregate_variant(contributions, "support_weighted", beta=1.0) + s0 = V.aggregate_variant(contributions, "v_shrink", beta=1.0, shrink_lambda=0.0) + s1 = V.aggregate_variant(contributions, "v_shrink", beta=1.0, shrink_lambda=1.0) + for c in equal: + np.testing.assert_allclose(s0[c], equal[c], atol=1e-12) + np.testing.assert_allclose(s1[c], sw[c], atol=1e-12) + + +def test_cap_extremes(): + contributions, _ = _fixture() + sw = V.aggregate_variant(contributions, "support_weighted", beta=1.0) + big = V.aggregate_variant(contributions, "v_cap", beta=1.0, cap_factor=1e9) + for c in sw: + np.testing.assert_allclose(big[c], sw[c], atol=1e-9) + small = V.aggregate_variant(contributions, "v_cap", beta=1.0, cap_factor=1.0) + for c in sw: + assert np.all(np.isfinite(small[c])) + + +def test_minsup_participant_change(): + rng = np.random.default_rng(3) + p_small, p_big = rng.normal(size=4), rng.normal(size=4) + contributions = {0: [(p_small, 1), (p_big, 100)]} + got = V.aggregate_variant(contributions, "v_minsup", beta=0.0, min_support=10) + np.testing.assert_allclose(got[0], p_big, atol=0) + # fallback when every contributor is below the threshold + contributions = {0: [(p_small, 1), (p_big, 2)]} + got = V.aggregate_variant(contributions, "v_minsup", beta=0.0, min_support=10) + np.testing.assert_allclose(got[0], _mean([p_small, p_big]), atol=1e-12) + + +def test_logweight_positive_and_deterministic(): + contributions, _ = _fixture() + a = V.aggregate_variant(contributions, "v_logweight", beta=0.0) + b = V.aggregate_variant(contributions, "v_logweight", beta=0.0) + for c in a: + assert np.array_equal(np.asarray(a[c]), np.asarray(b[c])) + assert np.all(np.isfinite(a[c])) + + +def test_invalid_parameters(): + contributions, _ = _fixture() + with pytest.raises(ValueError): + V.aggregate_variant(contributions, "v_beta", beta=2.5) + with pytest.raises(ValueError): + V.aggregate_variant(contributions, "v_shrink", beta=0.5, shrink_lambda=1.5) + with pytest.raises(ValueError): + V.aggregate_variant(contributions, "v_cap", beta=0.5, cap_factor=0.5) + with pytest.raises(ValueError): + V.aggregate_variant(contributions, "not_a_mode", beta=0.5) + with pytest.raises(ValueError): + V.aggregate_variant(contributions, "v_beta", beta=None)