willtanoe/thesis-prototype-agregation
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: add exploration_v2 aggregation diagnosis and confirmation

Browse Source 
- Phase 1/2 inventory and diagnosis of the restored 108-run grid
- six candidate aggregators screened on validation (seed 0)
- top-2 confirmed on new seeds 3/4; v_shrink met the pre-registered improvement rule (4/4 pairs, mean +0.0180 validation), v_cap4 did not
- report, tables, frozen plan/ranking hashes, driver scripts and tests
This commit is contained in:
willtanoe committed 2026-11-02 08:09:29 +01:00
1 parent ffbef12812
commit 7a538e7316
75 files changed
+8460
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

exploration_v2/.gitignore
+11
View File
@@ -0,0 +1,11 @@
# Transient driver runtime files (not part of the research record).
driver.lock
summary.lock
HEARTBEAT
# Per-run result trees are regenerable/ignored by the repository-level
# `results/` rule; the analysis tables and REPORT_DATA.md are tracked instead.
# Smoke-run prototype snapshots are bulky and not part of the scientific record.
smoke/analysis/*__prototype_snapshot.json
smoke/processed/
exploration_v2/ACCESS_LOG.txt
+52
View File
@@ -0,0 +1,52 @@
2026-10-30T04:26:48Z /home/willtanoe/project/prototype-agregation/results/xiiotid__alpha0.25__seed0__original_protean/val_global_metrics.json
2026-10-30T04:26:48Z /home/willtanoe/project/prototype-agregation/results/xiiotid__alpha0.25__seed0__original_protean/val_selection.json
# ranking.py run at 2026-10-31T05:03:55Z
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean/results/xiiotid__alpha0.25__seed0__original_protean/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean/results/xiiotid__alpha0.25__seed0__original_protean/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0/results/xiiotid__alpha0.25__seed0__support_weighted__beta0/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0/results/xiiotid__alpha0.25__seed0__support_weighted__beta0/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.25/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.5/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/results/xiiotid__alpha0.25__seed0__support_weighted__beta0.75/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta1/results/xiiotid__alpha0.25__seed0__support_weighted__beta1/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/xiiotid__alpha0.25__seed0__support_weighted__beta1/results/xiiotid__alpha0.25__seed0__support_weighted__beta1/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean/results/5gnidd__alpha0.5__seed0__original_protean/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean/results/5gnidd__alpha0.5__seed0__original_protean/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0/results/5gnidd__alpha0.5__seed0__support_weighted__beta0/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0/results/5gnidd__alpha0.5__seed0__support_weighted__beta0/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.25/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.5/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/results/5gnidd__alpha0.5__seed0__support_weighted__beta0.75/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta1/results/5gnidd__alpha0.5__seed0__support_weighted__beta1/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/results/sweep/runs/5gnidd__alpha0.5__seed0__support_weighted__beta1/results/5gnidd__alpha0.5__seed0__support_weighted__beta1/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/queue_screen.csv
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5/results/xiiotid__alpha0.25__seed0__v_beta__b1.5/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5/results/xiiotid__alpha0.25__seed0__v_beta__b1.5/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/results/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/results/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4/results/xiiotid__alpha0.25__seed0__v_cap__b1__c4/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4/results/xiiotid__alpha0.25__seed0__v_cap__b1__c4/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2/results/xiiotid__alpha0.25__seed0__v_cap__b1__c2/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2/results/xiiotid__alpha0.25__seed0__v_cap__b1__c2/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10/results/xiiotid__alpha0.25__seed0__v_minsup__t10/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10/results/xiiotid__alpha0.25__seed0__v_minsup__t10/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight/results/xiiotid__alpha0.25__seed0__v_logweight/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight/results/xiiotid__alpha0.25__seed0__v_logweight/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5/results/5gnidd__alpha0.5__seed0__v_beta__b1.5/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5/results/5gnidd__alpha0.5__seed0__v_beta__b1.5/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/results/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/results/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4/results/5gnidd__alpha0.5__seed0__v_cap__b1__c4/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4/results/5gnidd__alpha0.5__seed0__v_cap__b1__c4/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2/results/5gnidd__alpha0.5__seed0__v_cap__b1__c2/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2/results/5gnidd__alpha0.5__seed0__v_cap__b1__c2/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10/results/5gnidd__alpha0.5__seed0__v_minsup__t10/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10/results/5gnidd__alpha0.5__seed0__v_minsup__t10/val_selection.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight/results/5gnidd__alpha0.5__seed0__v_logweight/val_global_metrics.json
2026-10-31T05:03:55Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight/results/5gnidd__alpha0.5__seed0__v_logweight/val_selection.json
exploration_v2/BUDGET_START
+1
View File
@@ -0,0 +1 @@
2026-10-30T04:11:55Z
exploration_v2/CANDIDATES.md
+167
View File
@@ -0,0 +1,167 @@
# Phase 3 — candidate modifications
All candidates below are implemented **only inside `exploration_v2/`**
(`exploration_v2/scripts/variants.py`, `run_variant.py`, unit tests in
`exploration_v2/tests/`). No file under `src/`, `configs/`, `tests/`,
`scripts/`, `results/`, `artifacts/` or `planningawal/` was changed.
Evidence codes used below are the Phase 2 measurements in
`exploration_v2/results/phase2_*.csv` (restored 108-run grid, read only).
Baseline facts used by every candidate (measured, pooled over seeds 0–2):
| condition | max weight share at β=1 | classes with `|S_j|<M` | mean support CV | largest β step loss (validation) |
|---|---|---|---|---|
| X-IIoTID α=0.25 | 0.528 | 6.3 / 9 | 1.442 | 0.75→1.00: −0.047 |
| 5G-NIDD α=0.25 | 0.454 | 5.3 / 9 | 1.302 | 0.25→0.50: −0.024 |
| X-IIoTID α=0.50 | 0.400 | 2.3 / 9 | 1.183 | 0.75→1.00: −0.006 |
| 5G-NIDD α=0.50 | 0.414 | 1.3 / 9 | 1.225 | 0.00→0.25: −0.042 |
| X-IIoTID α=0.75 | 0.284 | 1.0 / 9 | 0.881 | 0.50→0.75: −0.004 |
| 5G-NIDD α=0.75 | 0.298 | 1.3 / 9 | 0.916 | 0.50→0.75: −0.018 |
The two failure modes the candidates target are therefore:
(a) at high heterogeneity the β=1 weight vector places >50 % of a class's
weight on one client and the rare-class metric falls; (b) the deployed
per-client-mean selection metric disagrees with the global headline metric
(Phase 2: X-IIoTID α=0.25 and 5G-NIDD α=0.50).
---
## C1. Extended beta grid (β = 1.5) — `v_beta` β=1.5
- **Mechanism.** Keep `w_i ∝ N_ij^β` but allow β > 1. Concentrates weight more
strongly than the frozen grid's endpoint.
- **Motivating evidence.** 5G-NIDD α=0.50 selected the boundary value β=1.00 and
its validation curve was still rising at the boundary (0.6872 → 0.6924),
so the frozen grid cannot tell whether the optimum lies beyond 1. Also tests
whether the X-IIoTID α=0.25 degradation is monotone in β.
- **Definition change.** Same operator, extended candidate range; not a new
method. Selection procedure unchanged apart from the extra candidate.
- **Cost.** One full run per condition, same as a normal beta run.
- **Failure modes.** At α=0.25 it should worsen concentration and rare-class
recall further; at 5G-NIDD α=0.50 the boundary gain may be seed noise.
## C2. Shrinkage toward equal weighting — `v_shrink` β=1, λ=0.5
- **Mechanism.** `p_i = N_ij^β / Σ N_ij^β`; `w_i = (1−λ)/|S_j| + λ·p_i`.
λ=0 is the equal mean, λ=1 is the current method. λ=0.5 is fixed a priori.
- **Motivating evidence.** The largest losses occur where β=1 concentration is
highest (X-IIoTID α=0.25: max share 0.528; 5G-NIDD α=0.25: 0.454). Halving
the displacement should retain some support signal while bounding the tail.
- **Definition change.** New aggregation operator; would need its own name
(“λ-shrunk support-weighted aggregation”) if adopted.
- **Cost.** One run per condition. λ=0.5 is not tuned on screening data; if it
ranks top-2 it goes to confirmation unchanged.
- **Failure modes.** λ=0.5 may be too weak to prevent the α=0.25 loss, or too
strong to keep the 5G-NIDD α=0.75 gain; a single λ is unlikely to be right
everywhere.
## C3. Weight capping at 4× median — `v_cap` β=1, c=4
- **Mechanism.** `w_i ∝ min(N_ij^β, c · median_k(N_kj^β))`, then normalise.
Caps the influence of a single high-support client without discarding it.
- **Motivating evidence.** In the worst cells the top client holds ~0.45–0.53
of a class's weight; 53/162 class-cells have `|S_j|<M`, where that
concentration is largest (mean max share 0.463 vs 0.364 for full `S_j`).
- **Definition change.** New operator (“capped support-weighted aggregation”).
- **Cost.** One run per condition; `c` fixed at 4.
- **Failure modes.** Cap may bind on legitimate high-support classes and
reduce the benefit seen at 5G-NIDD α=0.75; median is unstable when `|S_j|`
is 2–3.
## C4. Weight capping at 2× median — `v_cap` β=1, c=2 (likely negative)
- **Mechanism.** Same as C3 with a much tighter cap.
- **Motivating evidence.** Deliberate stress test of the capping mechanism;
it should approach equal weighting in cells with `|S_j|` = 2–3.
- **Definition change.** Same operator family as C3.
- **Cost.** One run per condition.
- **Failure modes.** Expected to erase most support signal and behave like
β=0; included to show whether C3's effect is monotone in the cap. This is a
candidate we expect **not** to help.
## C5. Minimum-support threshold — `v_minsup` t=10, equal weights
- **Mechanism.** Drop contributors with `N_ij < 10` from `S_j`, then equal-mean
the remaining prototypes. This is a **participant-rule change**, not a
weighting change.
- **Motivating evidence.** Phase 2 reports that at α=0.25 many classes have
`|S_j|<M`; the intuition is that tiny-support prototypes are noisy.
- **Definition change.** New method (“minimum-support-filtered equal
aggregation”); it also changes the meaning of `S_j`, so it must not be
described as the same method with a different weight.
- **Cost.** One run per condition.
- **Failure modes.** The project's own rule is that `N_ij` is support, not
quality; low-support clients may carry the only examples of a rare attack.
Dropping them is expected to **hurt** rare-class recall and is included as a
negative candidate. It also leaves classes with no contributor if the
threshold is too high (documented fallback: keep all).
## C6. Logarithmic weight function — `v_logweight` w ∝ log(1+N)
- **Mechanism.** Replace the power law with a concave function of support.
Unlike `N^β`, the ratio between a support-1 and a support-1000 client is
bounded by log(2)/log(1001) ≈ 0.10 instead of 1/1000.
- **Motivating evidence.** The β grid compresses support differences only
gradually; a log weight is a stronger, differently-shaped compression that
cannot be reached by any β ∈ [0,1]. It tests whether the losses are caused
by the *shape* of the power law rather than by concentration alone.
- **Definition change.** New weight function (“log-support-weighted
aggregation”).
- **Cost.** One run per condition.
- **Failure modes.** It nearly equalises clients with support above ~100 and
may lose the gains at 5G-NIDD α=0.75; the `1+N` offset is arbitrary.
## C7. Different selection metric (analysis-level, zero runs)
- **Mechanism.** Keep the current aggregation but select β with validation
**global** rare-class macro-F1 (`val_rare_class_macro_f1_global`) instead of
the deployed per-client mean (`val_rare_class_macro_f1_per_client_mean`).
- **Motivating evidence (measured).** In X-IIoTID α=0.25 the deployed metric
ranks β=0 first but the global validation metric ranks β=0.50 first; in
5G-NIDD α=0.50 the deployed metric ranks β=1.00 first while the global
metric ranks β=0.00 first. The global metric's ranking matches the test
ranking in 6/6 conditions; the per-client metric in 4/6. This is a
descriptive comparison on existing data, not a significance test.
- **Definition change.** No aggregation change; it changes the **selection
procedure**, so it should be reported as a procedure result, not as a new
aggregation method.
- **Cost.** Zero training runs; it re-ranks existing validation files.
- **Failure modes.** Using a different metric after seeing test outcomes is
exactly the kind of post-hoc choice the protocol forbids. In this
exploration it is pre-registered *before* any new run and justified by
metric-definition alignment (global validation vs global headline), not by
test performance. It still cannot be confirmed on the old seeds.
## C8. Finer beta grid (analysis-level, zero runs on existing seeds)
- **Mechanism.** Evaluate the current operator at β = 0.125, 0.375, 0.625,
0.875 (or interpolate) to see whether the coarse 0.25 spacing hides a
better point.
- **Motivating evidence.** The largest adjacent-step changes are 0.047
(X-IIoTID α=0.25, 0.75→1.00) and 0.065 test (5G-NIDD α=0.25, 0.25→0.50), so
the response is not flat between grid points in the worst cells.
- **Definition change.** Same operator; finer candidate set.
- **Cost.** New runs would be required for real evidence; **not** run in this
exploration because β is a one-dimensional grid and the budget is reserved
for mechanism candidates. Only the already-observed coarse grid is reported.
- **Failure modes.** Any apparent interior optimum could be seed noise; the
existing three-seed spread is already as large as the between-β spread.
---
## Candidates deliberately not implemented
- Client rejection / Byzantine defence, prototype clustering, cosine distance,
personalised FL, attention or quality scoring: out of the thesis scope
(`AGENTS.md` §8) and not motivated by the Phase 2 evidence.
- Per-class β or per-client β: would multiply the run count and change the
method substantially; not screened.
## Cost and ranking summary
Six run-based candidates (C1–C6) are screened on two conditions (12 runs).
The selection-metric candidate C7 and the finer-grid candidate C8 are
analysis-level and cost no training runs. The top two run-based candidates go
to confirmation on two new seeds (20 runs including three reference configs).
Total planned: **32 new training runs**.
exploration_v2/DONE
+6
View File
@@ -0,0 +1,6 @@
{
"status": "complete",
"utc": "2026-11-01T22:35:36Z",
"runs_done": 20,
"runs_failed": 0
}
exploration_v2/HANDOFF.md
+64
View File
@@ -0,0 +1,64 @@
# exploration_v2 — final handoff
Updated: 2026-11-02T03:36Z. Status: **complete**.
## Result in one line
The campaign finished on 2026-11-01T22:35:36Z: **34/34 scientific runs done, 0
terminal failures, 66.4 h of the 96 h budget**. The pre-registered top-2
screening candidates were `v_shrink` (β=1, λ=0.5) and `v_cap4` (β=1, cap
4×median). On the four confirmatory pairs (2 conditions × seeds 3 and 4),
`v_shrink` improved over the **original baseline** on validation in 4/4 pairs
(mean +0.0180, sd 0.0141; test +0.0184, 4/4). `v_cap4` did **not** improve
(+0.0047, 2/4). Neither variant dominated the current method's per-pair best of
{original, β=0.5, β=1}: `v_shrink` mean −0.0153 on validation.
## Frozen artefacts
| item | value |
|---|---|
| `PLAN.sha256` | `2118f31b…c978e` at 2026-10-30T04:07:47Z |
| `ranking.sha256` | `f43ebc76…b054d3` at 2026-10-31T05:03:55Z |
| `DONE` | `status=complete`, 20 confirmation done, 0 failed, 2026-11-01T22:35:36Z |
| `REPORT.md` | final narrative + auto tables |
| `REPORT_DATA.md` | all tables |
| `analysis/*.csv` | per-run, per-seed, summary, wall time |
## What a later session should do
1. Read `REPORT.md` §5.8 and §8. The narrative is complete; no re-analysis is
required.
2. Optional follow-ups (new work, not corrections): test `v_shrink` at α=0.75
and on other datasets; test a pooled-across-seeds selection; test the C7
selection-metric change on new seeds. These are hypotheses, not promises.
3. Do not re-rank. `ranking.sha256` is frozen and `REPORT.md` records the
pre-registered decision.
4. Do not edit any tracked file. `exploration_v2/` is the only new tree;
`planningawal/`, `results/`, `artifacts/partitions/`, `src/`, `configs/`,
`tests/`, `scripts/` were not modified.
## Operational notes
- Three seed-4 reference runs were OOM-killed at the old 11 GiB cap; recovery
raised the X-IIoTID cap to 13 GiB, set `max_procs=2`, added a host
`MemAvailable ≥ 12 GiB` launch guard, and fixed a retry unit-name collision.
Full timeline: `PLAN_AMENDMENTS.md` A1–A5 and `ISSUES.md`.
- All failed attempts are archived under `results/attempts/`; the pre-campaign
path-bug attempts under `results/_driver_bug_20261030T0410Z/`.
- No test metric was read before the ranking freeze; `ACCESS_LOG.txt` contains
only validation paths.
- The host is shared; the results are conditional on the 13 GiB X-IIoTID cap
and on the two conditions/seeds tested.
## Ready-to-paste prompt for a later session
```
Continue exploration_v2 in ~/project/prototype-agregation. Read
exploration_v2/REPORT.md, REPORT_DATA.md, PLAN.md, PLAN_AMENDMENTS.md,
ISSUES.md and analysis/*.csv. The campaign is complete (DONE status=complete,
34/34 runs, 66.4 h); do not re-rank or re-select. For new work, treat v_shrink
(β=1, λ=0.5) as the only candidate that met the pre-registered improvement rule
over the original baseline (4/4 pairs, mean +0.0180 val), and remember it did
not beat the current method's per-pair best (mean −0.0153). Do not edit tracked
files and do not claim significance or generalisation.
```
exploration_v2/ISSUES.md
+206
View File
@@ -0,0 +1,206 @@
# exploration_v2 — issues and deviations log
This file records anything that could not be completed, permission problems,
deviations from PLAN.md, and driver-level failures. Entries are appended by the
agent and by the driver.
## 2026-10-30 — initial state
- All work is confined to `exploration_v2/`. No tracked file, no file under
`src/`, `configs/`, `tests/`, `scripts/`, `results/`, `artifacts/partitions/`
or `planningawal/` was modified.
- A non-scientific engineering smoke run (3000 rows, 2 rounds, 4 clients,
X-IIoTID α=0.25 seed 0) was executed before the PLAN hash was recorded to
validate the copied runner. It is not a screening or confirmatory run and its
numbers are not used for any ranking.
- The restored results contain seeds 0, 1 and 2 only (`seed34_count=0`), so
seeds 3 and 4 are genuinely new for confirmation.
## 2026-10-30T04:09:36Z — driver
run `xiiotid__alpha0.25__seed0__v_beta__b1.5` wrapper exit 1; see its logs.
## 2026-10-30T04:09:36Z — driver
run `xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5` wrapper exit 1; see its logs.
## 2026-10-30T04:09:36Z — driver
run `xiiotid__alpha0.25__seed0__v_cap__b1__c4` wrapper exit 1; see its logs.
## 2026-10-30T04:09:47Z — driver
run `xiiotid__alpha0.25__seed0__v_beta__b1.5` wrapper exit 1; see its logs.
## 2026-10-30T04:09:47Z — driver
run `xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5` wrapper exit 1; see its logs.
## 2026-10-30T04:09:47Z — driver
run `xiiotid__alpha0.25__seed0__v_cap__b1__c4` wrapper exit 1; see its logs.
## 2026-10-30T04:09:47Z — driver
run `xiiotid__alpha0.25__seed0__v_beta__b1.5` failed after 2 attempt(s); marked terminal failed.
## 2026-10-30T04:09:47Z — driver
run `xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5` failed after 2 attempt(s); marked terminal failed.
## 2026-10-30T04:09:47Z — driver
run `xiiotid__alpha0.25__seed0__v_cap__b1__c4` failed after 2 attempt(s); marked terminal failed.
## 2026-10-30T04:09:57Z — driver
run `xiiotid__alpha0.25__seed0__v_cap__b1__c2` wrapper exit 1; see its logs.
## 2026-10-30T04:09:57Z — driver
run `xiiotid__alpha0.25__seed0__v_minsup__t10` wrapper exit 1; see its logs.
## 2026-10-30T04:09:57Z — driver
run `xiiotid__alpha0.25__seed0__v_logweight` wrapper exit 1; see its logs.
## 2026-10-30T04:10Z — driver path bug before any scientific run
The first driver launch failed all queued wrappers with exit 127 because
`run_one_v2.sh` and `run_one_inner_v2.sh` resolved the repository root as
`$HERE/../..` while `$HERE` is already `exploration_v2/`. The wrapper therefore
looked for `/home/willtanoe/project/.venv/bin/python`. **No training started**:
all six attempt directories contain only empty result/log trees and a
`.failed` marker with exit 127. They were moved to
`exploration_v2/results/_driver_bug_20261030T0410Z/` and are listed in the final
run inventory as non-scientific driver failures.
Fix applied after the PLAN hash: `REPO_ROOT="$HERE/.."` in both scripts. This is
an engineering fix, not a methodological change; no screening or confirmatory
result existed when it was applied.
## 2026-10-30T04:10Z — PLAN amendment A1
Reason: the driver wrapper path bug above. What had been seen: the smoke test
(PASS, non-scientific) and six exit-127 wrapper failures with empty run
directories. No metric file of any variant existed. The plan's variants,
conditions, seeds, ranking metric, budget and stop rules are unchanged.
## 2026-10-30T04:28Z — ranking/analysis pre-run bug fixes
While the first screening runs were training (no result file existed yet), a
synthetic fixture under `/tmp/opencode/ranktest` exposed two path bugs that
would have failed the post-screening ranking: `val_files()` double-counted the
`results/` component for both restored and new runs. The helper now takes the
run's base directory explicitly. A second fixture
(`/tmp/opencode/analyzetest`) exposed a missing `OUT.mkdir` in the analysis
script, now added. Neither bug touched any run output, no metric was read, and
the PLAN is unchanged. Recorded here because the fixes happened after the plan
hash.
## 2026-10-30T04:32Z — PLAN amendment A2 (in-environment baseline references)
Added two `original_protean` screening runs at the front of the queue because
the restored grid was produced in a different environment and the project's own
audit measured non-trivial host/version differences (median 0.0022, max 0.0127
test rare-F1). No screening result had been read. Full reason in
`PLAN_AMENDMENTS.md` A2. Screening queue is now 14 runs; total planned 34.
## 2026-10-31T08:20:46Z — driver
run `xiiotid__alpha0.25__seed4__original_protean` wrapper exit 1; see its logs.
## 2026-10-31T08:20:56Z — driver
run `xiiotid__alpha0.25__seed4__original_protean` wrapper exit 1; see its logs.
## 2026-10-31T08:20:57Z — driver
run `xiiotid__alpha0.25__seed4__original_protean` failed after 2 attempt(s); marked terminal failed.
## 2026-10-31T08:26:42Z — driver
run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` wrapper exit 1; see its logs.
## 2026-10-31T08:26:53Z — driver
run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` wrapper exit 1; see its logs.
## 2026-10-31T08:26:54Z — driver
run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` failed after 2 attempt(s); marked terminal failed.
## 2026-10-31T08:27:04Z — driver
stopping launches: failure rate 2/5 exceeds 20%
## 2026-10-31T08:35Z — confirmation OOM kills and recovery (amendment A3)
Two seed-4 confirmation **reference** runs (`original_protean`,
`support_weighted__beta0.5`) were OOM-killed inside their 11 GiB systemd scopes
(exit surfaced as 143; scope `Result=oom-kill`). Each failed twice; the driver's
20 % failure-rate rule stopped new launches at 08:27:04Z while three other runs
continued. Failed attempts produced zero-byte logs and no artifacts, so no
metric existed. Recovery: attempts archived, runs requeued, `max_procs` 3→2,
wrapper now maps scope OOM to 137, and the failure-rate stop yields to the
memory-kill throttle. Full rationale: `PLAN_AMENDMENTS.md` A3.
No confirmatory test metric has been read.
## 2026-10-31T09:50Z — OOM prevention settings (amendment A4)
Applied after three seed-4 reference OOM kills: X-IIoTID cap 11→13 GiB,
`MAX_PROCS=2`, host `MemAvailable >= 12 GiB` launch guard, OOM→137 mapping and
3→2→1 throttle. Two seed-3 confirmation runs were already in flight with the old
11 GiB cap; they are left to finish rather than killed. No test metric has been
read.
## 2026-10-31T14:06:20Z — driver
run `xiiotid__alpha0.25__seed4__original_protean` wrapper exit 1; see its logs.
## 2026-10-31T14:06:31Z — driver
run `xiiotid__alpha0.25__seed4__original_protean` wrapper exit 1; see its logs.
## 2026-10-31T14:06:31Z — driver
run `xiiotid__alpha0.25__seed4__original_protean` failed after 2 attempt(s); marked terminal failed.
## 2026-10-31T14:06:41Z — driver
run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` wrapper exit 1; see its logs.
## 2026-10-31T14:06:52Z — driver
run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` wrapper exit 1; see its logs.
## 2026-10-31T14:06:53Z — driver
run `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` failed after 2 attempt(s); marked terminal failed.
## 2026-10-31T14:07:03Z — driver
run `xiiotid__alpha0.25__seed4__support_weighted__beta1` wrapper exit 1; see its logs.
## 2026-10-31T14:07:03Z — driver
stopping launches: failure rate 2/6 exceeds 20%
## 2026-10-31T14:12Z — retry collision fixed (amendment A5)
The 14:06Z failures were retries blocked by a stale transient scope unit name,
not new OOM kills. Evidence: stderr “Failed to start transient scope unit: Unit
… was already loaded” and zero-byte `time.txt`; the wrapper then read the old
scope's `Result=oom-kill`. Fixed with `reset-failed` before launch and a guard
that ignores the old Result when the scope never started. Seed-4 references are
running again with the 13 GiB cap. No test metric read.
## 2026-10-31T14:30Z — A6: in-place wrapper edit caused exit 127 (results unaffected)
Editing `run_one_v2.sh` while two wrappers were already running corrupted their
remaining shell commands (bash reads scripts incrementally); both exited 127
after training had finished and were retried. The archived first attempts
(`results/attempts/*seed3*v_cap*attempt1`, `*v_shrink*attempt1`) are bit-identical
to the final runs for val/test metrics and the round-0 snapshot, so no result
changed. Lesson recorded: never edit a bash script while instances are running.
exploration_v2/KEY_RESULTS.md
+78
View File
@@ -0,0 +1,78 @@
# exploration_v2 — key results (one page)
Status: complete. `DONE` = `status=complete`, 2026-11-01T22:35:36Z.
34/34 scientific runs, 0 terminal failures, 66.4 h of 96 h.
Plan hash `2118f31b…c978e`; ranking hash `f43ebc76…b054d3` (frozen before any
confirmatory run).
## Question
Which pre-specified modification to PROTEAN's server-side prototype aggregation
improves validation rare-class macro-F1 over (a) the original baseline and
(b) the current support-weighted method, and does it hold on seeds never used in
the analysis?
## Screening (validation only; seed 0; 2 conditions)
Score = mean over X-IIoTID α=0.25 and 5G-NIDD α=0.50 of the paired validation
difference in global rare-class macro-F1 vs the in-environment original.
| rank | variant | score vs baseline | rank by deployed metric |
|---|---|---|---|
| 1 | `v_shrink` (β=1, λ=0.5) | **+0.0267** | 1 |
| 2 | `v_cap4` (β=1, cap 4×median) | **+0.0064** | 2 |
| 3 | `v_cap2` (cap 2×median) | −0.0024 | 3 |
| 4 | `v_logweight` | −0.0139 | 5 |
| 5 | `v_minsup10` | −0.0302 | 6 |
| 6 | `v_beta1.5` | −0.0485 | 4 |
Top-2 margin: 0.0203. Only `v_shrink` was above the current method's best β on
validation, by +0.0038.
## Confirmation (new seeds 3 and 4; 4 pairs)
| variant | mean Δval vs baseline | sd | positive | mean Δtest | positive | pre-registered improved? |
|---|---|---|---|---|---|---|
| `v_shrink` | **+0.0180** | 0.0141 | **4/4** | +0.0184 | 4/4 | **yes** |
| `v_cap4` | +0.0047 | 0.0263 | 2/4 | +0.0059 | 2/4 | no |
`v_shrink` per condition (validation, seeds 3/4):
X-IIoTID +0.0085 / +0.0329 (mean +0.0207);
5G-NIDD +0.0269 / +0.0038 (mean +0.0153).
## Versus the current method
Best of {original, β=0.5, β=1} per pair, validation:
| variant | mean Δ vs current best | sd | pairs better |
|---|---|---|---|
| `v_shrink` | **−0.0153** | 0.0521 | 2/4 |
| `v_cap4` | −0.0286 | 0.0597 | 1/4 |
The current method's β=1 is very seed-sensitive: mean −0.0163, sd 0.1040,
1/4 positive (5G-NIDD seed 3: +0.1158; X-IIoTID seed 4: −0.1384).
β=0.5 is more stable: +0.0109, sd 0.0190.
## Bottom line
- One modification, **`v_shrink` (half-way between equal and support-weighted)**,
met the pre-registered improvement rule over the original baseline on all four
confirmation pairs. The gain is modest (~+0.018 rare-class macro-F1) and the
test differences agree with validation.
- It does **not** beat the current method's per-pair best; it is more stable but
lower-peak. No superiority claim is made.
- The average hides heterogeneity: on X-IIoTID only 9/20 client-seed pairs
improve (mean +0.0295) and the gain is concentrated in C&C/Weaponization/
Tampering; on 5G-NIDD 12/20 improve (mean +0.0119). See
`analysis/confirm_per_class_variant.csv` and `confirm_per_client_variant.csv`.
- The capping candidate (`v_cap4`) did not improve. Extended β=1.5,
minimum-support filtering, log-weights and 2× capping were negative in
screening.
- Four paired observations, no significance test; only two conditions and two
new seeds. See `REPORT.md` §6–8 for negative results, deviations and limits.
## Where the numbers live
`REPORT.md` (narrative) · `REPORT_DATA.md` (tables) · `analysis/*.csv`
(per-run/per-seed) · `ranking.json` + `ranking.sha256` · `ACCESS_LOG.txt`
(validation-only ranking reads).
exploration_v2/PHASE12_ANALYSIS.md
+351
View File
@@ -0,0 +1,351 @@
# Phase 1 and Phase 2 — inventory and diagnosis (plain language)
Scope: read-only analysis of the repository, the restored 108-run grid
(`results/`), the 30-run 1-thread sweep (`results/sweep/`), the reproducibility
runs (`results/sweep_repro/`), and the committed analysis tables
(`artifacts/analysis/`). No training was run for this document. Numeric tables
are reproduced in `exploration_v2/results/phase2_*.csv`.
Labels used: **measured** (read or recomputed from an artifact), **inferred**
(a reading of measured values), **unknown** (not established by the available
files).
---
## Phase 1 — what exists
### 1.1 The research object
The repository implements PROTEAN (Chennoufi et al., 2025) as a federated
intrusion detector. Each of 10 clients trains a small CNN on its local shard,
sends model weights and **class prototypes** (the mean embedding of each locally
observed class) to a server, and the server builds one **global prototype per
class** used for nearest-prototype classification. The thesis modifies exactly
one step: how the server combines the clients' class prototypes.
The implemented operator is:
```
Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) / ( sum_{i in S_j} N_ij^beta )
S_j = { clients with N_ij > 0 and a valid local prototype for class j }
```
`N_ij` is how many class-`j` training samples client `i` has. The baseline
`original_protean` uses equal weights `1/|S_j|`. The `S_j` rule itself is an
explicit **reproduction assumption**: no official PROTEAN code is available, so
the participant/denominator rule is a project decision
(`docs/research/EXPERIMENT_PROTOCOL.md`, `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`).
### 1.2 The beta grid
`beta ∈ {0, 0.25, 0.50, 0.75, 1.00}`. The immutable specification
(`planningawal/PROTEAN_Revisi_Lurus.md`, line 78) says, in Indonesian, that
initial testing uses this set. It does **not** derive the spacing, does not say
why five points were chosen, and does not justify why not a finer or wider grid.
The choice is therefore **documented as the initial candidate set but its
rationale is undocumented**. This exploration does not invent one. The endpoints
have known semantics: β=0 is equal weighting over the shared `S_j`; β=1 is
proportional to class support. β=0 is not claimed to be bitwise identical to
`original_protean` (floating-point differences; project equivalence class B).
### 1.3 Experiments in the repository and in the restored results
| # | experiment | what it tests | runs | where |
|---|---|---|---|---|
| E1 | **108-run grid** | 2 datasets × 3 Dirichlet α × 3 seeds × (original + 5 betas). The main thesis experiment: does validation-selected β beat/equal the reproduced baseline on rare-class detection? | 108 | `results/<dataset>__alpha<α>__seed<s>__<mode>` (restored) |
| E2 | **30-run 1-thread sweep** | Re-runs seed 0 only at 1 thread/process, all 2×3×5 beta cells, to test deterministic threading and to sanity-check the seed-0 β selection. No `original_protean` runs. | 30 | `results/sweep/runs/` |
| E3 | **Reproducibility runs** | One 1-thread repeat and one 8-thread run for X-IIoTID α=0.25 seed 0 β=0, and for 5G-NIDD α=0.5 seed 0 β=0.75, to see whether thread count changes results. | 4 | `results/sweep_repro/{a_1thread_repeat,b_8thread}/` |
| E4 | **Smoke tests** | Tiny end-to-end runs (few clients/rounds/rows) to check the pipeline; excluded from science. | a few | `results/smoke/`, `report_smoke_run.md` |
| E5 | **Multiseed / cross-alpha / heterogeneity analyses** | Read-only reductions of E1: pooled validation curves, realized support imbalance, per-client deltas, selected-vs-original summaries. | 0 new | `scripts/analyze_*.py`, `artifacts/analysis/*multiseed*`, `*cross_alpha*`, `*partition*` |
| E6 | **Prototype-effect analysis** | Recomputes global prototypes from fixed round-0 `{C_ij, N_ij}` snapshots at every β, without retraining, and measures displacement. | 0 new | `scripts/analyze_prototype_effect.py`, 18 surviving snapshots |
| E7 | **Deep paired comparison** | original vs support_weighted(selected β) on test and validation, per seed/client/class. | 0 new | `scripts/deep_comparison.py`, `artifacts/analysis/deepcmp__*` |
| E8 | **Post-hoc statistical audit** | Sign-flip tests / paired summaries over the 3 seeds; explicitly exploratory (minimum attainable p = 0.25). | 0 new | `scripts/posthoc_statistical_audit.py`, `artifacts/analysis/posthoc__*` |
| E9 | **S_j participant-rule audit** | Counts how often `|S_j| < M` and compares the adopted `S_j` rule with a literal `1/M` hypothetical. | 0 new | `scripts/validate_sj_participant_rule.py`, `sj_*csv` |
| E10 | **β=0 equivalence audit** | Compares `original_protean` and `support_weighted` β=0 snapshots/metrics. | 0 new | `scripts/verify_beta0_equivalence.py`, `*beta0_equivalence.csv` |
Restored-result inventory (measured): 108 run directories, 1,507 files, 36 runs
per seed {0,1,2}. One run,
`xiiotid__alpha0.5__seed0__original_protean`, is missing its five `val_*` files
and has an older `metadata.json` without the `beta` key. That run is a baseline,
not a β-selection candidate, so it does not affect β selection; it does mean that
the X-IIoTID α=0.5 baseline has no validation metrics from that directory. Only
18 prototype snapshots survived the restore (16 β=0 snapshots plus 2
`original_protean` snapshots); the rest are represented only by committed
derived tables.
### 1.4 Configs and scripts
Configs (frozen):
- `configs/datasets/datasets.yaml` — dataset paths, label columns, benign class,
dropped labels, feature/categorical/non-feature columns.
- `configs/experiments/experiment_protocol.yaml` — α grid, 10 clients, β
candidates, training hyperparameters, 70/10/20 split, seeds {0,1,2},
selection metric.
- `configs/models/protean_cnn.yaml` — the frozen model architecture.
Training and analysis scripts (`scripts/`): `run_baseline.py` (single run),
`generate_partitions.py`, `analyze_multiseed_beta.py`,
`analyze_cross_alpha.py`, `analyze_partition_heterogeneity.py`,
`analyze_prototype_effect.py`, `deep_comparison.py`,
`make_comparison_figures.py`, `make_final_figures.py`,
`posthoc_statistical_audit.py`, `validate_sj_participant_rule.py`,
`verify_beta0_equivalence.py`, plus `scripts/sweep/{analyze_sweep,check_noise,
compare_repro,compare_with_grid}.py`.
Source modules (`src/protean_research/`): `data/` (loading, preprocessing,
splits, Dirichlet partition, manifest, rare/unseen, characterization, pipeline),
`federated/` (client, server, round), `aggregation/` (the two aggregation modes),
`models/` (CNN, prototypes), `evaluation/metrics.py`, `utils/`.
Tests (`tests/`): 75 tests, including `test_baseline_aggregation.py`,
`test_support_weighted_aggregation.py`, `test_prototypes.py`,
`test_federated_smoke.py`, `test_metrics.py`, `test_data_pipeline.py`,
`test_partition.py`, `test_baseline_validation.py`, `test_prototype_effect.py`.
They cover aggregation invariants and the baseline/β=0 equivalence, not a full
sweep.
### 1.5 MAE / MAPE
**There is no MAE, MAPE, or any regression-error experiment or metric in this
repository.** A full-text search finds only the phrase “mean absolute
difference” in the sweep noise-check discussion
(`docs/research/SWEEP_1THREAD_SEED0_CHECK.md`,
`kisikisithesis/SWEEP_SUMMARY_REPORT.md`), which is a descriptive statistic
comparing two runs, not a model metric. Classification is evaluated with
accuracy, macro-F1, weighted-F1, per-class precision/recall/F1, and rare-class
macro-F1/recall. Any statement that MAE/MAPE was used would be false.
---
## Phase 2 — why the proposed method falls below the baseline
### 2.1 Where it falls below (measured)
Pooled over seeds 0–2, test rare-class macro-F1 (mean):
| condition | original | β=0 | β=0.25 | β=0.50 | β=0.75 | β=1.00 | lowest β | Δ lowest vs orig |
|---|---|---|---|---|---|---|---|---|
| X-IIoTID α=0.25 | 0.6328 | 0.6321 | 0.6368 | **0.6406** | 0.6272 | 0.5795 | 1.00 | **−0.0533** |
| X-IIoTID α=0.50 | 0.6425 | 0.6443 | 0.6574 | 0.6631 | **0.6655** | 0.6599 | 0.00 | +0.0018 |
| X-IIoTID α=0.75 | 0.6039 | 0.6059 | 0.6084 | **0.6052** | 0.6023 | 0.6024 | 0.75 | −0.0016 |
| 5G-NIDD α=0.25 | 0.8423 | 0.8412 | 0.8293 | 0.7644 | 0.7533 | **0.7817** | 0.75 | **−0.0890** |
| 5G-NIDD α=0.50 | 0.8805 | 0.8784 | 0.8573 | 0.8699 | 0.8754 | 0.8720 | 0.25 | **−0.0232** |
| 5G-NIDD α=0.75 | 0.8799 | 0.8765 | 0.8861 | **0.8967** | 0.8886 | 0.8945 | 0.00 | −0.0034 |
Measured facts:
- 41 of 90 (condition, seed, β>0) cells are **below** the original on test
rare-F1; 49 are above (the remaining are β=0 cells and FP ties).
- The damage is concentrated at α=0.25 in both datasets. 5G-NIDD α=0.25 is
below the baseline at **all three seeds** for β=1 (Δ −0.059, −0.104, −0.018).
- At the *pooled-selected* β the proposed method is mostly indistinguishable
from the baseline, because both α=0.25 conditions select β=0 by construction.
The large negatives appear when a positive β is used, not necessarily at the
deployed β. This nuance matters for how the result is phrased.
### 2.2 The three-seed grid and the single-seed sweep are different experiments
**Grid (3 seeds, pooled):** the validation-selected β is 0 at X-IIoTID α=0.25,
0 at 5G-NIDD α=0.25, 0.75 at X-IIoTID α=0.5, 1.0 at 5G-NIDD α=0.5, 0.5 at
X-IIoTID α=0.75, 0.5 at 5G-NIDD α=0.75. At the two α=0.25 cells the selected
"proposed method" is therefore equal weighting, so its difference from the
baseline is only floating-point divergence (measured |Δ| ≤ 0.001 in the pooled
table).
**Sweep (seed 0 only, 1 thread, no baseline run):** validation-selected β is 0.75,
0.5, 0.5 (X-IIoTID α=0.25/0.5/0.75) and 0.25, 0.75, 0.25 (5G-NIDD
α=0.25/0.5/0.75). Within the sweep, selected-β test rare-F1 minus β=0 test
rare-F1 is +0.027, +0.031, **−0.020**, +0.026, +0.010, +0.004 (same order). So
the single-seed sweep shows a positive weighting effect in 5/6 cells, while the
pooled grid selects β=0 at both α=0.25 cells. These are **not contradictory**:
they are different selection units (one seed vs three pooled) and the sweep has
no `original_protean` reference and a different threading/host environment.
Measured per-seed instability (grid): at 5G-NIDD α=0.5 the per-seed selected β
is 1.0 / 0.0 / 1.0; at X-IIoTID α=0.25 it is 0.75 / 0.5 / 0.0. The pooled
selection agrees with the per-seed choice in only 1–2 of 3 seeds per condition.
The sweep's seed-0 β equals the grid's seed-0 β in 4/6 conditions. None of this
is a significance test.
### 2.3 Which β step hurts most (measured)
Adjacent-step change in pooled validation per-client rare-F1:
| condition | 0→0.25 | 0.25→0.50 | 0.50→0.75 | 0.75→1.00 |
|---|---|---|---|---|
| X-IIoTID α=0.25 | −0.003 | −0.003 | −0.006 | **−0.047** |
| X-IIoTID α=0.50 | +0.011 | +0.012 | +0.012 | −0.006 |
| X-IIoTID α=0.75 | +0.003 | +0.002 | −0.004 | −0.004 |
| 5G-NIDD α=0.25 | −0.012 | **−0.024** | −0.004 | +0.015 |
| 5G-NIDD α=0.50 | **−0.042** | +0.030 | +0.014 | +0.004 |
| 5G-NIDD α=0.75 | +0.012 | +0.019 | −0.018 | +0.013 |
The worst single step is X-IIoTID α=0.25 from 0.75 to 1.00 (−0.047 validation,
−0.048 test) and 5G-NIDD α=0.25 from 0.25 to 0.50 (−0.024 validation, −0.065
test). Both are sharp transitions, not a smooth trend. The response is
non-monotone in every condition.
At β=1 the other test metrics move with rare-F1 (pooled deltas vs baseline):
X-IIoTID α=0.25 rare-F1 −0.053, rare-recall −0.114, macro-F1 −0.050, accuracy
−0.021; X-IIoTID α=0.50 +0.017 / −0.026 / +0.019 / −0.001; X-IIoTID α=0.75
−0.002 / −0.009 / +0.005 / +0.007; 5G-NIDD α=0.25 −0.061 / −0.065 / −0.037 /
−0.011; 5G-NIDD α=0.50 −0.009 / −0.006 / −0.004 / −0.001; 5G-NIDD α=0.75
+0.015 / +0.000 / +0.010 / +0.001. Accuracy is nearly flat while rare-F1 moves
by up to 0.06, so accuracy hides the effect.
### 2.4 Per class: what actually degrades (measured)
At β=1, pooled test-F1 deltas versus the baseline (mean over 3 seeds):
- **X-IIoTID α=0.25:** Lateral_movement −0.159, Weaponization −0.139,
Tampering −0.106, C&C −0.036; the benign class and Reconnaissance are
essentially flat (+0.005, +0.003). The loss is concentrated in behavioural
attack classes, not in the majority class.
- **5G-NIDD α=0.25:** ICMPFlood F1 −0.363 with **no recall change**
(precision collapse), HTTPFlood −0.126 (recall −0.182), UDPFlood −0.047
(recall −0.204), while Benign gains +0.096. This is the signature of the
decision regions shifting toward high-support traffic and cross-class
confusion.
- **X-IIoTID α=0.50:** C&C −0.079, Exfiltration −0.020, but Reconnaissance
+0.070 and Tampering +0.166; the aggregate is positive because gains and
losses partly cancel.
- **5G-NIDD α=0.50:** small losses (SYNFlood −0.043, SlowrateDoS −0.013) and a
small TCPConnectScan gain (+0.034); the condition-level loss is driven by
a few seeds, not by one class.
### 2.5 Per client and per seed (measured)
At β=1, validation per-client rare-F1 change versus the baseline (10 clients ×
3 seeds; positive / negative client-seed cells):
| condition | mean Δ | improve / degrade |
|---|---|---|
| X-IIoTID α=0.25 | −0.059 | 12 / 15 |
| X-IIoTID α=0.50 | +0.047 | 16 / 2 |
| X-IIoTID α=0.75 | −0.000 | 16 / 13 |
| 5G-NIDD α=0.25 | −0.022 | 10 / 11 |
| 5G-NIDD α=0.50 | +0.003 | 16 / 10 |
| 5G-NIDD α=0.75 | +0.019 | 14 / 11 |
Even in the best condition some clients degrade; even in the worst some
improve. Client identities are per-seed realisations, so they are not pooled.
Unseen classes (attack classes with `N_ij=0` for a client; never merged with
rare classes) also degrade at β=1. Mean recall over the union of unseen classes:
X-IIoTID α=0.25 −0.196, α=0.50 −0.029, α=0.75 −0.009; 5G-NIDD α=0.25 −0.017,
α=0.50 ~0.000, α=0.75 +0.000. Mean unseen F1: −0.073 (X-IIoTID α=0.25) and
−0.082 (5G-NIDD α=0.25). Table: `exploration_v2/results/phase2_unseen.csv`.
### 2.6 Validation versus test, and the selection/headline mismatch (measured)
The deployed selection metric is the **per-client mean** of rare-class macro-F1
on validation. The headline metric is the **global pooled** rare-class macro-F1
on test. These are different estimands: the per-client mean gives every client
equal weight regardless of how many rare samples it has; the global metric is
sample-weighted.
| condition | validation ranking by per-client mean | validation ranking by global rare-F1 | test ranking | per-client ranking matches test? |
|---|---|---|---|---|
| X-IIoTID α=0.25 | 0 > .25 > .50 > .75 > 1 | **.50** > .25 > 0 > .75 > 1 | .50 > .25 > 0 > .75 > 1 | no (6/6 for global) |
| X-IIoTID α=0.50 | .75 > 1 > .50 > .25 > 0 | .75 > .50 > 1 > .25 > 0 | .75 > .50 > 1 > .25 > 0 | yes |
| X-IIoTID α=0.75 | .50 > .25 > .75 > 0 > 1 | .25 > 0 > .50 > .75 > 1 | .25 > 0 > .50 > 1 > .75 | partly |
| 5G-NIDD α=0.25 | 0 > .25 > 1 > .50 > .75 | same order | same order | yes |
| 5G-NIDD α=0.50 | **1** > .75 > 0 > .50 > .25 | **0** > .75 > 1 > .50 > .25 | 0 > .75 > 1 > .50 > .25 | no |
| 5G-NIDD α=0.75 | .50 > 1 > .75 > .25 > 0 | same order | same order | yes |
Measured: the per-client-mean ranking matches the test ranking in 4/6
conditions, the global-validation ranking in 6/6. The two clearest failures are
exactly the two conditions with the largest proposed-vs-baseline loss:
X-IIoTID α=0.25 (selection picks β=0, test prefers 0.50) and 5G-NIDD α=0.50
(selection picks β=1, test prefers 0). This is a **selection-metric problem**,
not only a weighting problem. It is the direct motivation for candidate C7.
Validation-to-test *level* gaps are small for the global metric (e.g. 5G-NIDD
α=0.25 validation 0.8424 vs test 0.8423), so the mismatch is about **ranking**,
not about a systematic validation/test scale shift.
### 2.7 Support heterogeneity and the participant rule (measured)
| condition | mean support CV | classes with `|S_j|<M` (of 9) | mean max weight share at β=1 |
|---|---|---|---|
| X-IIoTID α=0.25 | 1.442 | 6.3 | 0.528 |
| X-IIoTID α=0.50 | 1.183 | 2.3 | 0.400 |
| X-IIoTID α=0.75 | 0.881 | 1.0 | 0.284 |
| 5G-NIDD α=0.25 | 1.302 | 5.3 | 0.454 |
| 5G-NIDD α=0.50 | 1.225 | 1.3 | 0.414 |
| 5G-NIDD α=0.75 | 0.916 | 1.3 | 0.298 |
At α=0.25, one client often holds more than half of a class's β=1 weight. Over
the whole grid, 53/162 class-cells have `|S_j| < 10`; in those cells the mean
top-weight share at β=1 is 0.463 versus 0.364 when all 10 clients contribute.
The participant rule is the same for baseline and weighted modes, so `S_j`
itself is not the cause of the *difference*; the difference is how weight is
distributed **inside** `S_j`. What the data cannot separate: whether the losses
come from concentration alone, from a few low-quality high-support prototypes,
or from the retraining feedback (changing the global prototypes changes the next
round's local alignment and therefore the local embeddings). The last is a
hypothesis, not measured.
### 2.8 Prototype displacement versus detection (measured / inferred)
Committed round-0 snapshots give mean `||Cbar_j(β) − Cbar_j(0)||` increasing
monotonically in β in all six conditions (e.g. X-IIoTID α=0.25: 1.48, 2.40,
3.00, 3.42 for β = .25/.50/.75/1.0). Across the 30 condition-β cells the
descriptive Pearson correlation between displacement and the test rare-F1 delta
is **−0.48**: larger displacement tends to accompany worse detection in this
grid. This is descriptive and confounded (displacement is larger exactly in the
cells where support is most imbalanced). **Inferred, not causal:** displacement
is a representation metric, not a quality metric. The project's own claim
register forbids reading displacement as detection quality, and this analysis
does not.
### 2.9 Reproduced baseline versus the paper (measured)
The paper reports “PROTEAN (Ours)” macro accuracy 92.67 / 93.62 / 93.43 on
X-IIoTID and 85.83 / 82.50 / 79.97 on 5G-NIDD for α = 0.25 / 0.50 / 0.75. A
macro-accuracy number is the mean per-class recall, so the reproduced baseline
can be compared on that definition (computed from the restored
`original_protean/test_per_class.csv`):
| α | X-IIoTID reproduced | X-IIoTID paper | 5G-NIDD reproduced | 5G-NIDD paper |
|---|---|---|---|---|
| 0.25 | 83.78 | 92.67 | 88.37 | 85.83 |
| 0.50 | 86.56 | 93.62 | 89.76 | 82.50 |
| 0.75 | 87.61 | 93.43 | 90.41 | 79.97 |
Measured gap: X-IIoTID is reproduced 5–9 points lower than the paper; 5G-NIDD is
3–10 points higher. The direction differs by dataset. Causes are **unknown**:
the paper's split (80/20, no validation), exact seeds, preprocessing, model
initialisation and whether “macro accuracy” is exactly mean per-class recall are
not fully specified or not available (no official code). All proposed-vs-baseline
statements are therefore conditional on this reproduced baseline, not on the
paper's numbers. This is a reproduction gap, not evidence about β.
### 2.10 Summary of the diagnosis
Measured:
1. The proposed operator loses most at α=0.25, especially 5G-NIDD (all three
seeds at β=1) and X-IIoTID at β≥0.75.
2. The loss is class-specific (rare attack classes lose F1/recall; benign/high
support classes gain), and client-specific in both directions.
3. The response is non-monotone in β; the sharpest harmful step is 0.75→1.00
(X-IIoTID α=0.25) and 0.25→0.50 (5G-NIDD α=0.25).
4. Weight concentration is high exactly where the loss is high (top weight share
0.45–0.53 at α=0.25).
5. The deployed per-client selection metric disagrees with the global headline
ranking in 2/6 conditions; the global validation metric agrees in 6/6.
6. Proposal displacement grows monotonically with β, while detection does not;
their descriptive correlation is negative.
Hypotheses (not measured):
- High-support clients may have prototypes that are good for majority traffic
but misplaced for rare attacks; down-weighting low-support clients removes
the only rare-class evidence. This is consistent with the class pattern but
not isolated.
- Part of the effect is end-to-end: changing global prototypes changes the
prototype-alignment loss in the next round, which changes the embeddings.
A fixed-snapshot analysis cannot separate this from the direct weighting
effect.
- Seed-level variation (selection changes in 2/3 seeds in some conditions) may
be larger than the between-β effect; three seeds cannot resolve it.
exploration_v2/PLAN.md
+283
View File
@@ -0,0 +1,283 @@
# exploration_v2 — pre-registered plan
Status: written before any screening or confirmatory run.
Repository: `/home/willtanoe/project/prototype-agregation`.
Scope: diagnostic follow-up to the completed 108-run PROTEAN beta grid.
Nothing in `src/`, `configs/`, `tests/`, `scripts/`, `results/`,
`artifacts/partitions/`, `planningawal/` is modified. All new material lives in
`exploration_v2/`.
A short engineering smoke run (3000 rows, 2 rounds, 4 clients, X-IIoTID seed 0)
was executed **before this file was hashed** to prove that the copied runner and
the monkey-patched aggregator work. It is not a screening run, not a
confirmatory run, and none of its numbers are used for ranking. The driver
re-runs the smoke test after this plan is hashed.
---
## 1. Question
Phase 2 measured that the current proposed method
(`support_weighted`, β selected on validation per-client-mean rare-class
macro-F1) scores **below** the reproduced original PROTEAN baseline in several
conditions, most clearly at X-IIoTID α=0.25 (β=1 test rare-F1 0.5795 vs 0.6328)
and 5G-NIDD α=0.50 (selected β=1 test 0.8720 vs 0.8805). This exploration asks:
> Among a small set of pre-specified modifications to the server-side
> aggregation, which ones improve validation rare-class macro-F1 over the
> original baseline and over the current proposed method, and do any of them
> keep that improvement on seeds that were never used in the analysis?
There is no target score. A modification that does not improve is a valid result
and stays in the tables.
## 2. Frozen inputs and seeds
- Existing partition manifests for seeds 0, 1, 2 are used **read-only** from
`artifacts/partitions/` for screening.
- Seeds 0, 1, 2 are **burned for confirmation** (Phase 2 read their test
metrics; the 108-run grid and the 30-run sweep both used them).
- Confirmatory seeds: **3 and 4**. New manifests are generated inside
`exploration_v2/partitions/` with the frozen procedure
(`protean_research.data.pipeline.create_partition`,
`per_class_dirichlet_multinomial`), never in `artifacts/partitions/`.
- Confirmatory manifest files:
`exploration_v2/partitions/xiiotid__alpha0.25__seed{3,4}.json`,
`exploration_v2/partitions/5gnidd__alpha0.5__seed{3,4}.json`.
## 3. Screening conditions (2)
Chosen from the Phase 2 diagnosis, before any new result is seen:
| id | dataset | alpha | seed | why |
|---|---|---|---|---|
| S1 | xiiotid | 0.25 | 0 | most heterogeneous cell; largest β-step validation loss (0.75→1.00: −0.047); maximum weight share 0.528 |
| S2 | 5gnidd | 0.50 | 0 | validation-selected β=1 lost to the baseline on test; deployed per-client metric and global metric disagree |
Screening uses these two conditions only. Both are existing frozen partitions;
all new runs in a condition reuse the same manifest, split and initialisation
policy, so comparisons are paired.
## 4. Screening variants (6) and references
| id | mode | fixed parameters | source |
|---|---|---|---|
| V1 `v_beta1.5` | `v_beta` | β=1.5 | extended grid |
| V2 `v_shrink0.5` | `v_shrink` | β=1, λ=0.5 | shrinkage toward equal |
| V3 `v_cap4` | `v_cap` | β=1, cap=4×median | capping |
| V4 `v_cap2` | `v_cap` | β=1, cap=2×median | aggressive capping (expected negative) |
| V5 `v_minsup10` | `v_minsup` | β=0, t=10 | participant-rule change (expected negative) |
| V6 `v_logweight` | `v_logweight` | β=0 | different weight function |
Hyperparameters are fixed above and are **not** tuned on screening outcomes.
Reference runs on the screening conditions are the **existing** seed-0 results
from the restored grid, read under identical conditions (same manifest, same
split, same initialisation policy):
- `original_protean`;
- `support_weighted` at β ∈ {0, 0.25, 0.50, 0.75, 1.00} (the current method's
beta grid).
No new reference training is needed for screening.
Analysis-level candidates with **zero** training cost:
- **C7 selection metric:** re-rank the same validation files using
`val_rare_class_macro_f1_global` instead of the deployed per-client mean.
- **C8 finer beta grid:** reported from the existing coarse grid only; no new
β points are run in this exploration.
## 5. Screening ranking metric (validation only)
Primary score for variant V:
```
score(V) = mean over c in {S1, S2} of
[ val_global_rare_f1(V, c, seed 0)
- val_global_rare_f1(original, c, seed 0) ]
```
where `val_global_rare_f1` is `rare_class_macro_f1` from
`val_global_metrics.json` (global pooled validation, the same definition as the
headline test metric). Ranking is by `score(V)`, descending. Tie-break:
larger minimum per-condition difference, then lexical id.
Justification (recorded before new results): the headline metric is the global
pooled rare-class macro-F1; Phase 2 measured that the deployed per-client mean
ranks β differently in 2/6 conditions while the global validation metric
matches the test ranking in 6/6. This is a metric-definition argument, not a
test-result-driven choice.
Secondary sensitivity ranking: the same score computed with
`val_rare_class_macro_f1_per_client_mean` from `val_selection.json` (the
deployed metric). Both rankings are reported.
The screening analysis code reads **only** `val_*` files and logs every path it
opens to `exploration_v2/ACCESS_LOG.txt`. It must not open any
`test_*`, `per_client.csv` or `history.json` file.
## 6. Confirmatory design
The **top 2** variants by the primary screening score go to confirmation.
Confirmation conditions: **S1 and S2** (same two conditions), seeds **3 and 4**.
Confirmation configurations per (condition, seed):
1. `original_protean` (baseline reference);
2. `support_weighted` β=1.00 (canonical current method / maximal support weight);
3. `support_weighted` β=0.50 (interior point of the current beta grid);
4. top-1 variant (fixed hyperparameters above);
5. top-2 variant (fixed hyperparameters above).
This is 5 configs × 2 conditions × 2 seeds = **20 confirmatory runs**.
Deviation from the literal "current proposed method with its beta grid": only
β=0.50 and β=1.00 are run on the new seeds, not all five betas, because the
full grid would add 12 runs and push the planned total past the conservative
96-hour capacity (§7). The full five-beta grid is the screening reference on
the existing seeds. This truncation is recorded here before any confirmation
result exists.
## 7. Budget, duration and maximum run count
Measured per-run wall times from `results/sweep/summary.csv` (30 completed
1-thread runs, seed 0):
| dataset | n | mean wall | min | max | mean peak RSS | max peak RSS | memory cap |
|---|---|---|---|---|---|---|---|
| xiiotid | 15 | 3.53 h | 2.94 h | 3.98 h | 7.02 GiB | 8.38 GiB | 11 G |
| 5gnidd | 15 | 5.49 h | 4.47 h | 6.09 h | 10.67 GiB | 13.12 GiB | 15 G |
| all | 30 | 4.51 h | | | | | |
Allowed concurrency (from the copied `run_one.sh` caps, sum ≤ 36 GiB, ≤ 3
processes): 3 × X-IIoTID = 33 GiB; 2 × 5G-NIDD = 30 GiB; 1 × 5G-NIDD + 1 ×
X-IIoTID = 26 GiB (a third process would be 37 GiB > 36). 5G-NIDD is never
allowed above 2 concurrent.
Conservative capacity bound (2 usable slots, mean run 4.51 h):
```
96 h / 4.51 h × 2 slots ≈ 42 runs
```
Planned: 12 screening + 20 confirmatory = **32 new runs**; 8 of the 32 are
X-IIoTID (S1) and 12 are 5G-NIDD (S2): screening 6+6, confirmation 10+10.
This leaves a margin below the bound. The run list is fixed by this plan.
Budget clock: the first scientific run's UTC start time is written to
`exploration_v2/BUDGET_START`; no new run may be launched after
`BUDGET_START + 96 h`.
If the budget turns out to be too small, reduce in this order:
1. drop screening condition S2 (keep S1) → 6 screening runs;
2. drop variants in this order: V4 `v_cap2`, V5 `v_minsup10`, V6
`v_logweight`, V3 `v_cap4`, V1 `v_beta1.5`. Never go below 3 variants;
the retained minimum set is V2 `v_shrink0.5`, V1 `v_beta1.5`, V3 `v_cap4`;
3. if still too small, reduce confirmatory conditions to S1 only (10
confirmatory runs), keeping 2 new seeds.
## 8. Improvement definition (confirmation)
For variant V and condition c ∈ {S1, S2}, seed s ∈ {3, 4}:
```
d(c,s) = val_global_rare_f1(V, c, s) - val_global_rare_f1(original, c, s)
```
- **Improvement over the original baseline**: mean over the four `d(c,s)` is
strictly positive **and** every one of the four differences is strictly
positive (same sign on every confirmatory seed and condition).
- **No improvement / mixed**: any other outcome. Mixed is reported, never
recoded as success.
- Secondary paired differences vs `support_weighted` β=1.00 and β=0.50 are
reported with mean, sample standard deviation and per-seed values. They are
descriptive; no significance test and no superiority claim.
Selection margins (best minus runner-up validation score within a condition)
are reported for the screening table. They are not used to claim a winner when
they are smaller than between-seed spread.
## 9. Stop rules
The driver stops launching new runs and records the reason in
`exploration_v2/ISSUES.md` and `exploration_v2/STOP_REASON` if any of:
1. `exploration_v2/STOP` exists;
2. more than 20 % of finished runs have failed (failed / (done + failed)), with
at least 5 finished runs;
3. the 96-hour budget is exhausted;
4. two memory-related kills (exit 137) occur in a row — then lower concurrency
by one and continue (this is a throttle, not a full stop, unless the
concurrency is already 1);
5. smoke test or the unit tests fail;
6. the pre-confirmation ranking script fails to produce a valid
`ranking.json`.
Runs already in flight are allowed to finish; the driver does not kill them.
## 10. Freezing and honesty notes
- The pre-confirmation ranking file is `exploration_v2/ranking.json`. Its
sha256 and UTC time are written to `exploration_v2/ranking.sha256` **before**
the first confirmatory run is launched.
- No test metric from any new run is read before that freeze. The screening
code reads only `val_*` files.
- After the freeze, confirmatory test metrics may be read by the final analysis
script only.
- Screening six variants raises the chance that the top one is lucky. The
confirmation step on unseen seeds exists precisely because of that; the
exploration does not adjust the plan after seeing confirmatory results.
- Amendments after the plan hash is recorded are allowed only as dated entries
in `exploration_v2/PLAN_AMENDMENTS.md`, each stating the reason and exactly
what had already been seen at that point. Silent edits to this file are not
permitted; the hash in `PLAN.sha256` is the frozen reference.
- No statistical significance, no superiority, no universal-beta claim will be
made. Differences are reported with sample standard deviations and per-seed
values.
- Every variant and every run is reported, including failures and negative
results.
## 11. Exact run list
Screening (12):
```
xiiotid__alpha0.25__seed0__v_beta__b1.5
xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5
xiiotid__alpha0.25__seed0__v_cap__b1__c4
xiiotid__alpha0.25__seed0__v_cap__b1__c2
xiiotid__alpha0.25__seed0__v_minsup__t10
xiiotid__alpha0.25__seed0__v_logweight
5gnidd__alpha0.5__seed0__v_beta__b1.5
5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5
5gnidd__alpha0.5__seed0__v_cap__b1__c4
5gnidd__alpha0.5__seed0__v_cap__b1__c2
5gnidd__alpha0.5__seed0__v_minsup__t10
5gnidd__alpha0.5__seed0__v_logweight
```
Confirmation (20; the two variant labels are filled from `ranking.json`):
```
{d}__alpha{a}__seed{s}__original_protean
{d}__alpha{a}__seed{s}__support_weighted__beta0.5
{d}__alpha{a}__seed{s}__support_weighted__beta1
{d}__alpha{a}__seed{s}__{top1_label}
{d}__alpha{a}__seed{s}__{top2_label}
```
for `(d,a) ∈ {(xiiotid,0.25),(5gnidd,0.5)}` and `s ∈ {3,4}`.
## 12. What will be reported
- Full Phase 1 inventory and Phase 2 plain-language diagnosis.
- Plan hash, every variant, every run (including failures), wall time and peak
memory.
- Screening ranking table with mean and per-condition paired differences over
the baseline and over the current method.
- Confirmatory per-seed differences over the baseline, β=0.5 and β=1.0, with
mean and sample standard deviation, and direction agreement.
- Negative results and deviations with timestamps.
- Explicit separation of measured / inferred / unknown statements.
exploration_v2/PLAN.sha256
+2
View File
@@ -0,0 +1,2 @@
2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e PLAN.md
2026-10-30T04:07:47Z
exploration_v2/PLAN_AMENDMENTS.md
+114
View File
@@ -0,0 +1,114 @@
# Plan amendments
## 2026-10-30T04:10Z — A1 (engineering only)
- **Reason:** `run_one_v2.sh` / `run_one_inner_v2.sh` used `$HERE/../..` as the
repository root while `$HERE` is already `exploration_v2/`, so the first six
wrapper attempts exited 127 before Python started.
- **What had been seen:** the non-scientific smoke test (PASS) and six
exit-127 wrapper failures with empty run trees. No validation or test metric
of any screening or confirmatory run existed.
- **Change:** repository root corrected to `$HERE/..`; failed attempt
directories archived under `exploration_v2/results/_driver_bug_20261030T0410Z/`.
- **Unchanged:** variants, conditions, seeds, ranking metric, budget,
concurrency, improvement definition, stop rules, PLAN.md content (hash still
valid for the methodological plan).
## 2026-10-30T04:32Z — A2 (reference runs moved in-environment)
- **Reason:** the plan's screening references were the restored 108-run grid,
which was produced on a different host/library environment. The project's own
reproducibility audit measured that a 1-thread rerun on this host is *not*
bitwise identical to that grid: over 30 seed-0 pairs the absolute test
rare-F1 difference has median 0.0022 and max 0.0127; a 1-thread *repeat* in
the current environment is bitwise identical to the sweep. Pairing new
variant runs against a restored baseline would therefore confound the
variant effect with an environment effect of the same order as the
differences being ranked.
- **What had been seen:** no validation or test metric from any screening or
confirmatory run existed. Only the non-scientific smoke test (PASS) and the
six archived exit-127 wrapper attempts existed.
- **Change:** two `original_protean` screening reference runs were appended to
the front of `queue_screen.csv` (X-IIoTID α=0.25 seed 0; 5G-NIDD α=0.5 seed 0)
so that the primary baseline is measured in the same environment and with the
same frozen partition as the variants. The ranking script now prefers:
1. the new in-environment original run for the baseline;
2. the current-environment 1-thread sweep for the β-grid reference (same host,
same partitions);
3. the restored grid as a fallback and sensitivity reference.
- **Cost:** +2 runs (14 screening instead of 12; 34 total instead of 32).
Conservative capacity was ≈42 runs, so the budget still holds.
- **Unchanged:** variants, hyperparameters, ranking metric definition, seeds,
confirmation design, improvement definition, stop rules. The primary metric
is still “paired validation difference versus the original baseline”.
## 2026-10-31T08:35Z — A3 (OOM recovery in confirmation)
- **Reason:** during confirmation, `xiiotid__alpha0.25__seed4__original_protean`
and `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` were killed by
systemd cgroup OOM (`Result: oom-kill`, MemoryMax=11 GiB) after ~5m39s, twice
each. The wrapper saw exit 143 and the driver's 20 % failure-rate stop rule
fired. The plan's memory-kill rule says: two memory-related kills in a row →
lower concurrency by one and continue. These are memory kills, so the throttle
rule is the applicable one.
- **What had been seen:** 14/14 screening runs done; ranking frozen at
05:03:55Z; 5 confirmation runs finished (3 seed-3 references); the two failed
runs produced **zero-byte stdout/stderr/time files and no result artifacts**.
No validation or test metric from the failed attempts existed, and none was
read.
- **Change:**
1. the two OOM-killed attempts were archived to
`results/attempts/<run>__attempt2` and requeued as clean directories;
2. `max_procs` lowered from 3 to 2 (memory throttle);
3. `run_one_v2.sh` now queries the transient scope's `Result` and records a
cgroup OOM as exit 137, so the driver's throttle logic sees memory kills;
4. the failure-rate stop is skipped when the last two exits were memory kills,
so the throttle rule wins over the full stop.
- **Retry accounting:** this gives the two environmental OOM victims one more
attempt than the nominal "one retry" rule. They are references, not variants,
and no result was ever produced; the deviation is recorded here rather than
silently hidden.
- **Unchanged:** ranking (frozen), variants, seeds, metrics, improvement
definition, confirmation design, 96 h budget.
## 2026-10-31T09:50Z — A4 (OOM prevention: cap, concurrency, host guard)
- **Reason:** three seed-4 confirmation reference runs were killed by
`oom-kill` inside 11 GiB systemd scopes; the user asked explicitly to prevent
further OOM kills. Memory safety is an operational setting, not a scientific
parameter.
- **What had been seen:** 14/14 screening done, ranking frozen, 3/20
confirmation done, three seed-4 references OOM-killed with zero-byte outputs.
No new test metric had been read.
- **Changes:**
1. X-IIoTID per-run systemd cap raised from 11 GiB to **13 GiB** (5G-NIDD
stays 15 GiB). Two concurrent X-IIoTID runs therefore request 26 GiB,
still below the 36 GiB campaign budget.
2. `MAX_PROCS` fixed at **2** (not 3).
3. New host-level guard: the driver refuses to launch unless
`/proc/meminfo MemAvailable` is at least **12 GiB**; this prevents starting
a run while another service is squeezing the host.
4. The wrapper maps a cgroup `oom-kill` to exit 137; two consecutive memory
kills lower `max_procs` to 1 and continue.
- **Unchanged:** ranking (frozen), variants, seeds, metrics, improvement
definition, 96 h budget, total run list.
## 2026-10-31T14:12Z — A5 (retry unit-name collision, not a new OOM)
- **Reason:** the 14:06Z failures were **not** new OOM kills. A failed/OOM-killed
transient systemd scope keeps its deterministic unit name loaded, so
`systemd-run --unit=<same name>` refused to start the retry with “Unit … was
already loaded”. The wrapper's new OOM check then read the *old* scope's
`Result=oom-kill` and mislabelled the failed start as 137. This consumed the
retry and tripped the 20 % failure-rate stop.
- **What had been seen:** 14/14 screening; 5/20 confirmation (all of seed-3 S1);
ranking frozen; failed retries had zero-byte logs, no artifacts, no metrics.
- **Change:**
1. `run_one_v2.sh` now runs `systemctl --user reset-failed <unit>.scope` and a
one-second pause before every launch, so a reused unit name is free;
2. if `stderr.log` contains “Failed to start transient scope unit”, the wrapper
does **not** map the old `Result` to an OOM; it records a failed start
instead.
- **Recovery:** the three blocked seed-4 reference attempts were archived
(`attempt2`/`attempt4`), requeued, and are running with the 13 GiB cap.
- **Unchanged:** ranking, variants, seeds, metrics, budget, improvement rule.
exploration_v2/PLAN_CHECK.md
+34
View File
@@ -0,0 +1,34 @@
# PLAN self-check (written before the plan hash is recorded)
Re-read `exploration_v2/PLAN.md` and checked each required item:
| requirement | where satisfied | verdict |
|---|---|---|
| at most 8 variants screened | PLAN §4: 6 run-based (V1–V6) + 2 analysis-level (C7, C8) = 8 | PASS |
| documented drop order if budget too small | PLAN §7: drop S2 first; then V4, V5, V6, V3, V1 down to 3; then one confirmation condition | PASS |
| reference runs: original + current method beta grid, identical conditions | PLAN §4 (screening: existing seed-0 grid, same manifests/splits); PLAN §6 + deviation note for the truncated confirmation grid (β=0.5, 1.0) | PASS, with the truncation stated as a deviation before any confirmation result |
| per-run duration estimate from sweep logs | PLAN §7: xiiotid mean 3.53 h, 5gnidd mean 5.49 h, all mean 4.51 h, from `results/sweep/summary.csv` | PASS |
| maximum runs at allowed concurrency within 96 h | PLAN §7: conservative bound ≈42 runs; planned 32 | PASS |
| exact validation ranking metric | PLAN §5: mean over S1/S2 of paired difference in `val_rare_class_macro_f1_global` vs original; tie-break defined | PASS |
| screening conditions and seeds | PLAN §3: S1 xiiotid α=0.25 seed 0; S2 5gnidd α=0.5 seed 0 | PASS |
| separate confirmatory seeds | PLAN §2/§6: seeds 3 and 4, new manifests inside exploration_v2/partitions | PASS |
| top 2 variants go to confirmation | PLAN §6 | PASS |
| improvement definition | PLAN §8: positive mean paired difference AND strictly positive on all four confirmation pairs | PASS |
| no-improvement definition | PLAN §8: any other outcome, mixed reported separately | PASS |
| stop rules | PLAN §9: STOP file, >20 % failure, budget exhausted, two consecutive memory kills → throttle, smoke/ranking failure | PASS |
| note that many variants raise luck risk | PLAN §10 | PASS |
| amendment rule | PLAN §10: dated entries in PLAN_AMENDMENTS.md only | PASS |
| no test metrics read before ranking freeze | PLAN §5/§10; ranking script restricted to val_* with ACCESS_LOG | PASS |
| all new outputs under exploration_v2 | PLAN header; runner defaults forced inside exploration_v2 | PASS |
| new seeds via frozen partition procedure | PLAN §2; `make_manifest_v2.py` calls the project's `create_partition` | PASS |
| one thread per process | PLAN §7 and the copied `run_one.sh` capping mechanism; `thread1_launcher.py` pins torch | PASS |
| memory caps ≤36 GiB and ≤3 processes | PLAN §7; driver scheduler enforces both | PASS |
| all new variants implemented only as exploration_v2 copies with unit tests | PLAN header; `variants.py`, `run_variant.py`, `tests/test_variants.py` (14 tests) | PASS |
Open caveat recorded deliberately: a non-scientific engineering smoke run was
executed before this self-check (3000 rows, 2 rounds, 4 clients). No screening
or confirmatory run has been launched, no new run's test metric has been read,
and no ranking decision depends on the smoke numbers. The driver repeats the
smoke after the hash.
Decision: proceed to `sha256sum PLAN.md > PLAN.sha256` and start the driver.
exploration_v2/README.md
+45
View File
@@ -0,0 +1,45 @@
# exploration_v2
Diagnostic follow-up to the completed 108-run PROTEAN beta grid. Everything in
this directory is new; the tracked research code, configs, tests, scripts,
results, partition manifests and the immutable research specification are
untouched.
## Layout
| path | purpose |
|---|---|
| `PLAN.md` | pre-registered plan (hash in `PLAN.sha256`) |
| `PLAN_CHECK.md` | self-check against the task constraints |
| `CANDIDATES.md` | Phase 3 candidate catalogue |
| `scripts/` | copied runner, variant implementations, ranking/analysis, driver |
| `tests/` | unit tests for the variants (baseline and β=0 unchanged) |
| `queue_screen.csv` | 12 screening runs |
| `queue_confirm.csv` | generated after the ranking freeze (20 runs) |
| `partitions/` | new-seed manifests (seeds 3 and 4) |
| `results/runs/` | per-run outputs, logs, COMPLETE markers |
| `analysis/` | final CSV tables |
| `REPORT_DATA.md` | all generated tables (written after confirmation) |
| `REPORT.md` | narrative report (Phase 6) |
| `ACCESS_LOG.txt` | every file opened by the validation-only ranking script |
| `HEARTBEAT`, `state.json`, `summary.csv` | driver state |
| `ISSUES.md` | failures, deviations and permission problems |
| `STOP`, `DONE` | stop request / completion marker |
## Entry points
```bash
# start / resume the detached driver in tmux session explore_v2
bash exploration_v2/resume.sh
# short status (returns immediately)
bash exploration_v2/status.sh
# manual re-run of one stage (only when the driver is not running)
.venv/bin/python exploration_v2/scripts/rank_screening.py
.venv/bin/python exploration_v2/scripts/analyze_results.py
```
The driver is idempotent: it reconstructs queue state from `COMPLETE`,
`.failed`, `.running` and live systemd scopes, retries a failed run once from a
clean directory, and never kills a process it did not start.
exploration_v2/REPORT.md
+662
View File
@@ -0,0 +1,662 @@
# exploration_v2 — final report
Status: **complete**. The driver finished on 2026-11-01T22:35:36Z with
**34/34 scientific runs done, 0 terminal failures**, in **66.4 h** of the 96 h
budget (`DONE` has `"status": "complete"`). This narrative covers Phase 1
(inventory), Phase 2 (diagnosis), the pre-registered design, and the screening
and confirmatory results. All numeric tables are in
`exploration_v2/REPORT_DATA.md` and `exploration_v2/analysis/*.csv`.
Plan hash: `2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e`
(recorded 2026-10-30T04:07:47Z in `exploration_v2/PLAN.sha256`).
Ranking hash: `f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3`
(recorded 2026-10-31T05:03:55Z in `exploration_v2/ranking.sha256`, before any
confirmatory run).
Labels: **measured** = read/recomputed from a file; **inferred** = a reading of
measured values; **unknown** = not established.
---
## 1. Phase 1 — what the repository contains and what each experiment tests
### 1.1 The method under study
PROTEAN is a federated intrusion detector. Ten clients train a CNN locally and
send the server (a) model weights and (b) one **prototype per locally observed
class** — the mean embedding of that class's training samples. The server
aggregates the local prototypes into one global prototype per class, and
classification is nearest global prototype. The thesis changes only the
server-side aggregation:
```
Cbar_j(beta) = sum_{i in S_j} N_ij^beta C_ij / sum_{i in S_j} N_ij^beta
S_j = { clients with N_ij > 0 and a valid local prototype for class j }
```
`N_ij` is the class-specific training support. The baseline
(`original_protean`) uses equal weights `1/|S_j|`. The `S_j` participant rule is
an explicit reproduction assumption, because no official PROTEAN code is
available. `N_ij` is support, not quality; nothing here assumes a larger
support means a better prototype.
### 1.2 The beta grid
The candidates are β ∈ {0, 0.25, 0.50, 0.75, 1.00}. The immutable specification
(`planningawal/PROTEAN_Revisi_Lurus.md`, line 78) names this set as the initial
test grid. It gives **no derivation of the spacing and no rationale for why
these five points** rather than a finer or wider grid. The endpoints have known
semantics (β=0 equal weighting; β=1 support-proportional), but the choice of
the intermediate points is **undocumented**. This report does not invent a
rationale.
### 1.3 Experiments (measured inventory)
| experiment | what it tests | runs | evidence |
|---|---|---|---|
| 108-run grid | 2 datasets × 3 α × 3 seeds × (baseline + 5 β): does validation-selected β change rare-class detection? | 108 | restored `results/` |
| 30-run 1-thread sweep | seed 0 only, 2×3×5 β cells, deterministic threading and seed-0 selection | 30 | `results/sweep/` |
| reproducibility runs | 1-thread repeat vs 8-thread on two cells | 4 | `results/sweep_repro/` |
| smoke tests | pipeline sanity only, excluded from science | several | `results/smoke/`, `report_smoke_run.md` |
| multiseed / cross-alpha / heterogeneity analyses | reductions of the grid | 0 | `scripts/analyze_*.py`, `artifacts/analysis/*` |
| prototype-effect analysis | recomputes prototypes from fixed round-0 `{C_ij,N_ij}` snapshots | 0 | `scripts/analyze_prototype_effect.py` |
| deep paired comparison | original vs selected-β per seed/client/class | 0 | `scripts/deep_comparison.py` |
| post-hoc audit | exploratory sign-flip tests, n=3 | 0 | `scripts/posthoc_statistical_audit.py` |
| `S_j` audit | how often `|S_j| < M`; literal `1/M` comparison | 0 | `scripts/validate_sj_participant_rule.py` |
| β=0 equivalence | baseline vs β=0 snapshots/metrics | 0 | `scripts/verify_beta0_equivalence.py` |
Restored results: 108 directories, 1,507 files, 36 per seed {0,1,2}. One
baseline run (`xiiotid__alpha0.5__seed0__original_protean`) lacks its five
`val_*` files and has an older metadata schema; it is not used for β selection.
Only 18 prototype snapshots survived the restore; most displacement claims rely
on the committed derived tables.
Configs: `configs/datasets/datasets.yaml`, `configs/experiments/
experiment_protocol.yaml`, `configs/models/protean_cnn.yaml`.
Scripts: `scripts/{run_baseline,generate_partitions,analyze_multiseed_beta,
analyze_cross_alpha,analyze_partition_heterogeneity,analyze_prototype_effect,
deep_comparison,make_comparison_figures,make_final_figures,
posthoc_statistical_audit,validate_sj_participant_rule,
verify_beta0_equivalence}.py` and `scripts/sweep/*.py`.
Tests: 75 tests covering aggregation invariants, the data pipeline, metrics and
smoke training.
### 1.4 MAE / MAPE
**No MAE, MAPE or any regression-error experiment or metric exists in this
repository.** A full-text search finds only the phrase “mean absolute
difference” in the sweep noise-check text, which compares two runs and is not a
model metric. Evaluation uses accuracy, macro-F1, weighted-F1, per-class
precision/recall/F1, and rare-class macro-F1/recall. Saying otherwise would be
false.
---
## 2. Phase 2 — why the proposed method scores below the baseline
All numbers below are **measured** from the restored grid / committed tables;
tables are in `exploration_v2/results/phase2_*.csv` and the long form in
`exploration_v2/PHASE12_ANALYSIS.md`.
### 2.1 Where it loses
Pooled test rare-class macro-F1 over seeds 0–2:
| condition | original | β=0 | β=0.25 | β=0.50 | β=0.75 | β=1.00 | worst β | Δ worst vs orig |
|---|---|---|---|---|---|---|---|---|
| X-IIoTID α=0.25 | 0.6328 | 0.6321 | 0.6368 | **0.6406** | 0.6272 | 0.5795 | 1.00 | **−0.0533** |
| X-IIoTID α=0.50 | 0.6425 | 0.6443 | 0.6574 | 0.6631 | **0.6655** | 0.6599 | 0.00 | +0.0018 |
| X-IIoTID α=0.75 | 0.6039 | 0.6059 | 0.6084 | **0.6052** | 0.6023 | 0.6024 | 0.75 | −0.0016 |
| 5G-NIDD α=0.25 | 0.8423 | 0.8412 | 0.8293 | 0.7644 | 0.7533 | **0.7817** | 0.75 | **−0.0890** |
| 5G-NIDD α=0.50 | 0.8805 | 0.8784 | 0.8573 | 0.8699 | 0.8754 | 0.8720 | 0.25 | **−0.0232** |
| 5G-NIDD α=0.75 | 0.8799 | 0.8765 | 0.8861 | **0.8967** | 0.8886 | 0.8945 | 0.00 | −0.0034 |
41 of 90 (condition, seed, β>0) cells are below the original; 49 are above.
5G-NIDD α=0.25 is below at all three seeds for β=1 (Δ −0.059, −0.104, −0.018).
At the **pooled-selected** β the proposed method is ~equal to the baseline at
both α=0.25 cells because those cells select β=0; the large losses appear when
a positive β is used. This distinction must be kept in every claim.
### 2.2 The three-seed grid and the single-seed sweep are separate evidence
- Grid (3 seeds, pooled): selected β = 0, 0.75, 0.50 (X-IIoTID α = .25/.50/.75)
and 0, 1.00, 0.50 (5G-NIDD). At both α=0.25 cells the selected method is equal
weighting, so its difference from the baseline is floating-point only.
- Sweep (seed 0, 1 thread, no baseline run): selected β = 0.75, 0.50, 0.50 and
0.25, 0.75, 0.25. Within the sweep, selected-β minus β=0 test rare-F1 is
+0.027, +0.031, **−0.020**, +0.026, +0.010, +0.004. So the single seed gives a
positive weighting effect in 5/6 cells, unlike the pooled grid at α=0.25.
They are not contradictory: different selection units, and the sweep has no
`original_protean` reference. Per-seed grid selections are unstable (e.g.
5G-NIDD α=0.50: 1.0 / 0.0 / 1.0; X-IIoTID α=0.25: 0.75 / 0.5 / 0.0).
### 2.3 Which β step hurts most
Validation per-client rare-F1 adjacent-step change:
| condition | 0→.25 | .25→.50 | .50→.75 | .75→1.00 |
|---|---|---|---|---|
| X-IIoTID α=0.25 | −0.003 | −0.003 | −0.006 | **−0.047** |
| X-IIoTID α=0.50 | +0.011 | +0.012 | +0.012 | −0.006 |
| X-IIoTID α=0.75 | +0.003 | +0.002 | −0.004 | −0.004 |
| 5G-NIDD α=0.25 | −0.012 | **−0.024** | −0.004 | +0.015 |
| 5G-NIDD α=0.50 | **−0.042** | +0.030 | +0.014 | +0.004 |
| 5G-NIDD α=0.75 | +0.012 | +0.019 | −0.018 | +0.013 |
The worst steps are X-IIoTID α=0.25 0.75→1.00 (−0.047 validation, −0.048 test)
and 5G-NIDD α=0.25 0.25→0.50 (−0.024 validation, −0.065 test). Response is
non-monotone everywhere.
At β=1 the other metrics move with the same sign as rare-F1 (pooled test deltas
versus the baseline):
| condition | rare-F1 | rare-recall | macro-F1 | accuracy |
|---|---|---|---|---|
| X-IIoTID α=0.25 | −0.053 | −0.114 | −0.050 | −0.021 |
| X-IIoTID α=0.50 | +0.017 | −0.026 | +0.019 | −0.001 |
| X-IIoTID α=0.75 | −0.002 | −0.009 | +0.005 | +0.007 |
| 5G-NIDD α=0.25 | −0.061 | −0.065 | −0.037 | −0.011 |
| 5G-NIDD α=0.50 | −0.009 | −0.006 | −0.004 | −0.001 |
| 5G-NIDD α=0.75 | +0.015 | +0.000 | +0.010 | +0.001 |
The key asymmetry: in X-IIoTID α=0.50 rare-F1 rises while rare-recall falls
(+precision, −recall). Overall accuracy barely moves even when rare-F1 moves by
0.05–0.06, which is why accuracy alone hides the effect.
### 2.4 Per class, per client, per seed
- X-IIoTID α=0.25 at β=1: Lateral_movement −0.159 F1, Weaponization −0.139,
Tampering −0.106, C&C −0.036; benign +0.005, Reconnaissance +0.003.
- 5G-NIDD α=0.25 at β=1: ICMPFlood −0.363 F1 with **no recall change**
(precision collapse), HTTPFlood −0.126 (recall −0.182), UDPFlood −0.047
(recall −0.204); benign +0.096. The decision regions shift toward
high-support traffic.
- X-IIoTID α=0.50 at β=1: C&C −0.079, Exfiltration −0.020, but
Reconnaissance +0.070, Tampering +0.166; gains and losses partly cancel.
- Per-client β=1 validation deltas: worst condition X-IIoTID α=0.25 mean −0.059
(12 client-seed cells improve, 15 degrade); best X-IIoTID α=0.50 +0.047
(16 improve, 2 degrade). Even the worst condition has improving clients and
the best has degrading clients. Client identities are per-seed realisations.
- Unseen-class behaviour (attack classes with `N_ij=0` for a client; **not**
merged with rare classes): at β=1 the mean recall over the union of unseen
classes falls by 0.196 on X-IIoTID α=0.25, 0.029 on X-IIoTID α=0.50, 0.009 on
X-IIoTID α=0.75, 0.017 on 5G-NIDD α=0.25, and is ~0 on the other two
conditions; mean unseen F1 falls by 0.073 on X-IIoTID α=0.25 and 0.082 on
5G-NIDD α=0.25. The weighting also damages classes a client never trained on,
not only its two designated rare classes. Full table:
`exploration_v2/results/phase2_unseen.csv`.
### 2.5 Validation vs test and the selection/headline mismatch
The deployed selector maximises the **per-client mean** validation rare-F1; the
headline is the **global pooled** test rare-F1. The two rankings agree with the
test ranking in 4/6 and 6/6 conditions respectively. The two failures are the
two conditions with the largest proposed-vs-baseline loss:
- X-IIoTID α=0.25: per-client selector picks β=0; global validation (and test)
prefer β=0.50.
- 5G-NIDD α=0.50: per-client selector picks β=1; global validation (and test)
prefer β=0 (the baseline).
Validation-to-test *levels* are close for the global metric (e.g. 5G-NIDD
α=0.25 validation 0.8424 vs test 0.8423), so the problem is ranking, not a
global scale shift. This is the direct motivation for the selection-metric
candidate.
### 2.6 Support heterogeneity and the participant rule
| condition | mean support CV | classes `|S_j|<M` (of 9) | mean max weight share at β=1 |
|---|---|---|---|
| X-IIoTID α=0.25 | 1.442 | 6.3 | 0.528 |
| X-IIoTID α=0.50 | 1.183 | 2.3 | 0.400 |
| X-IIoTID α=0.75 | 0.881 | 1.0 | 0.284 |
| 5G-NIDD α=0.25 | 1.302 | 5.3 | 0.454 |
| 5G-NIDD α=0.50 | 1.225 | 1.3 | 0.414 |
| 5G-NIDD α=0.75 | 0.916 | 1.3 | 0.298 |
At α=0.25 one client often holds more than half of a class's β=1 weight.
53/162 class-cells have `|S_j|<10`; their mean top-weight share is 0.463 versus
0.364 when all clients contribute. Both modes share the same `S_j`, so the
participant rule is not the cause of the *difference*; the weight distribution
inside `S_j` is.
### 2.7 Displacement vs detection
Mean `||Cbar_j(β)−Cbar_j(0)||` on fixed round-0 snapshots rises monotonically
with β in all six conditions (e.g. X-IIoTID α=0.25: 1.48, 2.40, 3.00, 3.42).
Across the 30 condition-β cells the descriptive Pearson correlation between
displacement and test rare-F1 delta is **−0.48**. This is descriptive and
confounded; displacement is a representation metric, not detection quality.
**Inferred, not causal.**
### 2.8 Reproduced baseline vs the paper
Paper “PROTEAN (Ours)” macro accuracy (mean per-class recall), as quoted in
`kisikisithesis/RESTORE_VERIFICATION_REPORT.md` §12 (the paper PDF is not in the
repository), vs the reproduced baseline's mean per-class recall:
| α | X-IIoTID reproduced | paper | 5G-NIDD reproduced | paper |
|---|---|---|---|---|
| 0.25 | 83.78 | 92.67 | 88.37 | 85.83 |
| 0.50 | 86.56 | 93.62 | 89.76 | 82.50 |
| 0.75 | 87.61 | 93.43 | 90.41 | 79.97 |
X-IIoTID reproduces 5–9 points below the paper; 5G-NIDD 3–10 points above. The
cause is **unknown** (split, seeds, preprocessing and the exact definition of
“macro accuracy” are not fully specified; no official code). Every
proposed-vs-baseline statement is conditional on this reproduced baseline.
### 2.9 Diagnosis summary
Measured: the loss is concentrated at α=0.25 (5G-NIDD at all seeds for β=1);
it is class- and client-specific; response is non-monotone; weight
concentration is highest exactly where the loss is highest; the deployed
selection metric disagrees with the global headline in 2/6 conditions while
global validation agrees in 6/6; displacement grows monotonically while
detection does not.
Hypotheses (not measured): high-support prototypes may be good for majority
traffic but misplaced for rare attacks; end-to-end prototype-alignment feedback
may amplify the direct weighting effect; three seeds cannot resolve the
between-β effect.
---
## 3. Phase 3 — candidate modifications
Full catalogue with mechanism, motivating evidence, definition change, cost and
failure modes: `exploration_v2/CANDIDATES.md`. Summary:
| id | mechanism | changes the method definition? | expected |
|---|---|---|---|
| V1 β=1.5 | extended β grid, stronger concentration | no (same operator, wider range) | may help where β=1 was the boundary; likely hurts α=0.25 |
| V2 shrink λ=0.5 | `w = 0.5·equal + 0.5·N^β` | yes (new operator) | bound the concentration tail |
| V3 cap 4×median | clip raw weights at 4× median | yes (new operator) | bound single-client dominance |
| V4 cap 2×median | tighter cap | yes (same family) | **expected negative** (approaches equal weighting) |
| V5 min support 10 | drop clients with `N_ij<10`, equal mean | yes (participant-rule change) | **expected negative** (removes rare evidence) |
| V6 log weight | `w ∝ log(1+N)` | yes (new weight function) | may over-equalise |
| C7 global selection metric | select β by global validation rare-F1 | no (selection procedure only) | fixes the two ranking mismatches |
| C8 finer β grid | more β points | no (candidate set only) | not run; coarse-grid noise is already large |
New implementations are **copies inside `exploration_v2/`**; the original code is
untouched. 14 unit tests verify that `original_protean` is bit-for-bit the
project implementation, that `support_weighted` (including β=0) delegates
exactly, and that a single contributor, equal supports, missing classes and
invalid parameters behave as documented.
**Exploratory result for C7 on the old grid (measured, not confirmatory).**
Selecting β by validation *global* rare-F1 instead of the deployed per-client
mean changes the selected β in X-IIoTID α=0.25 (0→0.50) and 5G-NIDD α=0.50
(1.00→0). Mean test rare-F1 difference versus the original baseline across the
six conditions is +0.0082 with global selection versus +0.0052 with the deployed
selection; the 5G-NIDD α=0.50 loss shrinks from −0.0086 to −0.0021 and the
X-IIoTID α=0.25 difference turns from −0.0007 to +0.0078. These are descriptive
numbers from the same three burned seeds, not a confirmation, and they must not
be presented as a new-run result.
---
## 4. Phase 4/5 — pre-registered execution
- `PLAN.md` frozen at 2026-10-30T04:07:47Z (hash above).
- Screening: 2 conditions (X-IIoTID α=0.25 seed 0; 5G-NIDD α=0.5 seed 0),
6 variants. Screening references: the current-environment original baseline
(2 runs, amendment A2) and the current-environment 1-thread β grid from
`results/sweep/`; restored-grid values are a sensitivity reference.
- Ranking metric: mean over the two conditions of the paired validation
difference in **global** rare-class macro-F1 versus the original. Secondary:
the deployed per-client-mean metric.
- Confirmation: top 2 variants + original + β=0.5 + β=1 on **new seeds 3 and 4**
in both conditions (20 runs). All new-seed manifests were generated with the
frozen partition procedure inside `exploration_v2/partitions/`.
- Video/audio hyperparameters are fixed a priori; no tuning on screening
outcomes. Budget: 34 planned runs, conservative capacity ≈42 at the allowed
concurrency, 96 h cap.
- Deviations: A1 (driver path bug, no scientific run started) and A2
(in-environment baseline references). Both dated in
`exploration_v2/PLAN_AMENDMENTS.md`; engineering fixes are in `ISSUES.md`.
**Pairing verification (measured).** For each screening condition, all seven
runs (baseline + six variants) have an identical manifest content hash, an
identical preprocessing hash, a byte-identical round-0 prototype snapshot, and
an identical `N_ij` support matrix. The round-0 snapshot is recorded before the
first aggregation, so identical snapshots confirm that the only difference
between the runs is the aggregation operator. Hashes:
`exploration_v2/analysis/screening_pairing_check.csv`.
<!-- AUTO_RESULTS_START -->
## 5. Results of the new runs
### 5.1 Screening ranking (validation only)
Top-1 minus top-2 validation score (selection margin): 0.020278 (top2 = ['v_shrink__b1__lam0.5', 'v_cap__b1__c4']).
Score = mean over S1,S2 of the paired validation difference in global rare-class macro-F1 versus the in-environment original baseline. `rank_secondary` uses the deployed per-client-mean metric.
| rank | variant | β | λ | cap | min N | score | min cond Δ | S1 Δ | S2 Δ | rank_secondary |
|---|---|---|---|---|---|---|---|---|---|
| 1 | `v_shrink__b1__lam0.5` | 1.00 | 0.50 | | | 0.0267 | 0.0142 | 0.0391 | 0.0142 | 1 |
| 2 | `v_cap__b1__c4` | 1.00 | | 4.0 | | 0.0064 | 0.0035 | 0.0035 | 0.0093 | 2 |
| 3 | `v_cap__b1__c2` | 1.00 | | 2.0 | | -0.0024 | -0.0056 | -0.0056 | 0.0007 | 3 |
| 4 | `v_logweight` | 0.00 | | | | -0.0139 | -0.0346 | 0.0067 | -0.0346 | 5 |
| 5 | `v_minsup__t10` | 0.00 | | | 10 | -0.0302 | -0.0438 | -0.0166 | -0.0438 | 6 |
| 6 | `v_beta__b1.5` | 1.50 | | | | -0.0485 | -0.0918 | -0.0918 | -0.0052 | 4 |
### 5.2 Confirmatory runs, per seed
| dataset | α | seed | config | val rare-F1 | test rare-F1 | Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 |
|---|---|---|---|---|---|---|---|---|---|
| xiiotid | 0.25 | 3 | `v_shrink__b1__lam0.5` | 0.6605 | 0.6611 | 0.0085 | 0.0094 | 0.0261 | 0.0241 |
| xiiotid | 0.25 | 3 | `v_cap__b1__c4` | 0.6930 | 0.6929 | 0.0410 | 0.0413 | 0.0587 | 0.0559 |
| xiiotid | 0.25 | 3 | `support_weighted__beta0.5` | 0.6695 | 0.6697 | 0.0175 | 0.0181 | 0.0352 | 0.0327 |
| xiiotid | 0.25 | 3 | `support_weighted__beta1` | 0.6343 | 0.6370 | -0.0177 | -0.0146 | 0.0000 | 0.0000 |
| xiiotid | 0.25 | 4 | `v_shrink__b1__lam0.5` | 0.7155 | 0.7164 | 0.0329 | 0.0332 | 0.1713 | 0.1745 |
| xiiotid | 0.25 | 4 | `v_cap__b1__c4` | 0.6612 | 0.6628 | -0.0214 | -0.0204 | 0.1170 | 0.1209 |
| xiiotid | 0.25 | 4 | `support_weighted__beta0.5` | 0.6745 | 0.6745 | -0.0082 | -0.0087 | 0.1302 | 0.1326 |
| xiiotid | 0.25 | 4 | `support_weighted__beta1` | 0.5443 | 0.5419 | -0.1384 | -0.1413 | 0.0000 | 0.0000 |
| 5gnidd | 0.50 | 3 | `v_shrink__b1__lam0.5` | 0.8314 | 0.8276 | 0.0269 | 0.0276 | -0.0888 | -0.0895 |
| 5gnidd | 0.50 | 3 | `v_cap__b1__c4` | 0.8064 | 0.8044 | 0.0020 | 0.0044 | -0.1138 | -0.1128 |
| 5gnidd | 0.50 | 3 | `support_weighted__beta0.5` | 0.8389 | 0.8370 | 0.0344 | 0.0370 | -0.0813 | -0.0801 |
| 5gnidd | 0.50 | 3 | `support_weighted__beta1` | 0.9202 | 0.9172 | 0.1158 | 0.1172 | 0.0000 | 0.0000 |
| 5gnidd | 0.50 | 4 | `v_shrink__b1__lam0.5` | 0.9297 | 0.9265 | 0.0038 | 0.0032 | 0.0285 | 0.0262 |
| 5gnidd | 0.50 | 4 | `v_cap__b1__c4` | 0.9231 | 0.9216 | -0.0029 | -0.0017 | 0.0219 | 0.0213 |
| 5gnidd | 0.50 | 4 | `support_weighted__beta0.5` | 0.9258 | 0.9255 | -0.0001 | 0.0023 | 0.0246 | 0.0253 |
| 5gnidd | 0.50 | 4 | `support_weighted__beta1` | 0.9012 | 0.9002 | -0.0248 | -0.0231 | 0.0000 | 0.0000 |
### 5.3 Confirmatory summary
A variant “improves” under the pre-registered rule only if the mean validation difference over the four pairs is positive **and** all four are positive.
| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | mean Δtest vs current best | improved |
|---|---|---|---|---|---|---|---|---|
| `v_shrink__b1__lam0.5` | 0.0180 | 0.0141 | 4/4 | 0.0184 | 0.0143 | 0.0343 | 0.0338 | -0.0153 | -0.0155 | True |
| `v_cap__b1__c4` | 0.0047 | 0.0263 | 2/4 | 0.0059 | 0.0258 | 0.0210 | 0.0213 | -0.0286 | -0.0279 | False |
### 5.4 Reference configurations on the confirmation seeds
| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive |
|---|---|---|---|---|---|---|
| `support_weighted__beta0.5` | 0.0109 | 0.0190 | 2 | 0.0122 | 0.0199 | 3 |
| `support_weighted__beta1` | -0.0163 | 0.1040 | 1 | -0.0155 | 0.1057 | 1 |
### 5.5 Reference consistency (environment check)
| condition | new original val | restored original val | sweep β=0 val | new − restored | new − sweep β=0 |
|---|---|---|---|---|---|
| S1 | 0.6380 | 0.6416 | 0.6433 | -0.0035 | -0.0052 |
| S2 | 0.8615 | 0.8694 | 0.8677 | -0.0079 | -0.0061 |
### 5.6 Wall time and memory
| phase | run | status | wall h | peak GiB |
|---|---|---|---|---|
| screening | `xiiotid__alpha0.25__seed0__original_protean` | done | 3.02 | 8.38 |
| screening | `5gnidd__alpha0.5__seed0__original_protean` | done | 4.67 | 10.30 |
| screening | `xiiotid__alpha0.25__seed0__v_beta__b1.5` | done | 3.07 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5` | done | 3.05 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c4` | done | 3.06 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c2` | done | 3.04 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_minsup__t10` | done | 3.03 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_logweight` | done | 2.96 | 8.38 |
| screening | `5gnidd__alpha0.5__seed0__v_beta__b1.5` | done | 4.88 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5` | done | 4.85 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c4` | done | 4.55 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c2` | done | 4.41 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_minsup__t10` | done | 4.51 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_logweight` | done | 4.82 | 10.31 |
| confirm | `xiiotid__alpha0.25__seed3__original_protean` | done | 3.08 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta0.5` | done | 3.08 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta1` | done | 3.12 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5` | done | 2.97 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__v_cap__b1__c4` | done | 2.95 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed4__original_protean` | done | 3.11 | 12.75 |
| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` | done | 3.10 | 12.76 |
| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta1` | done | 3.20 | 12.76 |
| confirm | `xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5` | done | 3.18 | 12.76 |
| confirm | `xiiotid__alpha0.25__seed4__v_cap__b1__c4` | done | 2.82 | 12.76 |
| confirm | `5gnidd__alpha0.5__seed3__original_protean` | done | 4.41 | 9.04 |
| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta0.5` | done | 4.48 | 9.07 |
| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta1` | done | 4.66 | 9.04 |
| confirm | `5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5` | done | 4.57 | 9.06 |
| confirm | `5gnidd__alpha0.5__seed3__v_cap__b1__c4` | done | 4.73 | 9.05 |
| confirm | `5gnidd__alpha0.5__seed4__original_protean` | done | 4.81 | 11.12 |
| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta0.5` | done | 4.77 | 11.16 |
| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta1` | done | 4.88 | 11.13 |
| confirm | `5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5` | done | 4.79 | 11.15 |
| confirm | `5gnidd__alpha0.5__seed4__v_cap__b1__c4` | done | 4.56 | 11.14 |
### 5.7 Failed or missing runs
None.
<!-- AUTO_RESULTS_END -->
### 5.8 What the confirmation actually shows
**Screening (validation only, seed 0).** `v_shrink` (λ=0.5) ranked first
(+0.0267 mean over S1/S2 vs the in-environment baseline) and `v_cap4` second
(+0.0064), with a selection margin of 0.0203 between them. The remaining four
candidates were at or below the baseline on the two-condition mean: `v_cap2` −0.0024,
`v_logweight` −0.0139, `v_minsup` −0.0302, `v_beta1.5` −0.0485.
The same screening scores expressed against the current method's best β on each
condition (β=0.75 for S1, +0.0310 vs baseline; β=1.00 for S2, +0.0147; mean
+0.0229):
| variant | Δ vs original (mean) | Δ vs current-method best (mean) |
|---|---|---|
| `v_shrink` | +0.0267 | **+0.0038** |
| `v_cap4` | +0.0064 | −0.0165 |
| `v_cap2` | −0.0024 | −0.0253 |
| `v_logweight` | −0.0139 | −0.0368 |
| `v_minsup10` | −0.0302 | −0.0531 |
| `v_beta1.5` | −0.0485 | −0.0714 |
Only `v_shrink` is above the current method's validation-selected best in this
screening comparison, and by a small margin (+0.0038). The confirmation then
shows that this small margin does not survive as a general advantage (§5.8).
**Top candidate, `v_shrink` (λ=0.5).** It meets the pre-registered improvement
rule against the original baseline:
| statistic (4 confirmatory pairs) | validation | test |
|---|---|---|
| mean paired difference vs baseline | **+0.0180** | **+0.0184** |
| sample sd of paired differences | 0.0141 | 0.0143 |
| positive pairs | **4/4** | **4/4** |
| X-IIoTID α=0.25, mean (seeds 3,4) | +0.0207 (+0.0085, +0.0329) | +0.0213 |
| 5G-NIDD α=0.50, mean (seeds 3,4) | +0.0153 (+0.0269, +0.0038) | +0.0154 |
The direction is the same on every confirmatory seed and both conditions, and
the test differences track the validation differences closely (mean
validation−test = −0.0004). That is the strongest statement this design
supports: a modest, direction-consistent gain over the **baseline** on the two
new seeds.
**Second candidate, `v_cap4`.** It does **not** meet the rule: mean validation
difference +0.0047, sd 0.0263, only **2/4** positive. The gain comes entirely
from X-IIoTID seed 3 (+0.0410) and reverses on seed 4 (−0.0214); on 5G-NIDD it
is +0.0020 / −0.0029. Mean test difference +0.0059 with the same mixed pattern.
So the capping candidate is a negative/mixed result and remains in the tables.
**Versus the current proposed method.** Comparing each variant with the best
validation score among {original, β=0.5, β=1} on the same pair:
- `v_shrink`: mean **−0.0153** (sd 0.0521, 2/4 pairs better). It improves on
both seed-4 pairs (+0.0329 X-IIoTID, +0.0038 5G-NIDD) but loses on seed 3,
mainly 5G-NIDD (−0.0888) where β=1 gains +0.1158 over the baseline.
- `v_cap4`: mean **−0.0286** (sd 0.0597, 1/4 pairs better).
So `v_shrink` is **more stable than the current method but does not dominate
it**. The current method's β=1 has a much higher peak on 5G-NIDD seed 3 and a
much deeper failure on X-IIoTID seed 4 (−0.1384). This is a measured trade-off,
not a win for either method.
**Reference behaviour on the new seeds.** β=0.5 has mean validation +0.0109
(sd 0.0190, 2/4 positive) and test +0.0122 (3/4). β=1 has mean validation
−0.0163 (sd 0.1040, 1/4 positive) and test −0.0155 (1/4): X-IIoTID −0.0780
(−0.0177, −0.1384), 5G-NIDD +0.0455 (+0.1158, −0.0248). The large β=1 spread is
the clearest measured instance of the seed sensitivity already visible in the
old grid.
**No significance claim.** Four paired observations (two conditions × two
seeds), no inferential test, and the sample standard deviations are of the same
order as the means for most statistics. `v_shrink` passing the pre-registered
rule means “positive on these four pairs”, not “superior in general”.
### 5.9 Where the `v_shrink` gain comes from (descriptive)
Condition-level means hide heterogeneity. Pooling the two confirmation seeds
per dataset, test-F1 class deltas for `v_shrink` versus the original:
| dataset | largest gains | losses |
|---|---|---|
| X-IIoTID α=0.25 | C&C +0.072, Weaponization +0.061, Tampering +0.054 | Exploitation −0.010, Exfiltration −0.003, Lateral −0.002 |
| 5G-NIDD α=0.50 | ICMPFlood +0.069 (seed 3 +0.143, seed 4 −0.004), SlowrateDoS +0.026, HTTPFlood +0.013 | SYNFlood −0.014 (both seeds) |
Per-client validation (10 clients × 2 seeds per dataset): X-IIoTID mean +0.0295
but only **9 improve / 11 degrade**; 5G-NIDD mean +0.0119 with **12 improve /
4 degrade**. So the positive condition-level result is not “every client wins”:
on X-IIoTID the mean is pulled up by a few large attack-class/client gains while
the majority of client-seed pairs lose a little.
The same view explains the other results:
- `v_cap4` on X-IIoTID gains Tampering +0.157 but loses **Normal −0.234**
(seed 4 −0.475) and C&C −0.123; client pairs are 8 improve / 12 degrade. That
is why its average is mixed and flips sign between seeds.
- β=1 on X-IIoTID loses Tampering −0.366 and Normal −0.245; client pairs are
3 improve / 17 degrade. On 5G-NIDD it gains ICMPFlood +0.200 and 11/5 client
pairs improve, but seed 4 reverses the aggregate. This is the microscale
origin of the large β=1 seed variance.
Full rows: `analysis/confirm_per_class_variant.csv` and
`analysis/confirm_per_client_variant.csv`.
---
## 6. Negative results, operational deviations and honesty notes
### 6.1 Negative and mixed results (measured)
- **`v_cap4` did not improve** under the pre-registered rule: mean validation
+0.0047, sd 0.0263, 2/4 positive; the gain is seed-3 only (+0.0410 on
X-IIoTID, −0.0214 on seed 4). It stays in the tables as a mixed/negative
result.
- **Expected-negative controls behaved as expected but not identically.**
`v_minsup10` was worst on screening (−0.0302; S1 −0.0166, S2 −0.0438),
consistent with removing low-support rare-class evidence. `v_cap2` was near
zero (−0.0024). `v_logweight` was mixed (+0.0067 S1, −0.0346 S2) and overall
negative. `v_beta1.5` was the worst (−0.0485; S1 −0.0918), confirming that
extending β beyond 1 concentrates weight in the wrong direction at α=0.25.
- **The current method's β=1 is strongly seed-dependent** on the new seeds:
mean validation −0.0163, sd 0.1040, 1/4 positive. It ranges from +0.1158
(5G-NIDD seed 3) to −0.1384 (X-IIoTID seed 4). β=0.5 is more stable
(+0.0109, sd 0.0190) but weaker.
- **The analysis-level selection-metric candidate (C7) was not confirmed.**
Its exploratory gain (+0.0082 vs +0.0052 mean test difference on the old
grid) uses the burned seeds and is not part of the new confirmation. It
remains a hypothesis for future work.
- **No variant was tested at α=0.75.** The confirmation conditions were fixed
to α=0.25 (X-IIoTID) and α=0.50 (5G-NIDD); the report does not extrapolate.
### 6.2 Operational deviations (with timestamps)
The ranking was frozen at 2026-10-31T05:03:55Z before any confirmatory run, and
was never recomputed. The following operational events are recorded because they
changed execution, not the scientific design:
| when (UTC) | id | event | action |
|---|---|---|---|
| 2026-10-30T04:10Z | A1 | wrapper resolved repo root one level too high; six attempts exited 127 before Python started | path fixed; attempts archived; no science |
| 2026-10-30T04:32Z | A2 | restored grid came from another host; environment difference measured at median 0.0022 / max 0.0127 test rare-F1 | +2 in-environment baseline runs; 34 runs total |
| 2026-10-31T08:35Z | A3 | two seed-4 reference runs OOM-killed twice at 11 GiB | requeued, `max_procs` 3→2, OOM→137 mapping |
| 2026-10-31T09:50Z | A4 | three OOM kills total; user asked to prevent further OOM | X-IIoTID cap 11→13 GiB, `max_procs=2`, host `MemAvailable ≥ 12 GiB` guard |
| 2026-10-31T14:12Z | A5 | retries blocked by stale systemd unit names, misread as OOM | `reset-failed` before launch; failed-start guard |
| 2026-10-31T09:41/11:08Z | A6 | editing `run_one_v2.sh` while two wrappers were running caused exit 127 after training finished | both retried; archived first attempts are bit-identical to the final runs |
| 2026-10-31T22:35Z | — | driver wrote `DONE` `status=complete` | 34/34 done, 0 terminal failures, 66.4 h |
The A6 retries are a free reproducibility check: for both affected seed-3
variants, the archived first attempt and the final run have identical
`val_global_metrics.json`, `test_global_metrics.json`, `val_selection.json` and
a byte-identical round-0 snapshot. The retry did not change any result.
A partial `DONE` (`status=stopped`) was produced at 2026-10-31T14:07Z during the
A5 incident; it was archived to `logs/DONE.partial_20261031T1412Z` and the run
resumed. The final `DONE` is `complete`. All failed attempts are kept under
`results/attempts/` and the pre-campaign bug attempts under
`results/_driver_bug_20261030T0410Z/`.
### 6.3 Honesty notes
- Screening six variants raises the chance that the top one is lucky; the
confirmation on seeds never used in any analysis exists for that reason.
- No test metric of any new run was read before the ranking was frozen. The
ranking code opened only `val_*` files (see `ACCESS_LOG.txt`); the
post-freeze analysis read test metrics only after `ranking.sha256` existed.
- No significance test is performed and no superiority is claimed.
- A modification that did not improve is reported as such (`v_cap4`, and the
four candidates that never reached confirmation).
## 7. Limitations
1. Two datasets, two conditions, ten clients, one architecture and one training
configuration; only α=0.25 (X-IIoTID) and α=0.50 (5G-NIDD) reached
confirmation.
2. Confirmation has **four paired observations** (two conditions × two seeds).
The sample standard deviations are of the same order as the means; no
inferential test and no generalization claim.
3. The `S_j` participant rule remains a reproduction assumption; all results are
conditional on it.
4. The paper comparison is limited by the reproduction gap in §2.8.
5. Host/library differences between the restored grid and this environment are
why A2 added in-environment baselines. Residual environment effects for the
β-grid reference (taken from the 1-thread sweep) cannot be excluded.
6. The confirmation β-grid reference is truncated to β=0.5, β=1.0 and the
original (a β=0 proxy); the full five-point grid was not run on the new
seeds, for budget reasons stated before any confirmation run.
7. X-IIoTID seed-4 confirmation runs peaked at **12.76 GiB**, above the old
11 GiB cap and close to the final 13 GiB cap. The results are conditional on
the 13 GiB cap; a smaller cap would have killed them.
8. Rare-class support differs across seeds, so absolute metric levels are not
comparable across seeds; only paired differences are used.
9. The “current best” comparison uses the per-pair best of
{original, β=0.5, β=1} on validation. It is a fair proxy for the deployed
method on these seeds, but it is not the pooled-across-seeds selection used
in the original thesis protocol.
## 8. What the evidence supports and what it does not
**Supports (measured):**
- `v_shrink` (β=1, λ=0.5, half-way between equal and support-weighted) improved
over the **original baseline** on validation in 4/4 confirmation pairs:
mean +0.0180 (sd 0.0141), with test differences of the same size and
direction (mean +0.0184, 4/4). This is a modest, direction-consistent gain on
two unseen seeds, not a large one.
- `v_cap4` did **not** improve (2/4, mean +0.0047): capping at 4×median is not
enough, or is the wrong shape, on these conditions.
- The current method's β=1 is highly seed-sensitive (sd 0.1040 across the four
pairs, 1/4 positive); β=0.5 is more stable but weaker.
- From Phase 2: support weighting can lose substantially at high
heterogeneity; the loss is class- and client-specific; the deployed
per-client selection metric can disagree with the global headline metric;
displacement is not a detection-quality proxy.
**Does not support:**
- Any claim that `v_shrink` is better than the current proposed method. Against
the per-pair best of {original, β=0.5, β=1} it is **−0.0153** on validation
(2/4 pairs better), mainly because β=1 gains +0.1158 on 5G-NIDD seed 3.
- Any claim of statistical significance, superiority, or generalisation beyond
these two datasets, two conditions and two new seeds.
- Any universal or globally optimal β, or any causal α→β rule.
- Any causal link from prototype displacement to detection performance.
- Any statement that support weighting always helps or always hurts.
**Unknown:**
- Why β=1 is so seed-sensitive on 5G-NIDD seed 3 versus X-IIoTID seed 4.
- Whether a pooled-across-seeds selection would change the current-method
comparison.
- Whether `v_shrink` helps at α=0.75 or on other datasets; not tested.
- Whether the C7 selection-metric change would survive new seeds; not
confirmed.
exploration_v2/REPORT_DATA.md
+102
View File
@@ -0,0 +1,102 @@
# exploration_v2 — report data
Generated: 2026-11-02T03:36:03Z
This file is generated by `exploration_v2/scripts/analyze_results.py`. All numbers come from files on disk. Missing rows mean a run did not produce the metric.
## Plan and ranking freeze
| item | value |
|---|---|
| `PLAN.sha256` | `2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e PLAN.md` at 2026-10-30T04:07:47Z |
| `ranking.sha256` | `f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3 ranking.json` at 2026-10-31T05:03:55Z |
## Screening ranking (validation only)
Top-1 minus top-2 validation score (selection margin): 0.020278 (top2 = ['v_shrink__b1__lam0.5', 'v_cap__b1__c4']).
Score = mean over S1,S2 of the paired validation difference in global rare-class macro-F1 versus the original baseline.
| rank | variant | β | λ | cap | min N | score | min cond Δ | S1 Δ | S2 Δ | secondary rank |
|---|---|---|---|---|---|---|---|---|---|---|
| 1 | `v_shrink__b1__lam0.5` | 1.00 | 0.50 | | | 0.0267 | 0.0142 | 0.0391 | 0.0142 | 1 |
| 2 | `v_cap__b1__c4` | 1.00 | | 4.0 | | 0.0064 | 0.0035 | 0.0035 | 0.0093 | 2 |
| 3 | `v_cap__b1__c2` | 1.00 | | 2.0 | | -0.0024 | -0.0056 | -0.0056 | 0.0007 | 3 |
| 4 | `v_logweight` | 0.00 | | | | -0.0139 | -0.0346 | 0.0067 | -0.0346 | 5 |
| 5 | `v_minsup__t10` | 0.00 | | | 10 | -0.0302 | -0.0438 | -0.0166 | -0.0438 | 6 |
| 6 | `v_beta__b1.5` | 1.50 | | | | -0.0485 | -0.0918 | -0.0918 | -0.0052 | 4 |
## Confirmatory results (per seed)
| dataset | α | seed | config | val rare-F1 | test rare-F1 | Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 |
|---|---|---|---|---|---|---|---|---|---|
| xiiotid | 0.25 | 3 | `v_shrink__b1__lam0.5` | 0.6605 | 0.6611 | 0.0085 | 0.0094 | 0.0261 | 0.0241 |
| xiiotid | 0.25 | 3 | `v_cap__b1__c4` | 0.6930 | 0.6929 | 0.0410 | 0.0413 | 0.0587 | 0.0559 |
| xiiotid | 0.25 | 3 | `support_weighted__beta0.5` | 0.6695 | 0.6697 | 0.0175 | 0.0181 | 0.0352 | 0.0327 |
| xiiotid | 0.25 | 3 | `support_weighted__beta1` | 0.6343 | 0.6370 | -0.0177 | -0.0146 | 0.0000 | 0.0000 |
| xiiotid | 0.25 | 4 | `v_shrink__b1__lam0.5` | 0.7155 | 0.7164 | 0.0329 | 0.0332 | 0.1713 | 0.1745 |
| xiiotid | 0.25 | 4 | `v_cap__b1__c4` | 0.6612 | 0.6628 | -0.0214 | -0.0204 | 0.1170 | 0.1209 |
| xiiotid | 0.25 | 4 | `support_weighted__beta0.5` | 0.6745 | 0.6745 | -0.0082 | -0.0087 | 0.1302 | 0.1326 |
| xiiotid | 0.25 | 4 | `support_weighted__beta1` | 0.5443 | 0.5419 | -0.1384 | -0.1413 | 0.0000 | 0.0000 |
| 5gnidd | 0.50 | 3 | `v_shrink__b1__lam0.5` | 0.8314 | 0.8276 | 0.0269 | 0.0276 | -0.0888 | -0.0895 |
| 5gnidd | 0.50 | 3 | `v_cap__b1__c4` | 0.8064 | 0.8044 | 0.0020 | 0.0044 | -0.1138 | -0.1128 |
| 5gnidd | 0.50 | 3 | `support_weighted__beta0.5` | 0.8389 | 0.8370 | 0.0344 | 0.0370 | -0.0813 | -0.0801 |
| 5gnidd | 0.50 | 3 | `support_weighted__beta1` | 0.9202 | 0.9172 | 0.1158 | 0.1172 | 0.0000 | 0.0000 |
| 5gnidd | 0.50 | 4 | `v_shrink__b1__lam0.5` | 0.9297 | 0.9265 | 0.0038 | 0.0032 | 0.0285 | 0.0262 |
| 5gnidd | 0.50 | 4 | `v_cap__b1__c4` | 0.9231 | 0.9216 | -0.0029 | -0.0017 | 0.0219 | 0.0213 |
| 5gnidd | 0.50 | 4 | `support_weighted__beta0.5` | 0.9258 | 0.9255 | -0.0001 | 0.0023 | 0.0246 | 0.0253 |
| 5gnidd | 0.50 | 4 | `support_weighted__beta1` | 0.9012 | 0.9002 | -0.0248 | -0.0231 | 0.0000 | 0.0000 |
## Confirmatory summary
| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | mean Δtest vs current best | improved per plan |
|---|---|---|---|---|---|---|---|---|
| `v_shrink__b1__lam0.5` | 0.0180 | 0.0141 | 4/4 | 0.0184 | 0.0143 | 0.0343 | 0.0338 | -0.0153 | -0.0155 | True |
| `v_cap__b1__c4` | 0.0047 | 0.0263 | 2/4 | 0.0059 | 0.0258 | 0.0210 | 0.0213 | -0.0286 | -0.0279 | False |
## Reference configurations on the confirmation seeds
| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive |
|---|---|---|---|---|---|---|
| `support_weighted__beta0.5` | 0.0109 | 0.0190 | 2 | 0.0122 | 0.0199 | 3 |
| `support_weighted__beta1` | -0.0163 | 0.1040 | 1 | -0.0155 | 0.1057 | 1 |
## Reference consistency (environment check)
| condition | new original val | restored original val | sweep β=0 val | new − restored | new − sweep β=0 |
|---|---|---|---|---|---|
| S1 | 0.6380 | 0.6416 | 0.6433 | -0.0035 | -0.0052 |
| S2 | 0.8615 | 0.8694 | 0.8677 | -0.0079 | -0.0061 |
## Wall time and memory (new runs)
| phase | run | status | wall h | peak GiB |
|---|---|---|---|---|
| screening | `xiiotid__alpha0.25__seed0__original_protean` | done | 3.02 | 8.38 |
| screening | `5gnidd__alpha0.5__seed0__original_protean` | done | 4.67 | 10.30 |
| screening | `xiiotid__alpha0.25__seed0__v_beta__b1.5` | done | 3.07 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5` | done | 3.05 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c4` | done | 3.06 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_cap__b1__c2` | done | 3.04 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_minsup__t10` | done | 3.03 | 8.38 |
| screening | `xiiotid__alpha0.25__seed0__v_logweight` | done | 2.96 | 8.38 |
| screening | `5gnidd__alpha0.5__seed0__v_beta__b1.5` | done | 4.88 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5` | done | 4.85 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c4` | done | 4.55 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_cap__b1__c2` | done | 4.41 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_minsup__t10` | done | 4.51 | 10.29 |
| screening | `5gnidd__alpha0.5__seed0__v_logweight` | done | 4.82 | 10.31 |
| confirm | `xiiotid__alpha0.25__seed3__original_protean` | done | 3.08 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta0.5` | done | 3.08 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__support_weighted__beta1` | done | 3.12 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5` | done | 2.97 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed3__v_cap__b1__c4` | done | 2.95 | 10.48 |
| confirm | `xiiotid__alpha0.25__seed4__original_protean` | done | 3.11 | 12.75 |
| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta0.5` | done | 3.10 | 12.76 |
| confirm | `xiiotid__alpha0.25__seed4__support_weighted__beta1` | done | 3.20 | 12.76 |
| confirm | `xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5` | done | 3.18 | 12.76 |
| confirm | `xiiotid__alpha0.25__seed4__v_cap__b1__c4` | done | 2.82 | 12.76 |
| confirm | `5gnidd__alpha0.5__seed3__original_protean` | done | 4.41 | 9.04 |
| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta0.5` | done | 4.48 | 9.07 |
| confirm | `5gnidd__alpha0.5__seed3__support_weighted__beta1` | done | 4.66 | 9.04 |
| confirm | `5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5` | done | 4.57 | 9.06 |
| confirm | `5gnidd__alpha0.5__seed3__v_cap__b1__c4` | done | 4.73 | 9.05 |
| confirm | `5gnidd__alpha0.5__seed4__original_protean` | done | 4.81 | 11.12 |
| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta0.5` | done | 4.77 | 11.16 |
| confirm | `5gnidd__alpha0.5__seed4__support_weighted__beta1` | done | 4.88 | 11.13 |
| confirm | `5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5` | done | 4.79 | 11.15 |
| confirm | `5gnidd__alpha0.5__seed4__v_cap__b1__c4` | done | 4.56 | 11.14 |
## Failed or missing runs
None.
exploration_v2/SETUP_CHECK.md
+19
View File
@@ -0,0 +1,19 @@
# exploration_v2 setup check
Generated: 2026-11-02T03:38:12.876063Z
| check | result | detail |
|---|---|---|
| plan_hash_matches | PASS | 2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e |
| no_tracked_modifications | PASS | |
| planningawal_unchanged | PASS | |
| no_raw_dataset_in_repo | PASS | |
| original_tests_pass | PASS | ... [100%] |
| variant_tests_pass | PASS | .............. [100%] |
| confirm_seeds_new | PASS | restored seeds=[0, 1, 2] |
| manifest_ok_5gnidd__alpha0.5__seed4.json | PASS | seed=4 |
| manifest_ok_xiiotid__alpha0.25__seed4.json | PASS | seed=4 |
| manifest_ok_xiiotid__alpha0.25__seed3.json | PASS | seed=3 |
| manifest_ok_5gnidd__alpha0.5__seed3.json | PASS | seed=3 |
Overall: PASS
exploration_v2/STATUS_AT_SESSION_END.md
+55
View File
@@ -0,0 +1,55 @@
# Final status — exploration_v2
UTC: 2026-11-02T03:36Z
Repository: `/home/willtanoe/project/prototype-agregation`
## Outcome
- Driver finished 2026-11-01T22:35:36Z; `DONE` says `status=complete`.
- **34/34 scientific runs done**, 0 terminal failures.
- Wall time: **66.4 h** of the 96 h budget.
- Screening: 14 runs (12 variants + 2 in-environment baselines).
- Confirmation: 20 runs (top-2 variants + original + β=0.5 + β=1 on seeds 3
and 4, two conditions).
- Frozen plan hash: `2118f31b30270e1d806cf7acdac2249da625646f2d9ce3a0cee96307317c978e`.
- Frozen ranking hash: `f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3`.
## Headline result
- `v_shrink` (β=1, λ=0.5) improved over the original baseline on validation in
**4/4** confirmation pairs: mean **+0.0180** (sd 0.0141); test mean +0.0184
(4/4). It therefore meets the pre-registered improvement rule.
- `v_cap4` did **not**: mean +0.0047, 2/4.
- Neither beats the current method's per-pair best
{original, β=0.5, β=1}: `v_shrink` mean **−0.0153** on validation (2/4 pairs
better), mainly because β=1 gains +0.1158 on 5G-NIDD seed 3.
- β=1 was highly seed-sensitive (mean −0.0163, sd 0.1040, 1/4 positive).
## Files
- `REPORT.md` — complete narrative (Phase 1, Phase 2, design, results,
negative results, deviations, limitations, support/does-not-support).
- `REPORT_DATA.md`, `analysis/*.csv` — all numeric tables.
- `PHASE12_ANALYSIS.md`, `CANDIDATES.md`, `PLAN.md`, `PLAN_CHECK.md`,
`PLAN_AMENDMENTS.md` (A1–A5), `ISSUES.md`, `HANDOFF.md`.
- `results/runs/` (34 COMPLETE), `results/attempts/` (archived failures),
`results/_driver_bug_20261030T0410Z/` (pre-campaign path bug).
- `ranking.json` / `ranking.sha256`, `ACCESS_LOG.txt` (52 validation-only
opens; zero test/per-client/history opens).
## Integrity
- No tracked file modified; `git diff --stat` empty; `planningawal/` unchanged.
- No raw dataset copied into the repository.
- No commit or push performed.
- No statistical significance or superiority claim is made.
## Open issues / limitations
- Four confirmatory pairs only; no inferential test.
- Confirmation conditions limited to X-IIoTID α=0.25 and 5G-NIDD α=0.50.
- X-IIoTID seed-4 runs peaked at 12.76 GiB, close to the 13 GiB cap adopted
after the OOM incident.
- The current-method reference on new seeds is truncated to β=0.5, β=1 and the
original (β=0 proxy).
- C7 (global selection metric) remains exploratory on burned seeds.
exploration_v2/analysis/confirm_paired.csv
+17
View File
@@ -0,0 +1,17 @@
dataset,alpha,seed,config,run_id,val_global_rare_f1,val_per_client_rare_f1,test_rare_f1,test_rare_recall,baseline_val_global,baseline_test_rare_f1,delta_val_vs_original,delta_test_vs_original,delta_val_vs_beta1,delta_test_vs_beta1,delta_val_vs_beta05,delta_test_vs_beta05,current_best_config,current_best_val_global,delta_val_vs_current_best,delta_test_vs_current_best
xiiotid,0.25,3,v_shrink__b1__lam0.5,xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,0.6604599138123004,0.6835769827947561,0.6611166614366842,0.834631327602874,0.6520038277014277,0.6516753883176936,0.008456086110872674,0.009441273118990678,0.026139916018731668,0.024068007268394864,-0.009056466053729562,-0.008628509118876138,support_weighted__beta0.5,0.66951637986603,-0.009056466053729562,-0.008628509118876138
xiiotid,0.25,3,v_cap__b1__c4,xiiotid__alpha0.25__seed3__v_cap__b1__c4,0.6930347111029873,0.6974933387696897,0.6929355556853871,0.8169158784862602,0.6520038277014277,0.6516753883176936,0.04103088340155958,0.041260167367693557,0.058714713309418576,0.05588690151709774,0.023518331236957346,0.02319038512982674,support_weighted__beta0.5,0.66951637986603,0.023518331236957346,0.02319038512982674
xiiotid,0.25,3,support_weighted__beta0.5,xiiotid__alpha0.25__seed3__support_weighted__beta0.5,0.66951637986603,0.6806478241186118,0.6697451705555604,0.8147471250008732,0.6520038277014277,0.6516753883176936,0.017512552164602235,0.018069782237866816,0.03519638207246123,0.032696516387271,0.0,0.0,support_weighted__beta0.5,0.66951637986603,0.0,0.0
xiiotid,0.25,3,support_weighted__beta1,xiiotid__alpha0.25__seed3__support_weighted__beta1,0.6343199977935687,0.6443454840057614,0.6370486541682894,0.7953041343152203,0.6520038277014277,0.6516753883176936,-0.017683829907858994,-0.014626734149404186,0.0,0.0,-0.03519638207246123,-0.032696516387271,support_weighted__beta0.5,0.66951637986603,-0.03519638207246123,-0.032696516387271
xiiotid,0.25,4,v_shrink__b1__lam0.5,xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,0.7155211309912932,0.5875322150240144,0.7163859069449825,0.8549205204777552,0.6826579058578224,0.6832331434606697,0.03286322513347084,0.03315276348431284,0.17126530669398643,0.17449076882237846,0.04105079335795048,0.04186336031153037,original_protean,0.6826579058578224,0.03286322513347084,0.03315276348431284
xiiotid,0.25,4,v_cap__b1__c4,xiiotid__alpha0.25__seed4__v_cap__b1__c4,0.6612274834490823,0.5895548490218416,0.662809680890142,0.8345629884018604,0.6826579058578224,0.6832331434606697,-0.021430422408740046,-0.02042346257052763,0.11697165915177554,0.12091454276753799,-0.01324285418426041,-0.011712865743310097,original_protean,0.6826579058578224,-0.021430422408740046,-0.02042346257052763
xiiotid,0.25,4,support_weighted__beta0.5,xiiotid__alpha0.25__seed4__support_weighted__beta0.5,0.6744703376333427,0.5946075229514161,0.6745225466334521,0.8293580156927381,0.6826579058578224,0.6832331434606697,-0.008187568224479636,-0.008710596827217532,0.13021451333603595,0.1326274085108481,0.0,0.0,original_protean,0.6826579058578224,-0.008187568224479636,-0.008710596827217532
xiiotid,0.25,4,support_weighted__beta1,xiiotid__alpha0.25__seed4__support_weighted__beta1,0.5442558242973068,0.42910855853255436,0.541895138122604,0.6966345934869704,0.6826579058578224,0.6832331434606697,-0.1384020815605156,-0.14133800533806562,0.0,0.0,-0.13021451333603595,-0.1326274085108481,original_protean,0.6826579058578224,-0.1384020815605156,-0.14133800533806562
5gnidd,0.5,3,v_shrink__b1__lam0.5,5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,0.831360062139045,0.8667377269112777,0.8276438788886984,0.9892275573304957,0.8044578347431527,0.7999980487126149,0.02690222739589232,0.027645830176083463,-0.08884958289572842,-0.08952434009919907,-0.007525129739533343,-0.009401964142520147,support_weighted__beta1,0.9202096450347734,-0.08884958289572842,-0.08952434009919907
5gnidd,0.5,3,v_cap__b1__c4,5gnidd__alpha0.5__seed3__v_cap__b1__c4,0.8064198040760951,0.8437023476357212,0.8043697046783258,0.9921040309667984,0.8044578347431527,0.7999980487126149,0.0019619693329424592,0.0043716559657108744,-0.11378984095867828,-0.11279851430957166,-0.0324653878024832,-0.032676138352892736,support_weighted__beta1,0.9202096450347734,-0.11378984095867828,-0.11279851430957166
5gnidd,0.5,3,support_weighted__beta0.5,5gnidd__alpha0.5__seed3__support_weighted__beta0.5,0.8388851918785784,0.8727486080528835,0.8370458430312185,0.9909209257334471,0.8044578347431527,0.7999980487126149,0.03442735713542566,0.03704779431860361,-0.08132445315619508,-0.08012237595667893,0.0,0.0,support_weighted__beta1,0.9202096450347734,-0.08132445315619508,-0.08012237595667893
5gnidd,0.5,3,support_weighted__beta1,5gnidd__alpha0.5__seed3__support_weighted__beta1,0.9202096450347734,0.9429509748552004,0.9171682189878975,0.9866951889574849,0.8044578347431527,0.7999980487126149,0.11575181029162074,0.11717017027528254,0.0,0.0,0.08132445315619508,0.08012237595667893,support_weighted__beta1,0.9202096450347734,0.0,0.0
5gnidd,0.5,4,v_shrink__b1__lam0.5,5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,0.9296850930111598,0.8880565443840929,0.9264737095074091,0.9657604065424229,0.9259273982940985,0.9232963015030996,0.003757694717061222,0.0031774080043095454,0.028523543149283515,0.026239840648658586,0.0038793926743494467,0.0009244400791833529,original_protean,0.9259273982940985,0.003757694717061222,0.0031774080043095454
5gnidd,0.5,4,v_cap__b1__c4,5gnidd__alpha0.5__seed4__v_cap__b1__c4,0.9230772912397688,0.884532344077232,0.9215621117759981,0.967188414469838,0.9259273982940985,0.9232963015030996,-0.0028501070543297535,-0.0017341897271014872,0.02191574137789254,0.021328242917247553,-0.0027284090970415287,-0.00398715765222768,original_protean,0.9259273982940985,-0.0028501070543297535,-0.0017341897271014872
5gnidd,0.5,4,support_weighted__beta0.5,5gnidd__alpha0.5__seed4__support_weighted__beta0.5,0.9258057003368103,0.88601978325071,0.9255492694282258,0.9668784318423409,0.9259273982940985,0.9232963015030996,-0.00012169795728822486,0.0022529679251261925,0.024644150474934068,0.025315400569475233,0.0,0.0,original_protean,0.9259273982940985,-0.00012169795728822486,0.0022529679251261925
5gnidd,0.5,4,support_weighted__beta1,5gnidd__alpha0.5__seed4__support_weighted__beta1,0.9011615498618762,0.8562921340984854,0.9002338688587506,0.9672615025827914,0.9259273982940985,0.9232963015030996,-0.024765848432222293,-0.02306243264434904,0.0,0.0,-0.024644150474934068,-0.025315400569475233,original_protean,0.9259273982940985,-0.024765848432222293,-0.02306243264434904
1 dataset alpha seed config run_id val_global_rare_f1 val_per_client_rare_f1 test_rare_f1 test_rare_recall baseline_val_global baseline_test_rare_f1 delta_val_vs_original delta_test_vs_original delta_val_vs_beta1 delta_test_vs_beta1 delta_val_vs_beta05 delta_test_vs_beta05 current_best_config current_best_val_global delta_val_vs_current_best delta_test_vs_current_best
2 xiiotid 0.25 3 v_shrink__b1__lam0.5 xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5 0.6604599138123004 0.6835769827947561 0.6611166614366842 0.834631327602874 0.6520038277014277 0.6516753883176936 0.008456086110872674 0.009441273118990678 0.026139916018731668 0.024068007268394864 -0.009056466053729562 -0.008628509118876138 support_weighted__beta0.5 0.66951637986603 -0.009056466053729562 -0.008628509118876138
3 xiiotid 0.25 3 v_cap__b1__c4 xiiotid__alpha0.25__seed3__v_cap__b1__c4 0.6930347111029873 0.6974933387696897 0.6929355556853871 0.8169158784862602 0.6520038277014277 0.6516753883176936 0.04103088340155958 0.041260167367693557 0.058714713309418576 0.05588690151709774 0.023518331236957346 0.02319038512982674 support_weighted__beta0.5 0.66951637986603 0.023518331236957346 0.02319038512982674
4 xiiotid 0.25 3 support_weighted__beta0.5 xiiotid__alpha0.25__seed3__support_weighted__beta0.5 0.66951637986603 0.6806478241186118 0.6697451705555604 0.8147471250008732 0.6520038277014277 0.6516753883176936 0.017512552164602235 0.018069782237866816 0.03519638207246123 0.032696516387271 0.0 0.0 support_weighted__beta0.5 0.66951637986603 0.0 0.0
5 xiiotid 0.25 3 support_weighted__beta1 xiiotid__alpha0.25__seed3__support_weighted__beta1 0.6343199977935687 0.6443454840057614 0.6370486541682894 0.7953041343152203 0.6520038277014277 0.6516753883176936 -0.017683829907858994 -0.014626734149404186 0.0 0.0 -0.03519638207246123 -0.032696516387271 support_weighted__beta0.5 0.66951637986603 -0.03519638207246123 -0.032696516387271
6 xiiotid 0.25 4 v_shrink__b1__lam0.5 xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5 0.7155211309912932 0.5875322150240144 0.7163859069449825 0.8549205204777552 0.6826579058578224 0.6832331434606697 0.03286322513347084 0.03315276348431284 0.17126530669398643 0.17449076882237846 0.04105079335795048 0.04186336031153037 original_protean 0.6826579058578224 0.03286322513347084 0.03315276348431284
7 xiiotid 0.25 4 v_cap__b1__c4 xiiotid__alpha0.25__seed4__v_cap__b1__c4 0.6612274834490823 0.5895548490218416 0.662809680890142 0.8345629884018604 0.6826579058578224 0.6832331434606697 -0.021430422408740046 -0.02042346257052763 0.11697165915177554 0.12091454276753799 -0.01324285418426041 -0.011712865743310097 original_protean 0.6826579058578224 -0.021430422408740046 -0.02042346257052763
8 xiiotid 0.25 4 support_weighted__beta0.5 xiiotid__alpha0.25__seed4__support_weighted__beta0.5 0.6744703376333427 0.5946075229514161 0.6745225466334521 0.8293580156927381 0.6826579058578224 0.6832331434606697 -0.008187568224479636 -0.008710596827217532 0.13021451333603595 0.1326274085108481 0.0 0.0 original_protean 0.6826579058578224 -0.008187568224479636 -0.008710596827217532
9 xiiotid 0.25 4 support_weighted__beta1 xiiotid__alpha0.25__seed4__support_weighted__beta1 0.5442558242973068 0.42910855853255436 0.541895138122604 0.6966345934869704 0.6826579058578224 0.6832331434606697 -0.1384020815605156 -0.14133800533806562 0.0 0.0 -0.13021451333603595 -0.1326274085108481 original_protean 0.6826579058578224 -0.1384020815605156 -0.14133800533806562
10 5gnidd 0.5 3 v_shrink__b1__lam0.5 5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5 0.831360062139045 0.8667377269112777 0.8276438788886984 0.9892275573304957 0.8044578347431527 0.7999980487126149 0.02690222739589232 0.027645830176083463 -0.08884958289572842 -0.08952434009919907 -0.007525129739533343 -0.009401964142520147 support_weighted__beta1 0.9202096450347734 -0.08884958289572842 -0.08952434009919907
11 5gnidd 0.5 3 v_cap__b1__c4 5gnidd__alpha0.5__seed3__v_cap__b1__c4 0.8064198040760951 0.8437023476357212 0.8043697046783258 0.9921040309667984 0.8044578347431527 0.7999980487126149 0.0019619693329424592 0.0043716559657108744 -0.11378984095867828 -0.11279851430957166 -0.0324653878024832 -0.032676138352892736 support_weighted__beta1 0.9202096450347734 -0.11378984095867828 -0.11279851430957166
12 5gnidd 0.5 3 support_weighted__beta0.5 5gnidd__alpha0.5__seed3__support_weighted__beta0.5 0.8388851918785784 0.8727486080528835 0.8370458430312185 0.9909209257334471 0.8044578347431527 0.7999980487126149 0.03442735713542566 0.03704779431860361 -0.08132445315619508 -0.08012237595667893 0.0 0.0 support_weighted__beta1 0.9202096450347734 -0.08132445315619508 -0.08012237595667893
13 5gnidd 0.5 3 support_weighted__beta1 5gnidd__alpha0.5__seed3__support_weighted__beta1 0.9202096450347734 0.9429509748552004 0.9171682189878975 0.9866951889574849 0.8044578347431527 0.7999980487126149 0.11575181029162074 0.11717017027528254 0.0 0.0 0.08132445315619508 0.08012237595667893 support_weighted__beta1 0.9202096450347734 0.0 0.0
14 5gnidd 0.5 4 v_shrink__b1__lam0.5 5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5 0.9296850930111598 0.8880565443840929 0.9264737095074091 0.9657604065424229 0.9259273982940985 0.9232963015030996 0.003757694717061222 0.0031774080043095454 0.028523543149283515 0.026239840648658586 0.0038793926743494467 0.0009244400791833529 original_protean 0.9259273982940985 0.003757694717061222 0.0031774080043095454
15 5gnidd 0.5 4 v_cap__b1__c4 5gnidd__alpha0.5__seed4__v_cap__b1__c4 0.9230772912397688 0.884532344077232 0.9215621117759981 0.967188414469838 0.9259273982940985 0.9232963015030996 -0.0028501070543297535 -0.0017341897271014872 0.02191574137789254 0.021328242917247553 -0.0027284090970415287 -0.00398715765222768 original_protean 0.9259273982940985 -0.0028501070543297535 -0.0017341897271014872
16 5gnidd 0.5 4 support_weighted__beta0.5 5gnidd__alpha0.5__seed4__support_weighted__beta0.5 0.9258057003368103 0.88601978325071 0.9255492694282258 0.9668784318423409 0.9259273982940985 0.9232963015030996 -0.00012169795728822486 0.0022529679251261925 0.024644150474934068 0.025315400569475233 0.0 0.0 original_protean 0.9259273982940985 -0.00012169795728822486 0.0022529679251261925
17 5gnidd 0.5 4 support_weighted__beta1 5gnidd__alpha0.5__seed4__support_weighted__beta1 0.9011615498618762 0.8562921340984854 0.9002338688587506 0.9672615025827914 0.9259273982940985 0.9232963015030996 -0.024765848432222293 -0.02306243264434904 0.0 0.0 -0.024644150474934068 -0.025315400569475233 original_protean 0.9259273982940985 -0.024765848432222293 -0.02306243264434904
exploration_v2/analysis/confirm_per_class_variant.csv
+145
View File
@@ -0,0 +1,145 @@
dataset,alpha,seed,variant,class_id,class_name,test_support,base_f1,variant_f1,delta_f1,base_recall,variant_recall,delta_recall
xiiotid,0.25,3,v_shrink__b1__lam0.5,0,C&C,573,0.19558797985061663,0.29688838138738227,0.10130040153676564,0.9825479930191972,0.9075043630017452,-0.07504363001745196
xiiotid,0.25,3,v_shrink__b1__lam0.5,1,Exfiltration,4427,0.9995482267901513,0.9995481247175779,-1.0207257339356346e-07,0.9995482267901513,0.999322340185227,-0.0002258866049242858
xiiotid,0.25,3,v_shrink__b1__lam0.5,2,Exploitation,227,0.03455925277291302,0.029429192178550265,-0.005130060594362752,0.6519823788546255,0.6563876651982379,0.004405286343612369
xiiotid,0.25,3,v_shrink__b1__lam0.5,3,Lateral_movement,6319,0.7004691086934823,0.6687793907176446,-0.03168971797583775,0.5553093843962652,0.5176451970248457,-0.03766418737141952
xiiotid,0.25,3,v_shrink__b1__lam0.5,4,Normal,83651,0.9427787226987763,0.9586015558416595,0.01582283314288324,0.9177415691384443,0.9479384585958327,0.03019688945738841
xiiotid,0.25,3,v_shrink__b1__lam0.5,5,RDOS,28252,0.9952464259838485,0.9942155631423466,-0.0010308628415018894,0.9967435933739204,0.9946906413705224,-0.0020529520033979454
xiiotid,0.25,3,v_shrink__b1__lam0.5,6,Reconnaissance,25474,0.7694047991528076,0.7669666336944198,-0.00243816545838782,0.6274632959095549,0.6239695375677161,-0.0034937583418387375
xiiotid,0.25,3,v_shrink__b1__lam0.5,7,Tampering,1024,0.6162227602905569,0.5744920993227992,-0.04173066096775768,0.994140625,0.994140625,0.0
xiiotid,0.25,3,v_shrink__b1__lam0.5,8,Weaponization,12884,0.9023645530071718,0.9586139063327532,0.05624935332558145,0.9863396460726482,0.9833902514746973,-0.0029493945979509606
xiiotid,0.25,3,v_cap__b1__c4,0,C&C,573,0.19558797985061663,0.1510164569215876,-0.04457152292902902,0.9825479930191972,0.680628272251309,-0.3019197207678882
xiiotid,0.25,3,v_cap__b1__c4,1,Exfiltration,4427,0.9995482267901513,0.9808931860036832,-0.01865504078646807,0.9995482267901513,0.9625028235825616,-0.037045403207589755
xiiotid,0.25,3,v_cap__b1__c4,2,Exploitation,227,0.03455925277291302,0.027462035100481952,-0.007097217672431065,0.6519823788546255,0.6651982378854625,0.013215859030836996
xiiotid,0.25,3,v_cap__b1__c4,3,Lateral_movement,6319,0.7004691086934823,0.8096490432574589,0.10917993456397657,0.5553093843962652,0.706440892546289,0.15113150815002374
xiiotid,0.25,3,v_cap__b1__c4,4,Normal,83651,0.9427787226987763,0.948727631530862,0.005948908832085764,0.9177415691384443,0.931034894980335,0.01329332584189069
xiiotid,0.25,3,v_cap__b1__c4,5,RDOS,28252,0.9952464259838485,0.9936917461105007,-0.0015546798733477862,0.9967435933739204,0.9924607107461418,-0.004282882627778539
xiiotid,0.25,3,v_cap__b1__c4,6,Reconnaissance,25474,0.7694047991528076,0.7667060839346477,-0.0026987152181598972,0.6274632959095549,0.6235769804506556,-0.003886315458899281
xiiotid,0.25,3,v_cap__b1__c4,7,Tampering,1024,0.6162227602905569,0.8412353197042193,0.22501255941366238,0.994140625,0.9443359375,-0.0498046875
xiiotid,0.25,3,v_cap__b1__c4,8,Weaponization,12884,0.9023645530071718,0.972830574450517,0.07046602144334524,0.9863396460726482,0.9601831729276622,-0.026156473144986037
xiiotid,0.25,3,support_weighted__beta0.5,0,C&C,573,0.19558797985061663,0.20091789903110657,0.005329919180489945,0.9825479930191972,0.68760907504363,-0.29493891797556715
xiiotid,0.25,3,support_weighted__beta0.5,1,Exfiltration,4427,0.9995482267901513,0.9994349644027574,-0.00011326238739395311,0.9995482267901513,0.9988705669753783,-0.0006776598147729684
xiiotid,0.25,3,support_weighted__beta0.5,2,Exploitation,227,0.03455925277291302,0.027697741425783063,-0.006861511347129954,0.6519823788546255,0.6563876651982379,0.004405286343612369
xiiotid,0.25,3,support_weighted__beta0.5,3,Lateral_movement,6319,0.7004691086934823,0.7297718853577696,0.029302776664287267,0.5553093843962652,0.592340560215224,0.03703117581895876
xiiotid,0.25,3,support_weighted__beta0.5,4,Normal,83651,0.9427787226987763,0.9550678067643374,0.012289084065561173,0.9177415691384443,0.9433479575856833,0.02560638844723906
xiiotid,0.25,3,support_weighted__beta0.5,5,RDOS,28252,0.9952464259838485,0.9928932210899425,-0.0023532048939060024,0.9967435933739204,0.9915050261928359,-0.005238567181084508
xiiotid,0.25,3,support_weighted__beta0.5,6,Reconnaissance,25474,0.7694047991528076,0.7662707609115489,-0.0031340382412586854,0.6274632959095549,0.6230274004867709,-0.0044358954227839975
xiiotid,0.25,3,support_weighted__beta0.5,7,Tampering,1024,0.6162227602905569,0.6616688396349413,0.04544607934438438,0.994140625,0.9912109375,-0.0029296875
xiiotid,0.25,3,support_weighted__beta0.5,8,Weaponization,12884,0.9023645530071718,0.9793060525906333,0.07694149958346153,0.9863396460726482,0.9770257683949084,-0.009313877677739835
xiiotid,0.25,3,support_weighted__beta1,0,C&C,573,0.19558797985061663,0.15957228048529715,-0.03601569936531948,0.9825479930191972,0.6771378708551483,-0.30541012216404884
xiiotid,0.25,3,support_weighted__beta1,1,Exfiltration,4427,0.9995482267901513,0.9992087713349157,-0.0003394554552356155,0.9995482267901513,0.9984187937655297,-0.001129433024621651
xiiotid,0.25,3,support_weighted__beta1,2,Exploitation,227,0.03455925277291302,0.0239511332583186,-0.010608119514594418,0.6519823788546255,0.6563876651982379,0.004405286343612369
xiiotid,0.25,3,support_weighted__beta1,3,Lateral_movement,6319,0.7004691086934823,0.6436087092405458,-0.05686039945293653,0.5553093843962652,0.4888431713878778,-0.0664662130083874
xiiotid,0.25,3,support_weighted__beta1,4,Normal,83651,0.9427787226987763,0.9457320693052946,0.002953346606518359,0.9177415691384443,0.923635102987412,0.005893533848967736
xiiotid,0.25,3,support_weighted__beta1,5,RDOS,28252,0.9952464259838485,0.9929812123360511,-0.002265213647797415,0.9967435933739204,0.9915050261928359,-0.005238567181084508
xiiotid,0.25,3,support_weighted__beta1,6,Reconnaissance,25474,0.7694047991528076,0.7662337662337663,-0.0031710329190413056,0.6274632959095549,0.6230274004867709,-0.0044358954227839975
xiiotid,0.25,3,support_weighted__beta1,7,Tampering,1024,0.6162227602905569,0.5461868958109559,-0.07003586447960097,0.994140625,0.9931640625,-0.0009765625
xiiotid,0.25,3,support_weighted__beta1,8,Weaponization,12884,0.9023645530071718,0.9646464646464645,0.06228191163929275,0.9863396460726482,0.9339490841353617,-0.05239056193728653
xiiotid,0.25,4,v_shrink__b1__lam0.5,0,C&C,573,0.21977597528003087,0.26217401338564505,0.04239803810561418,0.9930191972076788,0.9912739965095986,-0.0017452006980802626
xiiotid,0.25,4,v_shrink__b1__lam0.5,1,Exfiltration,4427,0.9766797506349573,0.9704651162790697,-0.006214634355887538,0.9555003388299074,0.9426248023492206,-0.012875536480686733
xiiotid,0.25,4,v_shrink__b1__lam0.5,2,Exploitation,227,0.04720962951383461,0.03136033229491174,-0.01584929721892287,0.6651982378854625,0.6651982378854625,0.0
xiiotid,0.25,4,v_shrink__b1__lam0.5,3,Lateral_movement,6319,0.8849000425350915,0.9127720870678617,0.027872044532770257,0.8230732710871973,0.9025162209210318,0.07944294983383449
xiiotid,0.25,4,v_shrink__b1__lam0.5,4,Normal,83651,0.9508570036837458,0.9540897802551446,0.0032327765713987544,0.9334257809231211,0.9391997704749495,0.005773989551828396
xiiotid,0.25,4,v_shrink__b1__lam0.5,5,RDOS,28252,0.9948602009996997,0.995265184975796,0.00040498397609622927,0.99684978054651,0.9969913634432961,0.0001415828967861188
xiiotid,0.25,4,v_shrink__b1__lam0.5,6,Reconnaissance,25474,0.7627515886631068,0.7634216004260057,0.0006700117628989366,0.6196121535683442,0.6190625736044595,-0.0005495799638847165
xiiotid,0.25,4,v_shrink__b1__lam0.5,7,Tampering,1024,0.6814384308027607,0.8310552214324768,0.1496167906297161,0.916015625,0.7421875,-0.173828125
xiiotid,0.25,4,v_shrink__b1__lam0.5,8,Weaponization,12884,0.8982495292558755,0.964573699698093,0.0663241704422175,0.999689537410742,0.9795094691089724,-0.02018006830176955
xiiotid,0.25,4,v_cap__b1__c4,0,C&C,573,0.21977597528003087,0.019225284772433692,-0.2005506905075972,0.9930191972076788,1.0,0.006980802792321161
xiiotid,0.25,4,v_cap__b1__c4,1,Exfiltration,4427,0.9766797506349573,0.9561424192407451,-0.02053733139421221,0.9555003388299074,0.91597018296815,-0.03953015586175734
xiiotid,0.25,4,v_cap__b1__c4,2,Exploitation,227,0.04720962951383461,0.025953936060501892,-0.021255693453332718,0.6651982378854625,0.6651982378854625,0.0
xiiotid,0.25,4,v_cap__b1__c4,3,Lateral_movement,6319,0.8849000425350915,0.8543103448275863,-0.030589697707505192,0.8230732710871973,0.7841430606108561,-0.038930210476341154
xiiotid,0.25,4,v_cap__b1__c4,4,Normal,83651,0.9508570036837458,0.47602587077348524,-0.4748311329102606,0.9334257809231211,0.31278765346499143,-0.6206381274581296
xiiotid,0.25,4,v_cap__b1__c4,5,RDOS,28252,0.9948602009996997,0.9941342217654043,-0.0007259792342954041,0.99684978054651,0.992814667988107,-0.004035112558402942
xiiotid,0.25,4,v_cap__b1__c4,6,Reconnaissance,25474,0.7627515886631068,0.7467183551291804,-0.016033233533926383,0.6196121535683442,0.597354165031012,-0.022257988537332185
xiiotid,0.25,4,v_cap__b1__c4,7,Tampering,1024,0.6814384308027607,0.7708719851576995,0.08943355435493883,0.916015625,0.8115234375,-0.1044921875
xiiotid,0.25,4,v_cap__b1__c4,8,Weaponization,12884,0.8982495292558755,0.9351209001675844,0.03687137091170889,0.999689537410742,0.9095001552312947,-0.09018938217944727
xiiotid,0.25,4,support_weighted__beta0.5,0,C&C,573,0.21977597528003087,0.025710176195613086,-0.1940657990844178,0.9930191972076788,0.9982547993019197,0.005235602094240899
xiiotid,0.25,4,support_weighted__beta0.5,1,Exfiltration,4427,0.9766797506349573,0.9771256931608133,0.00044594252585605787,0.9555003388299074,0.9552744522249831,-0.0002258866049242858
xiiotid,0.25,4,support_weighted__beta0.5,2,Exploitation,227,0.04720962951383461,0.023436262866191604,-0.023773366647643006,0.6651982378854625,0.6519823788546255,-0.013215859030836996
xiiotid,0.25,4,support_weighted__beta0.5,3,Lateral_movement,6319,0.8849000425350915,0.8859115567976328,0.0010115142625413132,0.8230732710871973,0.8528248140528565,0.029751542965659183
xiiotid,0.25,4,support_weighted__beta0.5,4,Normal,83651,0.9508570036837458,0.652634283007811,-0.2982227206759348,0.9334257809231211,0.4859356134415608,-0.44749016748156034
xiiotid,0.25,4,support_weighted__beta0.5,5,RDOS,28252,0.9948602009996997,0.9940503594574495,-0.000809841542250278,0.99684978054651,0.9935225824720374,-0.0033271980744725704
xiiotid,0.25,4,support_weighted__beta0.5,6,Reconnaissance,25474,0.7627515886631068,0.7680698201456193,0.0053182314825125365,0.6196121535683442,0.6253042317657219,0.005692078197377715
xiiotid,0.25,4,support_weighted__beta0.5,7,Tampering,1024,0.6814384308027607,0.8291891891891893,0.1477507583864286,0.916015625,0.7490234375,-0.1669921875
xiiotid,0.25,4,support_weighted__beta0.5,8,Weaponization,12884,0.8982495292558755,0.8926873152551087,-0.005562214000766841,0.999689537410742,0.808677429369761,-0.191012108040981
xiiotid,0.25,4,support_weighted__beta1,0,C&C,573,0.21977597528003087,0.0738276919063458,-0.14594828337368507,0.9930191972076788,0.987783595113438,-0.005235602094240788
xiiotid,0.25,4,support_weighted__beta1,1,Exfiltration,4427,0.9766797506349573,0.8429168844746471,-0.1337628661603102,0.9555003388299074,0.7284843008809577,-0.22701603794894964
xiiotid,0.25,4,support_weighted__beta1,2,Exploitation,227,0.04720962951383461,0.005032216385304568,-0.042177413128530045,0.6651982378854625,0.6519823788546255,-0.013215859030836996
xiiotid,0.25,4,support_weighted__beta1,3,Lateral_movement,6319,0.8849000425350915,0.8383040711492754,-0.04659597138581606,0.8230732710871973,0.8651685393258427,0.0420952682386454
xiiotid,0.25,4,support_weighted__beta1,4,Normal,83651,0.9508570036837458,0.45723444826220333,-0.4936225554215425,0.9334257809231211,0.2966850366403271,-0.636740744282794
xiiotid,0.25,4,support_weighted__beta1,5,RDOS,28252,0.9948602009996997,0.9947536334633108,-0.00010656753638893512,0.99684978054651,0.9932748124026618,-0.0035749681438481673
xiiotid,0.25,4,support_weighted__beta1,6,Reconnaissance,25474,0.7627515886631068,0.7785080905375368,0.015756501874430007,0.6196121535683442,0.6393185208447829,0.019706367276438708
xiiotid,0.25,4,support_weighted__beta1,7,Tampering,1024,0.6814384308027607,0.019342359767891684,-0.662096071034869,0.916015625,0.009765625,-0.90625
xiiotid,0.25,4,support_weighted__beta1,8,Weaponization,12884,0.8982495292558755,0.7824761572965205,-0.115773371959355,0.999689537410742,0.6972989754734554,-0.30239056193728653
5gnidd,0.5,3,v_shrink__b1__lam0.5,0,Benign,95547,0.5529540050583834,0.5519122784416843,-0.0010417266166991013,0.3821260740787257,0.38113179900991134,-0.000994275068814332
5gnidd,0.5,3,v_shrink__b1__lam0.5,1,HTTPFlood,28162,0.8921519550586796,0.9118123892470686,0.01966043418838903,0.8543427313401036,0.8587458277111001,0.004403096370996451
5gnidd,0.5,3,v_shrink__b1__lam0.5,2,ICMPFlood,231,0.3996539792387543,0.5422535211267606,0.14259954188800633,1.0,1.0,0.0
5gnidd,0.5,3,v_shrink__b1__lam0.5,3,SYNFlood,1944,0.8737464643867319,0.858948432760364,-0.014798031626367836,0.8739711934156379,0.8739711934156379,0.0
5gnidd,0.5,3,v_shrink__b1__lam0.5,4,SYNScan,4009,0.9931412894375857,0.9941389200648459,0.0009976306272602153,0.9932651534048391,0.9942629084559741,0.0009977550511349742
5gnidd,0.5,3,v_shrink__b1__lam0.5,5,SlowrateDoS,14625,0.8143951383174021,0.8544430422604574,0.04004790394305535,0.8796581196581197,0.9490598290598291,0.06940170940170942
5gnidd,0.5,3,v_shrink__b1__lam0.5,6,TCPConnectScan,4010,0.848619957537155,0.8292531120331951,-0.01936684550395995,0.9967581047381546,0.9967581047381546,0.0
5gnidd,0.5,3,v_shrink__b1__lam0.5,7,UDPFlood,91468,0.7627768118117493,0.7623921650343822,-0.0003846467773670481,1.0,1.0,0.0
5gnidd,0.5,3,v_shrink__b1__lam0.5,8,UDPScan,3181,0.9814011159330439,0.9833825128125485,0.0019813968795046044,0.9952845017290161,0.9952845017290161,0.0
5gnidd,0.5,3,v_cap__b1__c4,0,Benign,95547,0.5529540050583834,0.5550438944717918,0.0020898894134083745,0.3821260740787257,0.3841250902697102,0.001999016190984526
5gnidd,0.5,3,v_cap__b1__c4,1,HTTPFlood,28162,0.8921519550586796,0.8941099122094598,0.0019579571507801763,0.8543427313401036,0.8209289112989134,-0.0334138200411902
5gnidd,0.5,3,v_cap__b1__c4,2,ICMPFlood,231,0.3996539792387543,0.3902027027027027,-0.0094512765360516,1.0,1.0,0.0
5gnidd,0.5,3,v_cap__b1__c4,3,SYNFlood,1944,0.8737464643867319,0.9018046709129511,0.028058206526219265,0.8739711934156379,0.8739711934156379,0.0
5gnidd,0.5,3,v_cap__b1__c4,4,SYNScan,4009,0.9931412894375857,0.9931464174454829,5.128007897203801e-06,0.9932651534048391,0.9940134696931904,0.0007483162883512584
5gnidd,0.5,3,v_cap__b1__c4,5,SlowrateDoS,14625,0.8143951383174021,0.8365808225658615,0.02218568424845946,0.8796581196581197,0.9673162393162393,0.08765811965811965
5gnidd,0.5,3,v_cap__b1__c4,6,TCPConnectScan,4010,0.848619957537155,0.8594792339143534,0.010859276377198412,0.9967581047381546,0.9960099750623441,-0.0007481296758105049
5gnidd,0.5,3,v_cap__b1__c4,7,UDPFlood,91468,0.7627768118117493,0.7626623308944614,-0.00011448091728782916,1.0,1.0,0.0
5gnidd,0.5,3,v_cap__b1__c4,8,UDPScan,3181,0.9814011159330439,0.9841467205470935,0.0027456046140496015,0.9952845017290161,0.9952845017290161,0.0
5gnidd,0.5,3,support_weighted__beta0.5,0,Benign,95547,0.5529540050583834,0.5518354603049501,-0.0011185447534333148,0.3821260740787257,0.38105853663641975,-0.0010675374423059214
5gnidd,0.5,3,support_weighted__beta0.5,1,HTTPFlood,28162,0.8921519550586796,0.9147573805791134,0.022605425520433764,0.8543427313401036,0.8598466018038492,0.005503870463745564
5gnidd,0.5,3,support_weighted__beta0.5,2,ICMPFlood,231,0.3996539792387543,0.6015625,0.2019085207612457,1.0,1.0,0.0
5gnidd,0.5,3,support_weighted__beta0.5,3,SYNFlood,1944,0.8737464643867319,0.8679438058748404,-0.005802658511891434,0.8739711934156379,0.8739711934156379,0.0
5gnidd,0.5,3,support_weighted__beta0.5,4,SYNScan,4009,0.9931412894375857,0.994517817094443,0.0013765276568572515,0.9932651534048391,0.9955101022698928,0.002244948865053664
5gnidd,0.5,3,support_weighted__beta0.5,5,SlowrateDoS,14625,0.8143951383174021,0.8599392507593654,0.045544112441963325,0.8796581196581197,0.9582222222222222,0.07856410256410251
5gnidd,0.5,3,support_weighted__beta0.5,6,TCPConnectScan,4010,0.848619957537155,0.8175959079283888,-0.031024049608766258,0.9967581047381546,0.9965087281795512,-0.00024937655860346464
5gnidd,0.5,3,support_weighted__beta0.5,7,UDPFlood,91468,0.7627768118117493,0.7623667475141483,-0.00041006429760093166,1.0,1.0,0.0
5gnidd,0.5,3,support_weighted__beta0.5,8,UDPScan,3181,0.9814011159330439,0.9862928348909656,0.004891718957921731,0.9952845017290161,0.9952845017290161,0.0
5gnidd,0.5,3,support_weighted__beta1,0,Benign,95547,0.5529540050583834,0.5562078318487121,0.0032538267903287066,0.3821260740787257,0.38524495797879577,0.0031188839000700908
5gnidd,0.5,3,support_weighted__beta1,1,HTTPFlood,28162,0.8921519550586796,0.9776701074362755,0.08551815237759586,0.8543427313401036,0.9887792060222995,0.13443647468219588
5gnidd,0.5,3,support_weighted__beta1,2,ICMPFlood,231,0.3996539792387543,0.9850746268656716,0.5854206476269173,1.0,1.0,0.0
5gnidd,0.5,3,support_weighted__beta1,3,SYNFlood,1944,0.8737464643867319,0.8328431372549019,-0.04090332713182998,0.8739711934156379,0.8739711934156379,0.0
5gnidd,0.5,3,support_weighted__beta1,4,SYNScan,4009,0.9931412894375857,0.9968824042898118,0.0037411148522260573,0.9932651534048391,0.9970067348465952,0.00374158144175607
5gnidd,0.5,3,support_weighted__beta1,5,SlowrateDoS,14625,0.8143951383174021,0.9538612336343907,0.13946609531698861,0.8796581196581197,0.9315555555555556,0.051897435897435895
5gnidd,0.5,3,support_weighted__beta1,6,TCPConnectScan,4010,0.848619957537155,0.8183587747157054,-0.030261182821449628,0.9967581047381546,0.9960099750623441,-0.0007481296758105049
5gnidd,0.5,3,support_weighted__beta1,7,UDPFlood,91468,0.7627768118117493,0.7631491122680549,0.000372300456305652,1.0,1.0,0.0
5gnidd,0.5,3,support_weighted__beta1,8,UDPScan,3181,0.9814011159330439,0.9856831621537504,0.0042820462207064924,0.9952845017290161,0.995598868280415,0.0003143665513989369
5gnidd,0.5,4,v_shrink__b1__lam0.5,0,Benign,95547,0.5564678777807154,0.5582255083179297,0.0017576305372143386,0.38550660931269426,0.3871916439030006,0.0016850345903063335
5gnidd,0.5,4,v_shrink__b1__lam0.5,1,HTTPFlood,28162,0.9678798586572438,0.9741026322786492,0.006222773621405442,0.972622683048079,0.9756764434344152,0.0030537603863362017
5gnidd,0.5,4,v_shrink__b1__lam0.5,2,ICMPFlood,231,0.9148514851485148,0.911242603550296,-0.003608881598218816,1.0,1.0,0.0
5gnidd,0.5,4,v_shrink__b1__lam0.5,3,SYNFlood,1944,0.870687384777456,0.8573651452282157,-0.013322239549240278,0.8503086419753086,0.8503086419753086,0.0
5gnidd,0.5,4,v_shrink__b1__lam0.5,4,SYNScan,4009,0.997880034917072,0.9980044898977302,0.00012445498065816185,0.9980044898977302,0.9980044898977302,0.0
5gnidd,0.5,4,v_shrink__b1__lam0.5,5,SlowrateDoS,14625,0.935706878672658,0.9481568007695744,0.012449922096916421,0.9254700854700855,0.9435213675213675,0.018051282051282036
5gnidd,0.5,4,v_shrink__b1__lam0.5,6,TCPConnectScan,4010,0.7895882818685669,0.8084287306250634,0.018840448756496486,0.9947630922693267,0.9950124688279302,0.00024937655860346464
5gnidd,0.5,4,v_shrink__b1__lam0.5,7,UDPFlood,91468,0.7635727672291811,0.7634739785484747,-9.878868070645197e-05,1.0,1.0,0.0
5gnidd,0.5,4,v_shrink__b1__lam0.5,8,UDPScan,3181,0.9864801864801864,0.9880155642023346,0.001535377722148179,0.9977994341402074,0.9977994341402074,0.0
5gnidd,0.5,4,v_cap__b1__c4,0,Benign,95547,0.5564678777807154,0.5587993514084241,0.002331473627708691,0.38550660931269426,0.38773587867750947,0.0022292693648152118
5gnidd,0.5,4,v_cap__b1__c4,1,HTTPFlood,28162,0.9678798586572438,0.9785105856572297,0.010630726999985907,0.972622683048079,0.9830622825083446,0.010439599460265625
5gnidd,0.5,4,v_cap__b1__c4,2,ICMPFlood,231,0.9148514851485148,0.8816793893129771,-0.03317209583553771,1.0,1.0,0.0
5gnidd,0.5,4,v_cap__b1__c4,3,SYNFlood,1944,0.870687384777456,0.82025819265144,-0.05042919212601604,0.8503086419753086,0.8497942386831275,-0.0005144032921811093
5gnidd,0.5,4,v_cap__b1__c4,4,SYNScan,4009,0.997880034917072,0.997880563520758,5.28603686045237e-07,0.9980044898977302,0.9982539286605139,0.0002494387627837158
5gnidd,0.5,4,v_cap__b1__c4,5,SlowrateDoS,14625,0.935706878672658,0.9564676616915423,0.020760783018884288,0.9254700854700855,0.9464615384615385,0.020991452991452997
5gnidd,0.5,4,v_cap__b1__c4,6,TCPConnectScan,4010,0.7895882818685669,0.827664869348818,0.03807658748025111,0.9947630922693267,0.9952618453865336,0.0004987531172069293
5gnidd,0.5,4,v_cap__b1__c4,7,UDPFlood,91468,0.7635727672291811,0.7635313366055628,-4.1430623618277096e-05,1.0,1.0,0.0
5gnidd,0.5,4,v_cap__b1__c4,8,UDPScan,3181,0.9864801864801864,0.9884735202492211,0.0019933337690346553,0.9977994341402074,0.9974850675888085,-0.0003143665513989369
5gnidd,0.5,4,support_weighted__beta0.5,0,Benign,95547,0.5564678777807154,0.5593643278777508,0.00289645009703543,0.38550660931269426,0.3882801134520184,0.0027735041393241455
5gnidd,0.5,4,support_weighted__beta0.5,1,HTTPFlood,28162,0.9678798586572438,0.9747545831922894,0.006874724535045562,0.972622683048079,0.9713798735885235,-0.0012428094595554606
5gnidd,0.5,4,support_weighted__beta0.5,2,ICMPFlood,231,0.9148514851485148,0.9041095890410958,-0.010741896107418936,1.0,1.0,0.0
5gnidd,0.5,4,support_weighted__beta0.5,3,SYNFlood,1944,0.870687384777456,0.8461735346813412,-0.024513850096114842,0.8503086419753086,0.8503086419753086,0.0
5gnidd,0.5,4,support_weighted__beta0.5,4,SYNScan,4009,0.997880034917072,0.998129442573887,0.0002494076568150261,0.9980044898977302,0.9982539286605139,0.0002494387627837158
5gnidd,0.5,4,support_weighted__beta0.5,5,SlowrateDoS,14625,0.935706878672658,0.950336757602558,0.014629878929899975,0.9254700854700855,0.9551452991452991,0.02967521367521364
5gnidd,0.5,4,support_weighted__beta0.5,6,TCPConnectScan,4010,0.7895882818685669,0.8173254147040754,0.027737132835508493,0.9947630922693267,0.9952618453865336,0.0004987531172069293
5gnidd,0.5,4,support_weighted__beta0.5,7,UDPFlood,91468,0.7635727672291811,0.7636588298156559,8.606258647481102e-05,1.0,1.0,0.0
5gnidd,0.5,4,support_weighted__beta0.5,8,UDPScan,3181,0.9864801864801864,0.9880155642023346,0.001535377722148179,0.9977994341402074,0.9977994341402074,0.0
5gnidd,0.5,4,support_weighted__beta1,0,Benign,95547,0.5564678777807154,0.5587993514084241,0.002331473627708691,0.38550660931269426,0.38773587867750947,0.0022292693648152118
5gnidd,0.5,4,support_weighted__beta1,1,HTTPFlood,28162,0.9678798586572438,0.9742469286149218,0.0063670699576779954,0.972622683048079,0.9658404942830765,-0.006782188765002473
5gnidd,0.5,4,support_weighted__beta1,2,ICMPFlood,231,0.9148514851485148,0.7287066246056783,-0.18614486054283652,1.0,1.0,0.0
5gnidd,0.5,4,support_weighted__beta1,3,SYNFlood,1944,0.870687384777456,0.8210735586481114,-0.04961382612934462,0.8503086419753086,0.8497942386831275,-0.0005144032921811093
5gnidd,0.5,4,support_weighted__beta1,4,SYNScan,4009,0.997880034917072,0.998003992015968,0.00012395709889601036,0.9980044898977302,0.9977550511349463,-0.0002494387627838268
5gnidd,0.5,4,support_weighted__beta1,5,SlowrateDoS,14625,0.935706878672658,0.9501515661839003,0.014444687511242282,0.9254700854700855,0.9644444444444444,0.03897435897435897
5gnidd,0.5,4,support_weighted__beta1,6,TCPConnectScan,4010,0.7895882818685669,0.8417501317870323,0.0521618499184654,0.9947630922693267,0.9955112219451372,0.0007481296758105049
5gnidd,0.5,4,support_weighted__beta1,7,UDPFlood,91468,0.7635727672291811,0.7633561029347331,-0.00021666429444799906,1.0,1.0,0.0
5gnidd,0.5,4,support_weighted__beta1,8,UDPScan,3181,0.9864801864801864,0.9877042801556419,0.0012240936754555065,0.9977994341402074,0.9974850675888085,-0.0003143665513989369
1 dataset alpha seed variant class_id class_name test_support base_f1 variant_f1 delta_f1 base_recall variant_recall delta_recall
2 xiiotid 0.25 3 v_shrink__b1__lam0.5 0 C&C 573 0.19558797985061663 0.29688838138738227 0.10130040153676564 0.9825479930191972 0.9075043630017452 -0.07504363001745196
3 xiiotid 0.25 3 v_shrink__b1__lam0.5 1 Exfiltration 4427 0.9995482267901513 0.9995481247175779 -1.0207257339356346e-07 0.9995482267901513 0.999322340185227 -0.0002258866049242858
4 xiiotid 0.25 3 v_shrink__b1__lam0.5 2 Exploitation 227 0.03455925277291302 0.029429192178550265 -0.005130060594362752 0.6519823788546255 0.6563876651982379 0.004405286343612369
5 xiiotid 0.25 3 v_shrink__b1__lam0.5 3 Lateral_movement 6319 0.7004691086934823 0.6687793907176446 -0.03168971797583775 0.5553093843962652 0.5176451970248457 -0.03766418737141952
6 xiiotid 0.25 3 v_shrink__b1__lam0.5 4 Normal 83651 0.9427787226987763 0.9586015558416595 0.01582283314288324 0.9177415691384443 0.9479384585958327 0.03019688945738841
7 xiiotid 0.25 3 v_shrink__b1__lam0.5 5 RDOS 28252 0.9952464259838485 0.9942155631423466 -0.0010308628415018894 0.9967435933739204 0.9946906413705224 -0.0020529520033979454
8 xiiotid 0.25 3 v_shrink__b1__lam0.5 6 Reconnaissance 25474 0.7694047991528076 0.7669666336944198 -0.00243816545838782 0.6274632959095549 0.6239695375677161 -0.0034937583418387375
9 xiiotid 0.25 3 v_shrink__b1__lam0.5 7 Tampering 1024 0.6162227602905569 0.5744920993227992 -0.04173066096775768 0.994140625 0.994140625 0.0
10 xiiotid 0.25 3 v_shrink__b1__lam0.5 8 Weaponization 12884 0.9023645530071718 0.9586139063327532 0.05624935332558145 0.9863396460726482 0.9833902514746973 -0.0029493945979509606
11 xiiotid 0.25 3 v_cap__b1__c4 0 C&C 573 0.19558797985061663 0.1510164569215876 -0.04457152292902902 0.9825479930191972 0.680628272251309 -0.3019197207678882
12 xiiotid 0.25 3 v_cap__b1__c4 1 Exfiltration 4427 0.9995482267901513 0.9808931860036832 -0.01865504078646807 0.9995482267901513 0.9625028235825616 -0.037045403207589755
13 xiiotid 0.25 3 v_cap__b1__c4 2 Exploitation 227 0.03455925277291302 0.027462035100481952 -0.007097217672431065 0.6519823788546255 0.6651982378854625 0.013215859030836996
14 xiiotid 0.25 3 v_cap__b1__c4 3 Lateral_movement 6319 0.7004691086934823 0.8096490432574589 0.10917993456397657 0.5553093843962652 0.706440892546289 0.15113150815002374
15 xiiotid 0.25 3 v_cap__b1__c4 4 Normal 83651 0.9427787226987763 0.948727631530862 0.005948908832085764 0.9177415691384443 0.931034894980335 0.01329332584189069
16 xiiotid 0.25 3 v_cap__b1__c4 5 RDOS 28252 0.9952464259838485 0.9936917461105007 -0.0015546798733477862 0.9967435933739204 0.9924607107461418 -0.004282882627778539
17 xiiotid 0.25 3 v_cap__b1__c4 6 Reconnaissance 25474 0.7694047991528076 0.7667060839346477 -0.0026987152181598972 0.6274632959095549 0.6235769804506556 -0.003886315458899281
18 xiiotid 0.25 3 v_cap__b1__c4 7 Tampering 1024 0.6162227602905569 0.8412353197042193 0.22501255941366238 0.994140625 0.9443359375 -0.0498046875
19 xiiotid 0.25 3 v_cap__b1__c4 8 Weaponization 12884 0.9023645530071718 0.972830574450517 0.07046602144334524 0.9863396460726482 0.9601831729276622 -0.026156473144986037
20 xiiotid 0.25 3 support_weighted__beta0.5 0 C&C 573 0.19558797985061663 0.20091789903110657 0.005329919180489945 0.9825479930191972 0.68760907504363 -0.29493891797556715
21 xiiotid 0.25 3 support_weighted__beta0.5 1 Exfiltration 4427 0.9995482267901513 0.9994349644027574 -0.00011326238739395311 0.9995482267901513 0.9988705669753783 -0.0006776598147729684
22 xiiotid 0.25 3 support_weighted__beta0.5 2 Exploitation 227 0.03455925277291302 0.027697741425783063 -0.006861511347129954 0.6519823788546255 0.6563876651982379 0.004405286343612369
23 xiiotid 0.25 3 support_weighted__beta0.5 3 Lateral_movement 6319 0.7004691086934823 0.7297718853577696 0.029302776664287267 0.5553093843962652 0.592340560215224 0.03703117581895876
24 xiiotid 0.25 3 support_weighted__beta0.5 4 Normal 83651 0.9427787226987763 0.9550678067643374 0.012289084065561173 0.9177415691384443 0.9433479575856833 0.02560638844723906
25 xiiotid 0.25 3 support_weighted__beta0.5 5 RDOS 28252 0.9952464259838485 0.9928932210899425 -0.0023532048939060024 0.9967435933739204 0.9915050261928359 -0.005238567181084508
26 xiiotid 0.25 3 support_weighted__beta0.5 6 Reconnaissance 25474 0.7694047991528076 0.7662707609115489 -0.0031340382412586854 0.6274632959095549 0.6230274004867709 -0.0044358954227839975
27 xiiotid 0.25 3 support_weighted__beta0.5 7 Tampering 1024 0.6162227602905569 0.6616688396349413 0.04544607934438438 0.994140625 0.9912109375 -0.0029296875
28 xiiotid 0.25 3 support_weighted__beta0.5 8 Weaponization 12884 0.9023645530071718 0.9793060525906333 0.07694149958346153 0.9863396460726482 0.9770257683949084 -0.009313877677739835
29 xiiotid 0.25 3 support_weighted__beta1 0 C&C 573 0.19558797985061663 0.15957228048529715 -0.03601569936531948 0.9825479930191972 0.6771378708551483 -0.30541012216404884
30 xiiotid 0.25 3 support_weighted__beta1 1 Exfiltration 4427 0.9995482267901513 0.9992087713349157 -0.0003394554552356155 0.9995482267901513 0.9984187937655297 -0.001129433024621651
31 xiiotid 0.25 3 support_weighted__beta1 2 Exploitation 227 0.03455925277291302 0.0239511332583186 -0.010608119514594418 0.6519823788546255 0.6563876651982379 0.004405286343612369
32 xiiotid 0.25 3 support_weighted__beta1 3 Lateral_movement 6319 0.7004691086934823 0.6436087092405458 -0.05686039945293653 0.5553093843962652 0.4888431713878778 -0.0664662130083874
33 xiiotid 0.25 3 support_weighted__beta1 4 Normal 83651 0.9427787226987763 0.9457320693052946 0.002953346606518359 0.9177415691384443 0.923635102987412 0.005893533848967736
34 xiiotid 0.25 3 support_weighted__beta1 5 RDOS 28252 0.9952464259838485 0.9929812123360511 -0.002265213647797415 0.9967435933739204 0.9915050261928359 -0.005238567181084508
35 xiiotid 0.25 3 support_weighted__beta1 6 Reconnaissance 25474 0.7694047991528076 0.7662337662337663 -0.0031710329190413056 0.6274632959095549 0.6230274004867709 -0.0044358954227839975
36 xiiotid 0.25 3 support_weighted__beta1 7 Tampering 1024 0.6162227602905569 0.5461868958109559 -0.07003586447960097 0.994140625 0.9931640625 -0.0009765625
37 xiiotid 0.25 3 support_weighted__beta1 8 Weaponization 12884 0.9023645530071718 0.9646464646464645 0.06228191163929275 0.9863396460726482 0.9339490841353617 -0.05239056193728653
38 xiiotid 0.25 4 v_shrink__b1__lam0.5 0 C&C 573 0.21977597528003087 0.26217401338564505 0.04239803810561418 0.9930191972076788 0.9912739965095986 -0.0017452006980802626
39 xiiotid 0.25 4 v_shrink__b1__lam0.5 1 Exfiltration 4427 0.9766797506349573 0.9704651162790697 -0.006214634355887538 0.9555003388299074 0.9426248023492206 -0.012875536480686733
40 xiiotid 0.25 4 v_shrink__b1__lam0.5 2 Exploitation 227 0.04720962951383461 0.03136033229491174 -0.01584929721892287 0.6651982378854625 0.6651982378854625 0.0
41 xiiotid 0.25 4 v_shrink__b1__lam0.5 3 Lateral_movement 6319 0.8849000425350915 0.9127720870678617 0.027872044532770257 0.8230732710871973 0.9025162209210318 0.07944294983383449
42 xiiotid 0.25 4 v_shrink__b1__lam0.5 4 Normal 83651 0.9508570036837458 0.9540897802551446 0.0032327765713987544 0.9334257809231211 0.9391997704749495 0.005773989551828396
43 xiiotid 0.25 4 v_shrink__b1__lam0.5 5 RDOS 28252 0.9948602009996997 0.995265184975796 0.00040498397609622927 0.99684978054651 0.9969913634432961 0.0001415828967861188
44 xiiotid 0.25 4 v_shrink__b1__lam0.5 6 Reconnaissance 25474 0.7627515886631068 0.7634216004260057 0.0006700117628989366 0.6196121535683442 0.6190625736044595 -0.0005495799638847165
45 xiiotid 0.25 4 v_shrink__b1__lam0.5 7 Tampering 1024 0.6814384308027607 0.8310552214324768 0.1496167906297161 0.916015625 0.7421875 -0.173828125
46 xiiotid 0.25 4 v_shrink__b1__lam0.5 8 Weaponization 12884 0.8982495292558755 0.964573699698093 0.0663241704422175 0.999689537410742 0.9795094691089724 -0.02018006830176955
47 xiiotid 0.25 4 v_cap__b1__c4 0 C&C 573 0.21977597528003087 0.019225284772433692 -0.2005506905075972 0.9930191972076788 1.0 0.006980802792321161
48 xiiotid 0.25 4 v_cap__b1__c4 1 Exfiltration 4427 0.9766797506349573 0.9561424192407451 -0.02053733139421221 0.9555003388299074 0.91597018296815 -0.03953015586175734
49 xiiotid 0.25 4 v_cap__b1__c4 2 Exploitation 227 0.04720962951383461 0.025953936060501892 -0.021255693453332718 0.6651982378854625 0.6651982378854625 0.0
50 xiiotid 0.25 4 v_cap__b1__c4 3 Lateral_movement 6319 0.8849000425350915 0.8543103448275863 -0.030589697707505192 0.8230732710871973 0.7841430606108561 -0.038930210476341154
51 xiiotid 0.25 4 v_cap__b1__c4 4 Normal 83651 0.9508570036837458 0.47602587077348524 -0.4748311329102606 0.9334257809231211 0.31278765346499143 -0.6206381274581296
52 xiiotid 0.25 4 v_cap__b1__c4 5 RDOS 28252 0.9948602009996997 0.9941342217654043 -0.0007259792342954041 0.99684978054651 0.992814667988107 -0.004035112558402942
53 xiiotid 0.25 4 v_cap__b1__c4 6 Reconnaissance 25474 0.7627515886631068 0.7467183551291804 -0.016033233533926383 0.6196121535683442 0.597354165031012 -0.022257988537332185
54 xiiotid 0.25 4 v_cap__b1__c4 7 Tampering 1024 0.6814384308027607 0.7708719851576995 0.08943355435493883 0.916015625 0.8115234375 -0.1044921875
55 xiiotid 0.25 4 v_cap__b1__c4 8 Weaponization 12884 0.8982495292558755 0.9351209001675844 0.03687137091170889 0.999689537410742 0.9095001552312947 -0.09018938217944727
56 xiiotid 0.25 4 support_weighted__beta0.5 0 C&C 573 0.21977597528003087 0.025710176195613086 -0.1940657990844178 0.9930191972076788 0.9982547993019197 0.005235602094240899
57 xiiotid 0.25 4 support_weighted__beta0.5 1 Exfiltration 4427 0.9766797506349573 0.9771256931608133 0.00044594252585605787 0.9555003388299074 0.9552744522249831 -0.0002258866049242858
58 xiiotid 0.25 4 support_weighted__beta0.5 2 Exploitation 227 0.04720962951383461 0.023436262866191604 -0.023773366647643006 0.6651982378854625 0.6519823788546255 -0.013215859030836996
59 xiiotid 0.25 4 support_weighted__beta0.5 3 Lateral_movement 6319 0.8849000425350915 0.8859115567976328 0.0010115142625413132 0.8230732710871973 0.8528248140528565 0.029751542965659183
60 xiiotid 0.25 4 support_weighted__beta0.5 4 Normal 83651 0.9508570036837458 0.652634283007811 -0.2982227206759348 0.9334257809231211 0.4859356134415608 -0.44749016748156034
61 xiiotid 0.25 4 support_weighted__beta0.5 5 RDOS 28252 0.9948602009996997 0.9940503594574495 -0.000809841542250278 0.99684978054651 0.9935225824720374 -0.0033271980744725704
62 xiiotid 0.25 4 support_weighted__beta0.5 6 Reconnaissance 25474 0.7627515886631068 0.7680698201456193 0.0053182314825125365 0.6196121535683442 0.6253042317657219 0.005692078197377715
63 xiiotid 0.25 4 support_weighted__beta0.5 7 Tampering 1024 0.6814384308027607 0.8291891891891893 0.1477507583864286 0.916015625 0.7490234375 -0.1669921875
64 xiiotid 0.25 4 support_weighted__beta0.5 8 Weaponization 12884 0.8982495292558755 0.8926873152551087 -0.005562214000766841 0.999689537410742 0.808677429369761 -0.191012108040981
65 xiiotid 0.25 4 support_weighted__beta1 0 C&C 573 0.21977597528003087 0.0738276919063458 -0.14594828337368507 0.9930191972076788 0.987783595113438 -0.005235602094240788
66 xiiotid 0.25 4 support_weighted__beta1 1 Exfiltration 4427 0.9766797506349573 0.8429168844746471 -0.1337628661603102 0.9555003388299074 0.7284843008809577 -0.22701603794894964
67 xiiotid 0.25 4 support_weighted__beta1 2 Exploitation 227 0.04720962951383461 0.005032216385304568 -0.042177413128530045 0.6651982378854625 0.6519823788546255 -0.013215859030836996
68 xiiotid 0.25 4 support_weighted__beta1 3 Lateral_movement 6319 0.8849000425350915 0.8383040711492754 -0.04659597138581606 0.8230732710871973 0.8651685393258427 0.0420952682386454
69 xiiotid 0.25 4 support_weighted__beta1 4 Normal 83651 0.9508570036837458 0.45723444826220333 -0.4936225554215425 0.9334257809231211 0.2966850366403271 -0.636740744282794
70 xiiotid 0.25 4 support_weighted__beta1 5 RDOS 28252 0.9948602009996997 0.9947536334633108 -0.00010656753638893512 0.99684978054651 0.9932748124026618 -0.0035749681438481673
71 xiiotid 0.25 4 support_weighted__beta1 6 Reconnaissance 25474 0.7627515886631068 0.7785080905375368 0.015756501874430007 0.6196121535683442 0.6393185208447829 0.019706367276438708
72 xiiotid 0.25 4 support_weighted__beta1 7 Tampering 1024 0.6814384308027607 0.019342359767891684 -0.662096071034869 0.916015625 0.009765625 -0.90625
73 xiiotid 0.25 4 support_weighted__beta1 8 Weaponization 12884 0.8982495292558755 0.7824761572965205 -0.115773371959355 0.999689537410742 0.6972989754734554 -0.30239056193728653
74 5gnidd 0.5 3 v_shrink__b1__lam0.5 0 Benign 95547 0.5529540050583834 0.5519122784416843 -0.0010417266166991013 0.3821260740787257 0.38113179900991134 -0.000994275068814332
75 5gnidd 0.5 3 v_shrink__b1__lam0.5 1 HTTPFlood 28162 0.8921519550586796 0.9118123892470686 0.01966043418838903 0.8543427313401036 0.8587458277111001 0.004403096370996451
76 5gnidd 0.5 3 v_shrink__b1__lam0.5 2 ICMPFlood 231 0.3996539792387543 0.5422535211267606 0.14259954188800633 1.0 1.0 0.0
77 5gnidd 0.5 3 v_shrink__b1__lam0.5 3 SYNFlood 1944 0.8737464643867319 0.858948432760364 -0.014798031626367836 0.8739711934156379 0.8739711934156379 0.0
78 5gnidd 0.5 3 v_shrink__b1__lam0.5 4 SYNScan 4009 0.9931412894375857 0.9941389200648459 0.0009976306272602153 0.9932651534048391 0.9942629084559741 0.0009977550511349742
79 5gnidd 0.5 3 v_shrink__b1__lam0.5 5 SlowrateDoS 14625 0.8143951383174021 0.8544430422604574 0.04004790394305535 0.8796581196581197 0.9490598290598291 0.06940170940170942
80 5gnidd 0.5 3 v_shrink__b1__lam0.5 6 TCPConnectScan 4010 0.848619957537155 0.8292531120331951 -0.01936684550395995 0.9967581047381546 0.9967581047381546 0.0
81 5gnidd 0.5 3 v_shrink__b1__lam0.5 7 UDPFlood 91468 0.7627768118117493 0.7623921650343822 -0.0003846467773670481 1.0 1.0 0.0
82 5gnidd 0.5 3 v_shrink__b1__lam0.5 8 UDPScan 3181 0.9814011159330439 0.9833825128125485 0.0019813968795046044 0.9952845017290161 0.9952845017290161 0.0
83 5gnidd 0.5 3 v_cap__b1__c4 0 Benign 95547 0.5529540050583834 0.5550438944717918 0.0020898894134083745 0.3821260740787257 0.3841250902697102 0.001999016190984526
84 5gnidd 0.5 3 v_cap__b1__c4 1 HTTPFlood 28162 0.8921519550586796 0.8941099122094598 0.0019579571507801763 0.8543427313401036 0.8209289112989134 -0.0334138200411902
85 5gnidd 0.5 3 v_cap__b1__c4 2 ICMPFlood 231 0.3996539792387543 0.3902027027027027 -0.0094512765360516 1.0 1.0 0.0
86 5gnidd 0.5 3 v_cap__b1__c4 3 SYNFlood 1944 0.8737464643867319 0.9018046709129511 0.028058206526219265 0.8739711934156379 0.8739711934156379 0.0
87 5gnidd 0.5 3 v_cap__b1__c4 4 SYNScan 4009 0.9931412894375857 0.9931464174454829 5.128007897203801e-06 0.9932651534048391 0.9940134696931904 0.0007483162883512584
88 5gnidd 0.5 3 v_cap__b1__c4 5 SlowrateDoS 14625 0.8143951383174021 0.8365808225658615 0.02218568424845946 0.8796581196581197 0.9673162393162393 0.08765811965811965
89 5gnidd 0.5 3 v_cap__b1__c4 6 TCPConnectScan 4010 0.848619957537155 0.8594792339143534 0.010859276377198412 0.9967581047381546 0.9960099750623441 -0.0007481296758105049
90 5gnidd 0.5 3 v_cap__b1__c4 7 UDPFlood 91468 0.7627768118117493 0.7626623308944614 -0.00011448091728782916 1.0 1.0 0.0
91 5gnidd 0.5 3 v_cap__b1__c4 8 UDPScan 3181 0.9814011159330439 0.9841467205470935 0.0027456046140496015 0.9952845017290161 0.9952845017290161 0.0
92 5gnidd 0.5 3 support_weighted__beta0.5 0 Benign 95547 0.5529540050583834 0.5518354603049501 -0.0011185447534333148 0.3821260740787257 0.38105853663641975 -0.0010675374423059214
93 5gnidd 0.5 3 support_weighted__beta0.5 1 HTTPFlood 28162 0.8921519550586796 0.9147573805791134 0.022605425520433764 0.8543427313401036 0.8598466018038492 0.005503870463745564
94 5gnidd 0.5 3 support_weighted__beta0.5 2 ICMPFlood 231 0.3996539792387543 0.6015625 0.2019085207612457 1.0 1.0 0.0
95 5gnidd 0.5 3 support_weighted__beta0.5 3 SYNFlood 1944 0.8737464643867319 0.8679438058748404 -0.005802658511891434 0.8739711934156379 0.8739711934156379 0.0
96 5gnidd 0.5 3 support_weighted__beta0.5 4 SYNScan 4009 0.9931412894375857 0.994517817094443 0.0013765276568572515 0.9932651534048391 0.9955101022698928 0.002244948865053664
97 5gnidd 0.5 3 support_weighted__beta0.5 5 SlowrateDoS 14625 0.8143951383174021 0.8599392507593654 0.045544112441963325 0.8796581196581197 0.9582222222222222 0.07856410256410251
98 5gnidd 0.5 3 support_weighted__beta0.5 6 TCPConnectScan 4010 0.848619957537155 0.8175959079283888 -0.031024049608766258 0.9967581047381546 0.9965087281795512 -0.00024937655860346464
99 5gnidd 0.5 3 support_weighted__beta0.5 7 UDPFlood 91468 0.7627768118117493 0.7623667475141483 -0.00041006429760093166 1.0 1.0 0.0
100 5gnidd 0.5 3 support_weighted__beta0.5 8 UDPScan 3181 0.9814011159330439 0.9862928348909656 0.004891718957921731 0.9952845017290161 0.9952845017290161 0.0
101 5gnidd 0.5 3 support_weighted__beta1 0 Benign 95547 0.5529540050583834 0.5562078318487121 0.0032538267903287066 0.3821260740787257 0.38524495797879577 0.0031188839000700908
102 5gnidd 0.5 3 support_weighted__beta1 1 HTTPFlood 28162 0.8921519550586796 0.9776701074362755 0.08551815237759586 0.8543427313401036 0.9887792060222995 0.13443647468219588
103 5gnidd 0.5 3 support_weighted__beta1 2 ICMPFlood 231 0.3996539792387543 0.9850746268656716 0.5854206476269173 1.0 1.0 0.0
104 5gnidd 0.5 3 support_weighted__beta1 3 SYNFlood 1944 0.8737464643867319 0.8328431372549019 -0.04090332713182998 0.8739711934156379 0.8739711934156379 0.0
105 5gnidd 0.5 3 support_weighted__beta1 4 SYNScan 4009 0.9931412894375857 0.9968824042898118 0.0037411148522260573 0.9932651534048391 0.9970067348465952 0.00374158144175607
106 5gnidd 0.5 3 support_weighted__beta1 5 SlowrateDoS 14625 0.8143951383174021 0.9538612336343907 0.13946609531698861 0.8796581196581197 0.9315555555555556 0.051897435897435895
107 5gnidd 0.5 3 support_weighted__beta1 6 TCPConnectScan 4010 0.848619957537155 0.8183587747157054 -0.030261182821449628 0.9967581047381546 0.9960099750623441 -0.0007481296758105049
108 5gnidd 0.5 3 support_weighted__beta1 7 UDPFlood 91468 0.7627768118117493 0.7631491122680549 0.000372300456305652 1.0 1.0 0.0
109 5gnidd 0.5 3 support_weighted__beta1 8 UDPScan 3181 0.9814011159330439 0.9856831621537504 0.0042820462207064924 0.9952845017290161 0.995598868280415 0.0003143665513989369
110 5gnidd 0.5 4 v_shrink__b1__lam0.5 0 Benign 95547 0.5564678777807154 0.5582255083179297 0.0017576305372143386 0.38550660931269426 0.3871916439030006 0.0016850345903063335
111 5gnidd 0.5 4 v_shrink__b1__lam0.5 1 HTTPFlood 28162 0.9678798586572438 0.9741026322786492 0.006222773621405442 0.972622683048079 0.9756764434344152 0.0030537603863362017
112 5gnidd 0.5 4 v_shrink__b1__lam0.5 2 ICMPFlood 231 0.9148514851485148 0.911242603550296 -0.003608881598218816 1.0 1.0 0.0
113 5gnidd 0.5 4 v_shrink__b1__lam0.5 3 SYNFlood 1944 0.870687384777456 0.8573651452282157 -0.013322239549240278 0.8503086419753086 0.8503086419753086 0.0
114 5gnidd 0.5 4 v_shrink__b1__lam0.5 4 SYNScan 4009 0.997880034917072 0.9980044898977302 0.00012445498065816185 0.9980044898977302 0.9980044898977302 0.0
115 5gnidd 0.5 4 v_shrink__b1__lam0.5 5 SlowrateDoS 14625 0.935706878672658 0.9481568007695744 0.012449922096916421 0.9254700854700855 0.9435213675213675 0.018051282051282036
116 5gnidd 0.5 4 v_shrink__b1__lam0.5 6 TCPConnectScan 4010 0.7895882818685669 0.8084287306250634 0.018840448756496486 0.9947630922693267 0.9950124688279302 0.00024937655860346464
117 5gnidd 0.5 4 v_shrink__b1__lam0.5 7 UDPFlood 91468 0.7635727672291811 0.7634739785484747 -9.878868070645197e-05 1.0 1.0 0.0
118 5gnidd 0.5 4 v_shrink__b1__lam0.5 8 UDPScan 3181 0.9864801864801864 0.9880155642023346 0.001535377722148179 0.9977994341402074 0.9977994341402074 0.0
119 5gnidd 0.5 4 v_cap__b1__c4 0 Benign 95547 0.5564678777807154 0.5587993514084241 0.002331473627708691 0.38550660931269426 0.38773587867750947 0.0022292693648152118
120 5gnidd 0.5 4 v_cap__b1__c4 1 HTTPFlood 28162 0.9678798586572438 0.9785105856572297 0.010630726999985907 0.972622683048079 0.9830622825083446 0.010439599460265625
121 5gnidd 0.5 4 v_cap__b1__c4 2 ICMPFlood 231 0.9148514851485148 0.8816793893129771 -0.03317209583553771 1.0 1.0 0.0
122 5gnidd 0.5 4 v_cap__b1__c4 3 SYNFlood 1944 0.870687384777456 0.82025819265144 -0.05042919212601604 0.8503086419753086 0.8497942386831275 -0.0005144032921811093
123 5gnidd 0.5 4 v_cap__b1__c4 4 SYNScan 4009 0.997880034917072 0.997880563520758 5.28603686045237e-07 0.9980044898977302 0.9982539286605139 0.0002494387627837158
124 5gnidd 0.5 4 v_cap__b1__c4 5 SlowrateDoS 14625 0.935706878672658 0.9564676616915423 0.020760783018884288 0.9254700854700855 0.9464615384615385 0.020991452991452997
125 5gnidd 0.5 4 v_cap__b1__c4 6 TCPConnectScan 4010 0.7895882818685669 0.827664869348818 0.03807658748025111 0.9947630922693267 0.9952618453865336 0.0004987531172069293
126 5gnidd 0.5 4 v_cap__b1__c4 7 UDPFlood 91468 0.7635727672291811 0.7635313366055628 -4.1430623618277096e-05 1.0 1.0 0.0
127 5gnidd 0.5 4 v_cap__b1__c4 8 UDPScan 3181 0.9864801864801864 0.9884735202492211 0.0019933337690346553 0.9977994341402074 0.9974850675888085 -0.0003143665513989369
128 5gnidd 0.5 4 support_weighted__beta0.5 0 Benign 95547 0.5564678777807154 0.5593643278777508 0.00289645009703543 0.38550660931269426 0.3882801134520184 0.0027735041393241455
129 5gnidd 0.5 4 support_weighted__beta0.5 1 HTTPFlood 28162 0.9678798586572438 0.9747545831922894 0.006874724535045562 0.972622683048079 0.9713798735885235 -0.0012428094595554606
130 5gnidd 0.5 4 support_weighted__beta0.5 2 ICMPFlood 231 0.9148514851485148 0.9041095890410958 -0.010741896107418936 1.0 1.0 0.0
131 5gnidd 0.5 4 support_weighted__beta0.5 3 SYNFlood 1944 0.870687384777456 0.8461735346813412 -0.024513850096114842 0.8503086419753086 0.8503086419753086 0.0
132 5gnidd 0.5 4 support_weighted__beta0.5 4 SYNScan 4009 0.997880034917072 0.998129442573887 0.0002494076568150261 0.9980044898977302 0.9982539286605139 0.0002494387627837158
133 5gnidd 0.5 4 support_weighted__beta0.5 5 SlowrateDoS 14625 0.935706878672658 0.950336757602558 0.014629878929899975 0.9254700854700855 0.9551452991452991 0.02967521367521364
134 5gnidd 0.5 4 support_weighted__beta0.5 6 TCPConnectScan 4010 0.7895882818685669 0.8173254147040754 0.027737132835508493 0.9947630922693267 0.9952618453865336 0.0004987531172069293
135 5gnidd 0.5 4 support_weighted__beta0.5 7 UDPFlood 91468 0.7635727672291811 0.7636588298156559 8.606258647481102e-05 1.0 1.0 0.0
136 5gnidd 0.5 4 support_weighted__beta0.5 8 UDPScan 3181 0.9864801864801864 0.9880155642023346 0.001535377722148179 0.9977994341402074 0.9977994341402074 0.0
137 5gnidd 0.5 4 support_weighted__beta1 0 Benign 95547 0.5564678777807154 0.5587993514084241 0.002331473627708691 0.38550660931269426 0.38773587867750947 0.0022292693648152118
138 5gnidd 0.5 4 support_weighted__beta1 1 HTTPFlood 28162 0.9678798586572438 0.9742469286149218 0.0063670699576779954 0.972622683048079 0.9658404942830765 -0.006782188765002473
139 5gnidd 0.5 4 support_weighted__beta1 2 ICMPFlood 231 0.9148514851485148 0.7287066246056783 -0.18614486054283652 1.0 1.0 0.0
140 5gnidd 0.5 4 support_weighted__beta1 3 SYNFlood 1944 0.870687384777456 0.8210735586481114 -0.04961382612934462 0.8503086419753086 0.8497942386831275 -0.0005144032921811093
141 5gnidd 0.5 4 support_weighted__beta1 4 SYNScan 4009 0.997880034917072 0.998003992015968 0.00012395709889601036 0.9980044898977302 0.9977550511349463 -0.0002494387627838268
142 5gnidd 0.5 4 support_weighted__beta1 5 SlowrateDoS 14625 0.935706878672658 0.9501515661839003 0.014444687511242282 0.9254700854700855 0.9644444444444444 0.03897435897435897
143 5gnidd 0.5 4 support_weighted__beta1 6 TCPConnectScan 4010 0.7895882818685669 0.8417501317870323 0.0521618499184654 0.9947630922693267 0.9955112219451372 0.0007481296758105049
144 5gnidd 0.5 4 support_weighted__beta1 7 UDPFlood 91468 0.7635727672291811 0.7633561029347331 -0.00021666429444799906 1.0 1.0 0.0
145 5gnidd 0.5 4 support_weighted__beta1 8 UDPScan 3181 0.9864801864801864 0.9877042801556419 0.0012240936754555065 0.9977994341402074 0.9974850675888085 -0.0003143665513989369
exploration_v2/analysis/confirm_per_client_variant.csv
+161
View File
@@ -0,0 +1,161 @@
dataset,alpha,seed,variant,client_id,n_samples,base_val_rare_f1,variant_val_rare_f1,delta_val_rare_f1
xiiotid,0.25,3,v_shrink__b1__lam0.5,0,2140,0.8561962585138841,0.8375523410734678,-0.01864391744041638
xiiotid,0.25,3,v_shrink__b1__lam0.5,1,18610,0.28169014084507044,0.3895131086142322,0.10782296776916178
xiiotid,0.25,3,v_shrink__b1__lam0.5,2,1581,0.5283208020050125,0.536380531594081,0.00805972958906842
xiiotid,0.25,3,v_shrink__b1__lam0.5,3,3178,0.8866432337434095,0.8850220264317181,-0.0016212073116913528
xiiotid,0.25,3,v_shrink__b1__lam0.5,4,12935,0.5414469291532702,0.5653124500337164,0.023865520880446245
xiiotid,0.25,3,v_shrink__b1__lam0.5,5,5628,0.6122881355932204,0.6888020833333333,0.07651394774011289
xiiotid,0.25,3,v_shrink__b1__lam0.5,6,13609,0.6420664206642066,0.626865671641791,-0.015200749022415638
xiiotid,0.25,3,v_shrink__b1__lam0.5,7,2268,0.9933548127265405,0.9943227899432279,0.000967977216687399
xiiotid,0.25,3,v_shrink__b1__lam0.5,8,5176,0.47174934993403267,0.4315775645103299,-0.040171785423702755
xiiotid,0.25,3,v_shrink__b1__lam0.5,9,16289,0.8829885544314278,0.8804212607716639,-0.002567293659763825
xiiotid,0.25,3,v_cap__b1__c4,0,2140,0.8561962585138841,0.9140103780578206,0.05781411954393645
xiiotid,0.25,3,v_cap__b1__c4,1,18610,0.28169014084507044,0.21875,-0.06294014084507044
xiiotid,0.25,3,v_cap__b1__c4,2,1581,0.5283208020050125,0.5383390750600271,0.01001827305501457
xiiotid,0.25,3,v_cap__b1__c4,3,3178,0.8866432337434095,0.886103781882146,-0.0005394518612634736
xiiotid,0.25,3,v_cap__b1__c4,4,12935,0.5414469291532702,0.5196141652815423,-0.021832763871727945
xiiotid,0.25,3,v_cap__b1__c4,5,5628,0.6122881355932204,0.629029029029029,0.016740893435808646
xiiotid,0.25,3,v_cap__b1__c4,6,13609,0.6420664206642066,0.726643598615917,0.08457717795171038
xiiotid,0.25,3,v_cap__b1__c4,7,2268,0.9933548127265405,0.9796953677153082,-0.013659445011232285
xiiotid,0.25,3,v_cap__b1__c4,8,5176,0.47174934993403267,0.6828009194286191,0.21105156949458642
xiiotid,0.25,3,v_cap__b1__c4,9,16289,0.8829885544314278,0.8799470726264877,-0.003041481804940016
xiiotid,0.25,3,support_weighted__beta0.5,0,2140,0.8561962585138841,0.8707592406090758,0.01456298209519169
xiiotid,0.25,3,support_weighted__beta0.5,1,18610,0.28169014084507044,0.2828282828282828,0.0011381419832123818
xiiotid,0.25,3,support_weighted__beta0.5,2,1581,0.5283208020050125,0.5399982514425599,0.011677449437547338
xiiotid,0.25,3,support_weighted__beta0.5,3,3178,0.8866432337434095,0.8850220264317181,-0.0016212073116913528
xiiotid,0.25,3,support_weighted__beta0.5,4,12935,0.5414469291532702,0.5417999072331323,0.0003529780798621207
xiiotid,0.25,3,support_weighted__beta0.5,5,5628,0.6122881355932204,0.647567517573067,0.03527938197984659
xiiotid,0.25,3,support_weighted__beta0.5,6,13609,0.6420664206642066,0.6666666666666666,0.024600246002460024
xiiotid,0.25,3,support_weighted__beta0.5,7,2268,0.9933548127265405,0.9930838079739626,-0.000271004752577908
xiiotid,0.25,3,support_weighted__beta0.5,8,5176,0.47174934993403267,0.4994349873234336,0.02768563738940094
xiiotid,0.25,3,support_weighted__beta0.5,9,16289,0.8829885544314278,0.8793175531042194,-0.003671001327208323
xiiotid,0.25,3,support_weighted__beta1,0,2140,0.8561962585138841,0.8255265538183227,-0.030669704695561406
xiiotid,0.25,3,support_weighted__beta1,1,18610,0.28169014084507044,0.23098591549295774,-0.05070422535211269
xiiotid,0.25,3,support_weighted__beta1,2,1581,0.5283208020050125,0.531031056751803,0.0027102547467904303
xiiotid,0.25,3,support_weighted__beta1,3,3178,0.8866432337434095,0.8850220264317181,-0.0016212073116913528
xiiotid,0.25,3,support_weighted__beta1,4,12935,0.5414469291532702,0.5323556211995784,-0.00909130795369184
xiiotid,0.25,3,support_weighted__beta1,5,5628,0.6122881355932204,0.5699022093444405,-0.042385926248779904
xiiotid,0.25,3,support_weighted__beta1,6,13609,0.6420664206642066,0.6060606060606061,-0.03600581460360053
xiiotid,0.25,3,support_weighted__beta1,7,2268,0.9933548127265405,0.9808013355592654,-0.012553477167275062
xiiotid,0.25,3,support_weighted__beta1,8,5176,0.47174934993403267,0.40165827142049576,-0.07009107851353691
xiiotid,0.25,3,support_weighted__beta1,9,16289,0.8829885544314278,0.8801112439784262,-0.002877310453001547
xiiotid,0.25,4,v_shrink__b1__lam0.5,0,2737,0.6914893617021277,0.8,0.10851063829787233
xiiotid,0.25,4,v_shrink__b1__lam0.5,1,16337,0.030266608391608392,0.13058659217877094,0.10031998378716256
xiiotid,0.25,4,v_shrink__b1__lam0.5,2,20397,0.9916883116883117,0.9899736147757255,-0.0017146969125861888
xiiotid,0.25,4,v_shrink__b1__lam0.5,3,6530,0.8631578947368421,0.8400000000000001,-0.023157894736842044
xiiotid,0.25,4,v_shrink__b1__lam0.5,4,7342,0.48807096365416736,0.4368949709744338,-0.05117599267973355
xiiotid,0.25,4,v_shrink__b1__lam0.5,5,11939,0.4859075535512965,0.4811858608893957,-0.0047216926619008115
xiiotid,0.25,4,v_shrink__b1__lam0.5,6,6280,0.8491131545208317,0.878584907422977,0.02947175290214532
xiiotid,0.25,4,v_shrink__b1__lam0.5,7,7900,0.45493042952208107,0.4273399014778325,-0.027590528044248552
xiiotid,0.25,4,v_shrink__b1__lam0.5,8,1256,0.19672131147540983,0.5714285714285715,0.3747072599531617
xiiotid,0.25,4,v_shrink__b1__lam0.5,9,696,0.3725490196078431,0.31932773109243695,-0.05322128851540614
xiiotid,0.25,4,v_cap__b1__c4,0,2737,0.6914893617021277,0.7638888888888888,0.07239952718676113
xiiotid,0.25,4,v_cap__b1__c4,1,16337,0.030266608391608392,0.2650974577580082,0.2348308493663998
xiiotid,0.25,4,v_cap__b1__c4,2,20397,0.9916883116883117,0.9509185632026322,-0.04076974848567949
xiiotid,0.25,4,v_cap__b1__c4,3,6530,0.8631578947368421,0.8172043010752689,-0.045953593661573255
xiiotid,0.25,4,v_cap__b1__c4,4,7342,0.48807096365416736,0.4566807056324985,-0.03139025802166884
xiiotid,0.25,4,v_cap__b1__c4,5,11939,0.4859075535512965,0.4727694090382387,-0.013138144513057781
xiiotid,0.25,4,v_cap__b1__c4,6,6280,0.8491131545208317,0.5698830559991757,-0.27923009852165603
xiiotid,0.25,4,v_cap__b1__c4,7,7900,0.45493042952208107,0.3986569227730594,-0.05627350674902165
xiiotid,0.25,4,v_cap__b1__c4,8,1256,0.19672131147540983,0.923076923076923,0.7263556116015132
xiiotid,0.25,4,v_cap__b1__c4,9,696,0.3725490196078431,0.2773722627737226,-0.09517675683412047
xiiotid,0.25,4,support_weighted__beta0.5,0,2737,0.6914893617021277,0.8125,0.12101063829787229
xiiotid,0.25,4,support_weighted__beta0.5,1,16337,0.030266608391608392,0.40527704485488125,0.37501043646327287
xiiotid,0.25,4,support_weighted__beta0.5,2,20397,0.9916883116883117,0.8921739130434782,-0.09951439864483358
xiiotid,0.25,4,support_weighted__beta0.5,3,6530,0.8631578947368421,0.8247422680412372,-0.03841562669560494
xiiotid,0.25,4,support_weighted__beta0.5,4,7342,0.48807096365416736,0.4370553291427782,-0.05101563451138913
xiiotid,0.25,4,support_weighted__beta0.5,5,11939,0.4859075535512965,0.4858757062146893,-3.184733660721584e-05
xiiotid,0.25,4,support_weighted__beta0.5,6,6280,0.8491131545208317,0.6092359988392819,-0.23987715568154977
xiiotid,0.25,4,support_weighted__beta0.5,7,7900,0.45493042952208107,0.4038434194839712,-0.05108701003810989
xiiotid,0.25,4,support_weighted__beta0.5,8,1256,0.19672131147540983,0.8333333333333334,0.6366120218579235
xiiotid,0.25,4,support_weighted__beta0.5,9,696,0.3725490196078431,0.24203821656050953,-0.13051080304733356
xiiotid,0.25,4,support_weighted__beta1,0,2737,0.6914893617021277,0.0,-0.6914893617021277
xiiotid,0.25,4,support_weighted__beta1,1,16337,0.030266608391608392,0.13121299272542716,0.10094638433381878
xiiotid,0.25,4,support_weighted__beta1,2,20397,0.9916883116883117,0.827670645852464,-0.16401766583584776
xiiotid,0.25,4,support_weighted__beta1,3,6530,0.8631578947368421,0.7321428571428571,-0.13101503759398503
xiiotid,0.25,4,support_weighted__beta1,4,7342,0.48807096365416736,0.034514697996392535,-0.45355626565777485
xiiotid,0.25,4,support_weighted__beta1,5,11939,0.4859075535512965,0.4083224967490247,-0.07758505680227179
xiiotid,0.25,4,support_weighted__beta1,6,6280,0.8491131545208317,0.6507254064416463,-0.19838774807918536
xiiotid,0.25,4,support_weighted__beta1,7,7900,0.45493042952208107,0.4015292906670286,-0.05340113885505249
xiiotid,0.25,4,support_weighted__beta1,8,1256,0.19672131147540983,0.9090909090909091,0.7123695976154992
xiiotid,0.25,4,support_weighted__beta1,9,696,0.3725490196078431,0.1958762886597938,-0.17667273094804928
5gnidd,0.5,3,v_shrink__b1__lam0.5,0,8233,0.997275204359673,0.997275204359673,0.0
5gnidd,0.5,3,v_shrink__b1__lam0.5,1,11855,1.0,1.0,0.0
5gnidd,0.5,3,v_shrink__b1__lam0.5,2,3482,0.822289156626506,0.8629737609329445,0.0406846043064385
5gnidd,0.5,3,v_shrink__b1__lam0.5,3,23984,0.5995379807039,0.6332307954013865,0.03369281469748653
5gnidd,0.5,3,v_shrink__b1__lam0.5,4,23835,0.9813571961222968,0.9835575485799701,0.00220035245767336
5gnidd,0.5,3,v_shrink__b1__lam0.5,5,14697,0.9188260179799048,0.9300813008130081,0.011255282833103286
5gnidd,0.5,3,v_shrink__b1__lam0.5,6,13864,0.8424967110833235,0.8844401120994274,0.04194340101610394
5gnidd,0.5,3,v_shrink__b1__lam0.5,7,2087,0.7710740265297389,0.8266129032258064,0.055538876696067496
5gnidd,0.5,3,v_shrink__b1__lam0.5,8,14920,0.6668387602336143,0.6606425938471879,-0.006196166386426394
5gnidd,0.5,3,v_shrink__b1__lam0.5,9,4632,0.804147465437788,0.8885630498533725,0.0844155844155845
5gnidd,0.5,3,v_cap__b1__c4,0,8233,0.997275204359673,0.997275204359673,0.0
5gnidd,0.5,3,v_cap__b1__c4,1,11855,1.0,1.0,0.0
5gnidd,0.5,3,v_cap__b1__c4,2,3482,0.822289156626506,0.8698140200286122,0.047524863402106154
5gnidd,0.5,3,v_cap__b1__c4,3,23984,0.5995379807039,0.5987810299244285,-0.0007569507794714658
5gnidd,0.5,3,v_cap__b1__c4,4,23835,0.9813571961222968,0.9835575485799701,0.00220035245767336
5gnidd,0.5,3,v_cap__b1__c4,5,14697,0.9188260179799048,0.9047131147540984,-0.014112903225806384
5gnidd,0.5,3,v_cap__b1__c4,6,13864,0.8424967110833235,0.8319189061763319,-0.010577804906991561
5gnidd,0.5,3,v_cap__b1__c4,7,2087,0.7710740265297389,0.8,0.028925973470261157
5gnidd,0.5,3,v_cap__b1__c4,8,14920,0.6668387602336143,0.6608854511655734,-0.005953309068040924
5gnidd,0.5,3,v_cap__b1__c4,9,4632,0.804147465437788,0.7900782013685239,-0.01406926406926412
5gnidd,0.5,3,support_weighted__beta0.5,0,8233,0.997275204359673,0.997275204359673,0.0
5gnidd,0.5,3,support_weighted__beta0.5,1,11855,1.0,1.0,0.0
5gnidd,0.5,3,support_weighted__beta0.5,2,3482,0.822289156626506,0.869942196531792,0.04765303990528591
5gnidd,0.5,3,support_weighted__beta0.5,3,23984,0.5995379807039,0.6328921430582934,0.03335416235439348
5gnidd,0.5,3,support_weighted__beta0.5,4,23835,0.9813571961222968,0.9894894894894894,0.008132293367192678
5gnidd,0.5,3,support_weighted__beta0.5,5,14697,0.9188260179799048,0.9424360400444939,0.023610022064589065
5gnidd,0.5,3,support_weighted__beta0.5,6,13864,0.8424967110833235,0.8964569058781104,0.053960194794786975
5gnidd,0.5,3,support_weighted__beta0.5,7,2087,0.7710740265297389,0.8444444444444446,0.07337041791470567
5gnidd,0.5,3,support_weighted__beta0.5,8,14920,0.6668387602336143,0.6606102627831439,-0.006228497450470383
5gnidd,0.5,3,support_weighted__beta0.5,9,4632,0.804147465437788,0.893939393939394,0.089791928501606
5gnidd,0.5,3,support_weighted__beta1,0,8233,0.997275204359673,0.997275204359673,0.0
5gnidd,0.5,3,support_weighted__beta1,1,11855,1.0,1.0,0.0
5gnidd,0.5,3,support_weighted__beta1,2,3482,0.822289156626506,0.9487603305785123,0.12647117395200624
5gnidd,0.5,3,support_weighted__beta1,3,23984,0.5995379807039,0.9202334630350195,0.32069548233111955
5gnidd,0.5,3,support_weighted__beta1,4,23835,0.9813571961222968,0.9894894894894894,0.008132293367192678
5gnidd,0.5,3,support_weighted__beta1,5,14697,0.9188260179799048,0.956,0.03717398202009514
5gnidd,0.5,3,support_weighted__beta1,6,13864,0.8424967110833235,0.9996466431095407,0.1571499320262172
5gnidd,0.5,3,support_weighted__beta1,7,2087,0.7710740265297389,0.9444444444444444,0.17337041791470553
5gnidd,0.5,3,support_weighted__beta1,8,14920,0.6668387602336143,0.679036517621346,0.012197757387731656
5gnidd,0.5,3,support_weighted__beta1,9,4632,0.804147465437788,0.9946236559139785,0.19047619047619047
5gnidd,0.5,4,v_shrink__b1__lam0.5,0,13066,0.9896907216494846,0.9795918367346939,-0.010098884914790718
5gnidd,0.5,4,v_shrink__b1__lam0.5,1,17365,0.7715385486244764,0.772027397260274,0.0004888486357976785
5gnidd,0.5,4,v_shrink__b1__lam0.5,2,1194,0.9745762711864406,0.9806866952789699,0.006110424092529265
5gnidd,0.5,4,v_shrink__b1__lam0.5,3,5360,0.7272727272727272,0.7272727272727272,0.0
5gnidd,0.5,4,v_shrink__b1__lam0.5,4,17032,0.9676691729323308,0.9648729446935725,-0.002796228238758247
5gnidd,0.5,4,v_shrink__b1__lam0.5,5,3520,0.9864864864864865,0.9864864864864865,0.0
5gnidd,0.5,4,v_shrink__b1__lam0.5,6,33983,0.6752988047808764,0.6904761904761905,0.015177385695314016
5gnidd,0.5,4,v_shrink__b1__lam0.5,7,3343,0.9150326797385621,0.8705882352941177,-0.0444444444444444
5gnidd,0.5,4,v_shrink__b1__lam0.5,8,17387,0.987171958429151,0.9893973501613671,0.002225391732216142
5gnidd,0.5,4,v_shrink__b1__lam0.5,9,9339,0.9104573073233131,0.9191655801825294,0.00870827285921627
5gnidd,0.5,4,v_cap__b1__c4,0,13066,0.9896907216494846,0.9696969696969697,-0.01999375195251485
5gnidd,0.5,4,v_cap__b1__c4,1,17365,0.7715385486244764,0.7212821201539976,-0.05025642847047873
5gnidd,0.5,4,v_cap__b1__c4,2,1194,0.9745762711864406,0.9806866952789699,0.006110424092529265
5gnidd,0.5,4,v_cap__b1__c4,3,5360,0.7272727272727272,0.7272727272727272,0.0
5gnidd,0.5,4,v_cap__b1__c4,4,17032,0.9676691729323308,0.966966966966967,-0.0007022059653637447
5gnidd,0.5,4,v_cap__b1__c4,5,3520,0.9864864864864865,0.9864864864864865,0.0
5gnidd,0.5,4,v_cap__b1__c4,6,33983,0.6752988047808764,0.71256038647343,0.03726158169255356
5gnidd,0.5,4,v_cap__b1__c4,7,3343,0.9150326797385621,0.8705882352941177,-0.0444444444444444
5gnidd,0.5,4,v_cap__b1__c4,8,17387,0.987171958429151,0.9914354230900994,0.004263464660948402
5gnidd,0.5,4,v_cap__b1__c4,9,9339,0.9104573073233131,0.9183474300585557,0.007890122735242588
5gnidd,0.5,4,support_weighted__beta0.5,0,13066,0.9896907216494846,0.9795918367346939,-0.010098884914790718
5gnidd,0.5,4,support_weighted__beta0.5,1,17365,0.7715385486244764,0.7563099668362826,-0.015228581788193751
5gnidd,0.5,4,support_weighted__beta0.5,2,1194,0.9745762711864406,0.9806866952789699,0.006110424092529265
5gnidd,0.5,4,support_weighted__beta0.5,3,5360,0.7272727272727272,0.7272727272727272,0.0
5gnidd,0.5,4,support_weighted__beta0.5,4,17032,0.9676691729323308,0.9662668665667167,-0.0014023063656141144
5gnidd,0.5,4,support_weighted__beta0.5,5,3520,0.9864864864864865,0.9864864864864865,0.0
5gnidd,0.5,4,support_weighted__beta0.5,6,33983,0.6752988047808764,0.7018348623853211,0.026536057604444663
5gnidd,0.5,4,support_weighted__beta0.5,7,3343,0.9150326797385621,0.8705882352941177,-0.0444444444444444
5gnidd,0.5,4,support_weighted__beta0.5,8,17387,0.987171958429151,0.9890885925632715,0.0019166341341205761
5gnidd,0.5,4,support_weighted__beta0.5,9,9339,0.9104573073233131,0.9020715630885123,-0.008385744234800763
5gnidd,0.5,4,support_weighted__beta1,0,13066,0.9896907216494846,0.9056603773584906,-0.08403034429099399
5gnidd,0.5,4,support_weighted__beta1,1,17365,0.7715385486244764,0.7214141414141414,-0.050124407210334954
5gnidd,0.5,4,support_weighted__beta1,2,1194,0.9745762711864406,0.9827586206896552,0.008182349503214614
5gnidd,0.5,4,support_weighted__beta1,3,5360,0.7272727272727272,0.7272727272727272,0.0
5gnidd,0.5,4,support_weighted__beta1,4,17032,0.9676691729323308,0.7975124378109453,-0.17015673512138552
5gnidd,0.5,4,support_weighted__beta1,5,3520,0.9864864864864865,0.9864864864864865,0.0
5gnidd,0.5,4,support_weighted__beta1,6,33983,0.6752988047808764,0.7268041237113402,0.05150531893046373
5gnidd,0.5,4,support_weighted__beta1,7,3343,0.9150326797385621,0.8686192468619247,-0.04641343287663735
5gnidd,0.5,4,support_weighted__beta1,8,17387,0.987171958429151,0.9887660607350757,0.0015941023059247161
5gnidd,0.5,4,support_weighted__beta1,9,9339,0.9104573073233131,0.8576271186440678,-0.05283018867924527
1 dataset alpha seed variant client_id n_samples base_val_rare_f1 variant_val_rare_f1 delta_val_rare_f1
2 xiiotid 0.25 3 v_shrink__b1__lam0.5 0 2140 0.8561962585138841 0.8375523410734678 -0.01864391744041638
3 xiiotid 0.25 3 v_shrink__b1__lam0.5 1 18610 0.28169014084507044 0.3895131086142322 0.10782296776916178
4 xiiotid 0.25 3 v_shrink__b1__lam0.5 2 1581 0.5283208020050125 0.536380531594081 0.00805972958906842
5 xiiotid 0.25 3 v_shrink__b1__lam0.5 3 3178 0.8866432337434095 0.8850220264317181 -0.0016212073116913528
6 xiiotid 0.25 3 v_shrink__b1__lam0.5 4 12935 0.5414469291532702 0.5653124500337164 0.023865520880446245
7 xiiotid 0.25 3 v_shrink__b1__lam0.5 5 5628 0.6122881355932204 0.6888020833333333 0.07651394774011289
8 xiiotid 0.25 3 v_shrink__b1__lam0.5 6 13609 0.6420664206642066 0.626865671641791 -0.015200749022415638
9 xiiotid 0.25 3 v_shrink__b1__lam0.5 7 2268 0.9933548127265405 0.9943227899432279 0.000967977216687399
10 xiiotid 0.25 3 v_shrink__b1__lam0.5 8 5176 0.47174934993403267 0.4315775645103299 -0.040171785423702755
11 xiiotid 0.25 3 v_shrink__b1__lam0.5 9 16289 0.8829885544314278 0.8804212607716639 -0.002567293659763825
12 xiiotid 0.25 3 v_cap__b1__c4 0 2140 0.8561962585138841 0.9140103780578206 0.05781411954393645
13 xiiotid 0.25 3 v_cap__b1__c4 1 18610 0.28169014084507044 0.21875 -0.06294014084507044
14 xiiotid 0.25 3 v_cap__b1__c4 2 1581 0.5283208020050125 0.5383390750600271 0.01001827305501457
15 xiiotid 0.25 3 v_cap__b1__c4 3 3178 0.8866432337434095 0.886103781882146 -0.0005394518612634736
16 xiiotid 0.25 3 v_cap__b1__c4 4 12935 0.5414469291532702 0.5196141652815423 -0.021832763871727945
17 xiiotid 0.25 3 v_cap__b1__c4 5 5628 0.6122881355932204 0.629029029029029 0.016740893435808646
18 xiiotid 0.25 3 v_cap__b1__c4 6 13609 0.6420664206642066 0.726643598615917 0.08457717795171038
19 xiiotid 0.25 3 v_cap__b1__c4 7 2268 0.9933548127265405 0.9796953677153082 -0.013659445011232285
20 xiiotid 0.25 3 v_cap__b1__c4 8 5176 0.47174934993403267 0.6828009194286191 0.21105156949458642
21 xiiotid 0.25 3 v_cap__b1__c4 9 16289 0.8829885544314278 0.8799470726264877 -0.003041481804940016
22 xiiotid 0.25 3 support_weighted__beta0.5 0 2140 0.8561962585138841 0.8707592406090758 0.01456298209519169
23 xiiotid 0.25 3 support_weighted__beta0.5 1 18610 0.28169014084507044 0.2828282828282828 0.0011381419832123818
24 xiiotid 0.25 3 support_weighted__beta0.5 2 1581 0.5283208020050125 0.5399982514425599 0.011677449437547338
25 xiiotid 0.25 3 support_weighted__beta0.5 3 3178 0.8866432337434095 0.8850220264317181 -0.0016212073116913528
26 xiiotid 0.25 3 support_weighted__beta0.5 4 12935 0.5414469291532702 0.5417999072331323 0.0003529780798621207
27 xiiotid 0.25 3 support_weighted__beta0.5 5 5628 0.6122881355932204 0.647567517573067 0.03527938197984659
28 xiiotid 0.25 3 support_weighted__beta0.5 6 13609 0.6420664206642066 0.6666666666666666 0.024600246002460024
29 xiiotid 0.25 3 support_weighted__beta0.5 7 2268 0.9933548127265405 0.9930838079739626 -0.000271004752577908
30 xiiotid 0.25 3 support_weighted__beta0.5 8 5176 0.47174934993403267 0.4994349873234336 0.02768563738940094
31 xiiotid 0.25 3 support_weighted__beta0.5 9 16289 0.8829885544314278 0.8793175531042194 -0.003671001327208323
32 xiiotid 0.25 3 support_weighted__beta1 0 2140 0.8561962585138841 0.8255265538183227 -0.030669704695561406
33 xiiotid 0.25 3 support_weighted__beta1 1 18610 0.28169014084507044 0.23098591549295774 -0.05070422535211269
34 xiiotid 0.25 3 support_weighted__beta1 2 1581 0.5283208020050125 0.531031056751803 0.0027102547467904303
35 xiiotid 0.25 3 support_weighted__beta1 3 3178 0.8866432337434095 0.8850220264317181 -0.0016212073116913528
36 xiiotid 0.25 3 support_weighted__beta1 4 12935 0.5414469291532702 0.5323556211995784 -0.00909130795369184
37 xiiotid 0.25 3 support_weighted__beta1 5 5628 0.6122881355932204 0.5699022093444405 -0.042385926248779904
38 xiiotid 0.25 3 support_weighted__beta1 6 13609 0.6420664206642066 0.6060606060606061 -0.03600581460360053
39 xiiotid 0.25 3 support_weighted__beta1 7 2268 0.9933548127265405 0.9808013355592654 -0.012553477167275062
40 xiiotid 0.25 3 support_weighted__beta1 8 5176 0.47174934993403267 0.40165827142049576 -0.07009107851353691
41 xiiotid 0.25 3 support_weighted__beta1 9 16289 0.8829885544314278 0.8801112439784262 -0.002877310453001547
42 xiiotid 0.25 4 v_shrink__b1__lam0.5 0 2737 0.6914893617021277 0.8 0.10851063829787233
43 xiiotid 0.25 4 v_shrink__b1__lam0.5 1 16337 0.030266608391608392 0.13058659217877094 0.10031998378716256
44 xiiotid 0.25 4 v_shrink__b1__lam0.5 2 20397 0.9916883116883117 0.9899736147757255 -0.0017146969125861888
45 xiiotid 0.25 4 v_shrink__b1__lam0.5 3 6530 0.8631578947368421 0.8400000000000001 -0.023157894736842044
46 xiiotid 0.25 4 v_shrink__b1__lam0.5 4 7342 0.48807096365416736 0.4368949709744338 -0.05117599267973355
47 xiiotid 0.25 4 v_shrink__b1__lam0.5 5 11939 0.4859075535512965 0.4811858608893957 -0.0047216926619008115
48 xiiotid 0.25 4 v_shrink__b1__lam0.5 6 6280 0.8491131545208317 0.878584907422977 0.02947175290214532
49 xiiotid 0.25 4 v_shrink__b1__lam0.5 7 7900 0.45493042952208107 0.4273399014778325 -0.027590528044248552
50 xiiotid 0.25 4 v_shrink__b1__lam0.5 8 1256 0.19672131147540983 0.5714285714285715 0.3747072599531617
51 xiiotid 0.25 4 v_shrink__b1__lam0.5 9 696 0.3725490196078431 0.31932773109243695 -0.05322128851540614
52 xiiotid 0.25 4 v_cap__b1__c4 0 2737 0.6914893617021277 0.7638888888888888 0.07239952718676113
53 xiiotid 0.25 4 v_cap__b1__c4 1 16337 0.030266608391608392 0.2650974577580082 0.2348308493663998
54 xiiotid 0.25 4 v_cap__b1__c4 2 20397 0.9916883116883117 0.9509185632026322 -0.04076974848567949
55 xiiotid 0.25 4 v_cap__b1__c4 3 6530 0.8631578947368421 0.8172043010752689 -0.045953593661573255
56 xiiotid 0.25 4 v_cap__b1__c4 4 7342 0.48807096365416736 0.4566807056324985 -0.03139025802166884
57 xiiotid 0.25 4 v_cap__b1__c4 5 11939 0.4859075535512965 0.4727694090382387 -0.013138144513057781
58 xiiotid 0.25 4 v_cap__b1__c4 6 6280 0.8491131545208317 0.5698830559991757 -0.27923009852165603
59 xiiotid 0.25 4 v_cap__b1__c4 7 7900 0.45493042952208107 0.3986569227730594 -0.05627350674902165
60 xiiotid 0.25 4 v_cap__b1__c4 8 1256 0.19672131147540983 0.923076923076923 0.7263556116015132
61 xiiotid 0.25 4 v_cap__b1__c4 9 696 0.3725490196078431 0.2773722627737226 -0.09517675683412047
62 xiiotid 0.25 4 support_weighted__beta0.5 0 2737 0.6914893617021277 0.8125 0.12101063829787229
63 xiiotid 0.25 4 support_weighted__beta0.5 1 16337 0.030266608391608392 0.40527704485488125 0.37501043646327287
64 xiiotid 0.25 4 support_weighted__beta0.5 2 20397 0.9916883116883117 0.8921739130434782 -0.09951439864483358
65 xiiotid 0.25 4 support_weighted__beta0.5 3 6530 0.8631578947368421 0.8247422680412372 -0.03841562669560494
66 xiiotid 0.25 4 support_weighted__beta0.5 4 7342 0.48807096365416736 0.4370553291427782 -0.05101563451138913
67 xiiotid 0.25 4 support_weighted__beta0.5 5 11939 0.4859075535512965 0.4858757062146893 -3.184733660721584e-05
68 xiiotid 0.25 4 support_weighted__beta0.5 6 6280 0.8491131545208317 0.6092359988392819 -0.23987715568154977
69 xiiotid 0.25 4 support_weighted__beta0.5 7 7900 0.45493042952208107 0.4038434194839712 -0.05108701003810989
70 xiiotid 0.25 4 support_weighted__beta0.5 8 1256 0.19672131147540983 0.8333333333333334 0.6366120218579235
71 xiiotid 0.25 4 support_weighted__beta0.5 9 696 0.3725490196078431 0.24203821656050953 -0.13051080304733356
72 xiiotid 0.25 4 support_weighted__beta1 0 2737 0.6914893617021277 0.0 -0.6914893617021277
73 xiiotid 0.25 4 support_weighted__beta1 1 16337 0.030266608391608392 0.13121299272542716 0.10094638433381878
74 xiiotid 0.25 4 support_weighted__beta1 2 20397 0.9916883116883117 0.827670645852464 -0.16401766583584776
75 xiiotid 0.25 4 support_weighted__beta1 3 6530 0.8631578947368421 0.7321428571428571 -0.13101503759398503
76 xiiotid 0.25 4 support_weighted__beta1 4 7342 0.48807096365416736 0.034514697996392535 -0.45355626565777485
77 xiiotid 0.25 4 support_weighted__beta1 5 11939 0.4859075535512965 0.4083224967490247 -0.07758505680227179
78 xiiotid 0.25 4 support_weighted__beta1 6 6280 0.8491131545208317 0.6507254064416463 -0.19838774807918536
79 xiiotid 0.25 4 support_weighted__beta1 7 7900 0.45493042952208107 0.4015292906670286 -0.05340113885505249
80 xiiotid 0.25 4 support_weighted__beta1 8 1256 0.19672131147540983 0.9090909090909091 0.7123695976154992
81 xiiotid 0.25 4 support_weighted__beta1 9 696 0.3725490196078431 0.1958762886597938 -0.17667273094804928
82 5gnidd 0.5 3 v_shrink__b1__lam0.5 0 8233 0.997275204359673 0.997275204359673 0.0
83 5gnidd 0.5 3 v_shrink__b1__lam0.5 1 11855 1.0 1.0 0.0
84 5gnidd 0.5 3 v_shrink__b1__lam0.5 2 3482 0.822289156626506 0.8629737609329445 0.0406846043064385
85 5gnidd 0.5 3 v_shrink__b1__lam0.5 3 23984 0.5995379807039 0.6332307954013865 0.03369281469748653
86 5gnidd 0.5 3 v_shrink__b1__lam0.5 4 23835 0.9813571961222968 0.9835575485799701 0.00220035245767336
87 5gnidd 0.5 3 v_shrink__b1__lam0.5 5 14697 0.9188260179799048 0.9300813008130081 0.011255282833103286
88 5gnidd 0.5 3 v_shrink__b1__lam0.5 6 13864 0.8424967110833235 0.8844401120994274 0.04194340101610394
89 5gnidd 0.5 3 v_shrink__b1__lam0.5 7 2087 0.7710740265297389 0.8266129032258064 0.055538876696067496
90 5gnidd 0.5 3 v_shrink__b1__lam0.5 8 14920 0.6668387602336143 0.6606425938471879 -0.006196166386426394
91 5gnidd 0.5 3 v_shrink__b1__lam0.5 9 4632 0.804147465437788 0.8885630498533725 0.0844155844155845
92 5gnidd 0.5 3 v_cap__b1__c4 0 8233 0.997275204359673 0.997275204359673 0.0
93 5gnidd 0.5 3 v_cap__b1__c4 1 11855 1.0 1.0 0.0
94 5gnidd 0.5 3 v_cap__b1__c4 2 3482 0.822289156626506 0.8698140200286122 0.047524863402106154
95 5gnidd 0.5 3 v_cap__b1__c4 3 23984 0.5995379807039 0.5987810299244285 -0.0007569507794714658
96 5gnidd 0.5 3 v_cap__b1__c4 4 23835 0.9813571961222968 0.9835575485799701 0.00220035245767336
97 5gnidd 0.5 3 v_cap__b1__c4 5 14697 0.9188260179799048 0.9047131147540984 -0.014112903225806384
98 5gnidd 0.5 3 v_cap__b1__c4 6 13864 0.8424967110833235 0.8319189061763319 -0.010577804906991561
99 5gnidd 0.5 3 v_cap__b1__c4 7 2087 0.7710740265297389 0.8 0.028925973470261157
100 5gnidd 0.5 3 v_cap__b1__c4 8 14920 0.6668387602336143 0.6608854511655734 -0.005953309068040924
101 5gnidd 0.5 3 v_cap__b1__c4 9 4632 0.804147465437788 0.7900782013685239 -0.01406926406926412
102 5gnidd 0.5 3 support_weighted__beta0.5 0 8233 0.997275204359673 0.997275204359673 0.0
103 5gnidd 0.5 3 support_weighted__beta0.5 1 11855 1.0 1.0 0.0
104 5gnidd 0.5 3 support_weighted__beta0.5 2 3482 0.822289156626506 0.869942196531792 0.04765303990528591
105 5gnidd 0.5 3 support_weighted__beta0.5 3 23984 0.5995379807039 0.6328921430582934 0.03335416235439348
106 5gnidd 0.5 3 support_weighted__beta0.5 4 23835 0.9813571961222968 0.9894894894894894 0.008132293367192678
107 5gnidd 0.5 3 support_weighted__beta0.5 5 14697 0.9188260179799048 0.9424360400444939 0.023610022064589065
108 5gnidd 0.5 3 support_weighted__beta0.5 6 13864 0.8424967110833235 0.8964569058781104 0.053960194794786975
109 5gnidd 0.5 3 support_weighted__beta0.5 7 2087 0.7710740265297389 0.8444444444444446 0.07337041791470567
110 5gnidd 0.5 3 support_weighted__beta0.5 8 14920 0.6668387602336143 0.6606102627831439 -0.006228497450470383
111 5gnidd 0.5 3 support_weighted__beta0.5 9 4632 0.804147465437788 0.893939393939394 0.089791928501606
112 5gnidd 0.5 3 support_weighted__beta1 0 8233 0.997275204359673 0.997275204359673 0.0
113 5gnidd 0.5 3 support_weighted__beta1 1 11855 1.0 1.0 0.0
114 5gnidd 0.5 3 support_weighted__beta1 2 3482 0.822289156626506 0.9487603305785123 0.12647117395200624
115 5gnidd 0.5 3 support_weighted__beta1 3 23984 0.5995379807039 0.9202334630350195 0.32069548233111955
116 5gnidd 0.5 3 support_weighted__beta1 4 23835 0.9813571961222968 0.9894894894894894 0.008132293367192678
117 5gnidd 0.5 3 support_weighted__beta1 5 14697 0.9188260179799048 0.956 0.03717398202009514
118 5gnidd 0.5 3 support_weighted__beta1 6 13864 0.8424967110833235 0.9996466431095407 0.1571499320262172
119 5gnidd 0.5 3 support_weighted__beta1 7 2087 0.7710740265297389 0.9444444444444444 0.17337041791470553
120 5gnidd 0.5 3 support_weighted__beta1 8 14920 0.6668387602336143 0.679036517621346 0.012197757387731656
121 5gnidd 0.5 3 support_weighted__beta1 9 4632 0.804147465437788 0.9946236559139785 0.19047619047619047
122 5gnidd 0.5 4 v_shrink__b1__lam0.5 0 13066 0.9896907216494846 0.9795918367346939 -0.010098884914790718
123 5gnidd 0.5 4 v_shrink__b1__lam0.5 1 17365 0.7715385486244764 0.772027397260274 0.0004888486357976785
124 5gnidd 0.5 4 v_shrink__b1__lam0.5 2 1194 0.9745762711864406 0.9806866952789699 0.006110424092529265
125 5gnidd 0.5 4 v_shrink__b1__lam0.5 3 5360 0.7272727272727272 0.7272727272727272 0.0
126 5gnidd 0.5 4 v_shrink__b1__lam0.5 4 17032 0.9676691729323308 0.9648729446935725 -0.002796228238758247
127 5gnidd 0.5 4 v_shrink__b1__lam0.5 5 3520 0.9864864864864865 0.9864864864864865 0.0
128 5gnidd 0.5 4 v_shrink__b1__lam0.5 6 33983 0.6752988047808764 0.6904761904761905 0.015177385695314016
129 5gnidd 0.5 4 v_shrink__b1__lam0.5 7 3343 0.9150326797385621 0.8705882352941177 -0.0444444444444444
130 5gnidd 0.5 4 v_shrink__b1__lam0.5 8 17387 0.987171958429151 0.9893973501613671 0.002225391732216142
131 5gnidd 0.5 4 v_shrink__b1__lam0.5 9 9339 0.9104573073233131 0.9191655801825294 0.00870827285921627
132 5gnidd 0.5 4 v_cap__b1__c4 0 13066 0.9896907216494846 0.9696969696969697 -0.01999375195251485
133 5gnidd 0.5 4 v_cap__b1__c4 1 17365 0.7715385486244764 0.7212821201539976 -0.05025642847047873
134 5gnidd 0.5 4 v_cap__b1__c4 2 1194 0.9745762711864406 0.9806866952789699 0.006110424092529265
135 5gnidd 0.5 4 v_cap__b1__c4 3 5360 0.7272727272727272 0.7272727272727272 0.0
136 5gnidd 0.5 4 v_cap__b1__c4 4 17032 0.9676691729323308 0.966966966966967 -0.0007022059653637447
137 5gnidd 0.5 4 v_cap__b1__c4 5 3520 0.9864864864864865 0.9864864864864865 0.0
138 5gnidd 0.5 4 v_cap__b1__c4 6 33983 0.6752988047808764 0.71256038647343 0.03726158169255356
139 5gnidd 0.5 4 v_cap__b1__c4 7 3343 0.9150326797385621 0.8705882352941177 -0.0444444444444444
140 5gnidd 0.5 4 v_cap__b1__c4 8 17387 0.987171958429151 0.9914354230900994 0.004263464660948402
141 5gnidd 0.5 4 v_cap__b1__c4 9 9339 0.9104573073233131 0.9183474300585557 0.007890122735242588
142 5gnidd 0.5 4 support_weighted__beta0.5 0 13066 0.9896907216494846 0.9795918367346939 -0.010098884914790718
143 5gnidd 0.5 4 support_weighted__beta0.5 1 17365 0.7715385486244764 0.7563099668362826 -0.015228581788193751
144 5gnidd 0.5 4 support_weighted__beta0.5 2 1194 0.9745762711864406 0.9806866952789699 0.006110424092529265
145 5gnidd 0.5 4 support_weighted__beta0.5 3 5360 0.7272727272727272 0.7272727272727272 0.0
146 5gnidd 0.5 4 support_weighted__beta0.5 4 17032 0.9676691729323308 0.9662668665667167 -0.0014023063656141144
147 5gnidd 0.5 4 support_weighted__beta0.5 5 3520 0.9864864864864865 0.9864864864864865 0.0
148 5gnidd 0.5 4 support_weighted__beta0.5 6 33983 0.6752988047808764 0.7018348623853211 0.026536057604444663
149 5gnidd 0.5 4 support_weighted__beta0.5 7 3343 0.9150326797385621 0.8705882352941177 -0.0444444444444444
150 5gnidd 0.5 4 support_weighted__beta0.5 8 17387 0.987171958429151 0.9890885925632715 0.0019166341341205761
151 5gnidd 0.5 4 support_weighted__beta0.5 9 9339 0.9104573073233131 0.9020715630885123 -0.008385744234800763
152 5gnidd 0.5 4 support_weighted__beta1 0 13066 0.9896907216494846 0.9056603773584906 -0.08403034429099399
153 5gnidd 0.5 4 support_weighted__beta1 1 17365 0.7715385486244764 0.7214141414141414 -0.050124407210334954
154 5gnidd 0.5 4 support_weighted__beta1 2 1194 0.9745762711864406 0.9827586206896552 0.008182349503214614
155 5gnidd 0.5 4 support_weighted__beta1 3 5360 0.7272727272727272 0.7272727272727272 0.0
156 5gnidd 0.5 4 support_weighted__beta1 4 17032 0.9676691729323308 0.7975124378109453 -0.17015673512138552
157 5gnidd 0.5 4 support_weighted__beta1 5 3520 0.9864864864864865 0.9864864864864865 0.0
158 5gnidd 0.5 4 support_weighted__beta1 6 33983 0.6752988047808764 0.7268041237113402 0.05150531893046373
159 5gnidd 0.5 4 support_weighted__beta1 7 3343 0.9150326797385621 0.8686192468619247 -0.04641343287663735
160 5gnidd 0.5 4 support_weighted__beta1 8 17387 0.987171958429151 0.9887660607350757 0.0015941023059247161
161 5gnidd 0.5 4 support_weighted__beta1 9 9339 0.9104573073233131 0.8576271186440678 -0.05283018867924527
exploration_v2/analysis/confirm_reference_summary.csv
+3
View File
@@ -0,0 +1,3 @@
config,n_pairs,mean_delta_val_vs_original,sd_delta_val_vs_original,positive_pairs_val,mean_delta_test_vs_original,sd_delta_test_vs_original,positive_pairs_test
support_weighted__beta0.5,4,0.01090766077956501,0.019000659916111093,2,0.012164986913594772,0.01990024144848278,3
support_weighted__beta1,4,-0.016274987402244034,0.10395541030505347,1,-0.015464250464134077,0.10566381521436177,1
1 config n_pairs mean_delta_val_vs_original sd_delta_val_vs_original positive_pairs_val mean_delta_test_vs_original sd_delta_test_vs_original positive_pairs_test
2 support_weighted__beta0.5 4 0.01090766077956501 0.019000659916111093 2 0.012164986913594772 0.01990024144848278 3
3 support_weighted__beta1 4 -0.016274987402244034 0.10395541030505347 1 -0.015464250464134077 0.10566381521436177 1
exploration_v2/analysis/confirm_summary.csv
+3
View File
@@ -0,0 +1,3 @@
variant,n_pairs,mean_delta_val_vs_original,sd_delta_val_vs_original,positive_pairs_val_vs_original,mean_delta_test_vs_original,sd_delta_test_vs_original,positive_pairs_test_vs_original,mean_delta_val_vs_beta1,sd_delta_val_vs_beta1,mean_delta_test_vs_beta1,sd_delta_test_vs_beta1,mean_delta_val_vs_beta05,sd_delta_val_vs_beta05,mean_delta_test_vs_beta05,sd_delta_test_vs_beta05,mean_delta_val_vs_current_best,sd_delta_val_vs_current_best,positive_pairs_val_vs_current_best,mean_delta_test_vs_current_best,sd_delta_test_vs_current_best,positive_pairs_test_vs_current_best,improved_vs_original_per_plan,S1_val_deltas,S2_val_deltas
v_shrink__b1__lam0.5,4,0.017994808339324264,0.014072367228704582,4,0.018354318695924132,0.014319068327916913,4,0.0342697957415683,0.10649767725215149,0.03381856916005821,0.1082507145556489,0.007087147559759255,0.023366313613711127,0.00618933178232936,0.024241921772951987,-0.01532128227473148,0.05206218407417258,2,-0.015455669432363206,0.052417393479881856,2,True,0.008456|0.032863,0.026902|0.003758
v_cap__b1__c4,4,0.00467808081785806,0.02625027663765537,2,0.005868542758943829,0.025844914703010976,2,0.020953068220102095,0.09798314473745867,0.021332793223077906,0.09849148681223491,-0.006229579961706949,0.023343093528822526,-0.006296444154650943,0.023094240221471166,-0.028638009796197683,0.05968831126237434,1,-0.02794144536934351,0.059325448594970964,1,False,0.041031|-0.021430,0.001962|-0.002850
1 variant n_pairs mean_delta_val_vs_original sd_delta_val_vs_original positive_pairs_val_vs_original mean_delta_test_vs_original sd_delta_test_vs_original positive_pairs_test_vs_original mean_delta_val_vs_beta1 sd_delta_val_vs_beta1 mean_delta_test_vs_beta1 sd_delta_test_vs_beta1 mean_delta_val_vs_beta05 sd_delta_val_vs_beta05 mean_delta_test_vs_beta05 sd_delta_test_vs_beta05 mean_delta_val_vs_current_best sd_delta_val_vs_current_best positive_pairs_val_vs_current_best mean_delta_test_vs_current_best sd_delta_test_vs_current_best positive_pairs_test_vs_current_best improved_vs_original_per_plan S1_val_deltas S2_val_deltas
2 v_shrink__b1__lam0.5 4 0.017994808339324264 0.014072367228704582 4 0.018354318695924132 0.014319068327916913 4 0.0342697957415683 0.10649767725215149 0.03381856916005821 0.1082507145556489 0.007087147559759255 0.023366313613711127 0.00618933178232936 0.024241921772951987 -0.01532128227473148 0.05206218407417258 2 -0.015455669432363206 0.052417393479881856 2 True 0.008456|0.032863 0.026902|0.003758
3 v_cap__b1__c4 4 0.00467808081785806 0.02625027663765537 2 0.005868542758943829 0.025844914703010976 2 0.020953068220102095 0.09798314473745867 0.021332793223077906 0.09849148681223491 -0.006229579961706949 0.023343093528822526 -0.006296444154650943 0.023094240221471166 -0.028638009796197683 0.05968831126237434 1 -0.02794144536934351 0.059325448594970964 1 False 0.041031|-0.021430 0.001962|-0.002850
exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__characterization.csv
+10
View File
@@ -0,0 +1,10 @@
class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio
0,Benign,334415,10,0,1571,112226,33441.5,37695.7725116491,1.127215361501401,71.43602800763844
1,HTTPFlood,98569,10,0,25,29013,9856.9,9965.921477214235,1.0110604223654736,1160.52
2,ICMPFlood,808,10,0,1,303,80.8,84.37155918910115,1.044202465211648,303.0
3,SYNFlood,6805,10,0,77,1688,680.5,557.1650114642878,0.8187582828277558,21.92207792207792
4,SYNScan,14030,9,1,1,9592,1558.888888888889,2931.2507169179676,1.8803461477021888,9592.0
5,SlowrateDoS,51187,10,0,28,20803,5118.7,7606.314338100943,1.48598557018402,742.9642857142857
6,TCPConnectScan,14037,9,1,7,4056,1559.6666666666667,1501.5754689288476,0.962754094205288,579.4285714285714
7,UDPFlood,320138,10,0,189,107583,32013.8,36126.15487371995,1.1284556932860188,569.2222222222222
8,UDPScan,11134,10,0,5,8048,1113.4,2354.6204874671416,2.1148019467102044,1609.6
1 class_id class_name total num_clients n_missing_clients min max mean std cv max_min_ratio
2 0 Benign 334415 10 0 1571 112226 33441.5 37695.7725116491 1.127215361501401 71.43602800763844
3 1 HTTPFlood 98569 10 0 25 29013 9856.9 9965.921477214235 1.0110604223654736 1160.52
4 2 ICMPFlood 808 10 0 1 303 80.8 84.37155918910115 1.044202465211648 303.0
5 3 SYNFlood 6805 10 0 77 1688 680.5 557.1650114642878 0.8187582828277558 21.92207792207792
6 4 SYNScan 14030 9 1 1 9592 1558.888888888889 2931.2507169179676 1.8803461477021888 9592.0
7 5 SlowrateDoS 51187 10 0 28 20803 5118.7 7606.314338100943 1.48598557018402 742.9642857142857
8 6 TCPConnectScan 14037 9 1 7 4056 1559.6666666666667 1501.5754689288476 0.962754094205288 579.4285714285714
9 7 UDPFlood 320138 10 0 189 107583 32013.8 36126.15487371995 1.1284556932860188 569.2222222222222
10 8 UDPScan 11134 10 0 5 8048 1113.4 2354.6204874671416 2.1148019467102044 1609.6
exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__manifest_meta.json
+13
View File
@@ -0,0 +1,13 @@
{
"alpha": 0.5,
"created_utc": "2026-10-31T05:06:50Z",
"dataset": "5gnidd",
"manifest_content_hash": "bf212b0e11ba63d44be64c11215d085658d506065922a26960391eece1143b5b",
"manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/5gnidd__alpha0.5__seed3.json",
"num_classes": 9,
"num_clients": 10,
"partition_hash": "0739f15c51fe4fa2250f717e8f52548706cb7093581447cd43a4b2364f6ec058",
"preprocessing_hash": "60b5682b39184fd7d24e117430a00c160c5c7e95d1bc1fcaa99745108f514d81",
"procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing",
"seed": 3
}
exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed3__rare_unseen.csv
+91
View File
@@ -0,0 +1,91 @@
dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag
5gnidd,0.5,3,0,0,Benign,12367,False,False
5gnidd,0.5,3,0,1,HTTPFlood,24330,False,False
5gnidd,0.5,3,0,2,ICMPFlood,109,True,False
5gnidd,0.5,3,0,3,SYNFlood,732,False,False
5gnidd,0.5,3,0,4,SYNScan,1299,False,False
5gnidd,0.5,3,0,5,SlowrateDoS,2036,False,False
5gnidd,0.5,3,0,6,TCPConnectScan,261,False,False
5gnidd,0.5,3,0,7,UDPFlood,107583,False,False
5gnidd,0.5,3,0,8,UDPScan,129,True,False
5gnidd,0.5,3,1,0,Benign,86013,False,False
5gnidd,0.5,3,1,1,HTTPFlood,9922,False,False
5gnidd,0.5,3,1,2,ICMPFlood,8,True,False
5gnidd,0.5,3,1,3,SYNFlood,104,False,False
5gnidd,0.5,3,1,4,SYNScan,31,False,False
5gnidd,0.5,3,1,5,SlowrateDoS,3166,False,False
5gnidd,0.5,3,1,6,TCPConnectScan,1928,False,False
5gnidd,0.5,3,1,7,UDPFlood,6739,False,False
5gnidd,0.5,3,1,8,UDPScan,5,True,False
5gnidd,0.5,3,2,0,Benign,112226,False,False
5gnidd,0.5,3,2,1,HTTPFlood,6136,False,False
5gnidd,0.5,3,2,2,ICMPFlood,97,True,False
5gnidd,0.5,3,2,3,SYNFlood,152,False,False
5gnidd,0.5,3,2,4,SYNScan,143,False,False
5gnidd,0.5,3,2,5,SlowrateDoS,28,True,False
5gnidd,0.5,3,2,6,TCPConnectScan,966,False,False
5gnidd,0.5,3,2,7,UDPFlood,13880,False,False
5gnidd,0.5,3,2,8,UDPScan,301,False,False
5gnidd,0.5,3,3,0,Benign,7322,False,False
5gnidd,0.5,3,3,1,HTTPFlood,29013,False,False
5gnidd,0.5,3,3,2,ICMPFlood,115,True,False
5gnidd,0.5,3,3,3,SYNFlood,213,False,False
5gnidd,0.5,3,3,4,SYNScan,9592,False,False
5gnidd,0.5,3,3,5,SlowrateDoS,159,True,False
5gnidd,0.5,3,3,6,TCPConnectScan,373,False,False
5gnidd,0.5,3,3,7,UDPFlood,2354,False,False
5gnidd,0.5,3,3,8,UDPScan,503,False,False
5gnidd,0.5,3,4,0,Benign,28752,False,False
5gnidd,0.5,3,4,1,HTTPFlood,497,False,False
5gnidd,0.5,3,4,2,ICMPFlood,15,True,False
5gnidd,0.5,3,4,3,SYNFlood,1688,False,False
5gnidd,0.5,3,4,4,SYNScan,71,False,False
5gnidd,0.5,3,4,5,SlowrateDoS,19630,False,False
5gnidd,0.5,3,4,6,TCPConnectScan,0,False,True
5gnidd,0.5,3,4,7,UDPFlood,607,False,False
5gnidd,0.5,3,4,8,UDPScan,67,True,False
5gnidd,0.5,3,5,0,Benign,11126,False,False
5gnidd,0.5,3,5,1,HTTPFlood,366,False,False
5gnidd,0.5,3,5,2,ICMPFlood,37,True,False
5gnidd,0.5,3,5,3,SYNFlood,786,False,False
5gnidd,0.5,3,5,4,SYNScan,519,False,False
5gnidd,0.5,3,5,5,SlowrateDoS,1272,False,False
5gnidd,0.5,3,5,6,TCPConnectScan,140,True,False
5gnidd,0.5,3,5,7,UDPFlood,84223,False,False
5gnidd,0.5,3,5,8,UDPScan,1613,False,False
5gnidd,0.5,3,6,0,Benign,7958,False,False
5gnidd,0.5,3,6,1,HTTPFlood,249,False,False
5gnidd,0.5,3,6,2,ICMPFlood,40,True,False
5gnidd,0.5,3,6,3,SYNFlood,912,False,False
5gnidd,0.5,3,6,4,SYNScan,93,True,False
5gnidd,0.5,3,6,5,SlowrateDoS,801,False,False
5gnidd,0.5,3,6,6,TCPConnectScan,3851,False,False
5gnidd,0.5,3,6,7,UDPFlood,45015,False,False
5gnidd,0.5,3,6,8,UDPScan,8048,False,False
5gnidd,0.5,3,7,0,Benign,64584,False,False
5gnidd,0.5,3,7,1,HTTPFlood,25,False,False
5gnidd,0.5,3,7,2,ICMPFlood,1,True,False
5gnidd,0.5,3,7,3,SYNFlood,77,False,False
5gnidd,0.5,3,7,4,SYNScan,2281,False,False
5gnidd,0.5,3,7,5,SlowrateDoS,20803,False,False
5gnidd,0.5,3,7,6,TCPConnectScan,7,True,False
5gnidd,0.5,3,7,7,UDPFlood,12615,False,False
5gnidd,0.5,3,7,8,UDPScan,17,False,False
5gnidd,0.5,3,8,0,Benign,2496,False,False
5gnidd,0.5,3,8,1,HTTPFlood,14929,False,False
5gnidd,0.5,3,8,2,ICMPFlood,303,False,False
5gnidd,0.5,3,8,3,SYNFlood,1582,False,False
5gnidd,0.5,3,8,4,SYNScan,1,True,False
5gnidd,0.5,3,8,5,SlowrateDoS,2192,False,False
5gnidd,0.5,3,8,6,TCPConnectScan,2455,False,False
5gnidd,0.5,3,8,7,UDPFlood,189,True,False
5gnidd,0.5,3,8,8,UDPScan,234,False,False
5gnidd,0.5,3,9,0,Benign,1571,False,False
5gnidd,0.5,3,9,1,HTTPFlood,13102,False,False
5gnidd,0.5,3,9,2,ICMPFlood,83,True,False
5gnidd,0.5,3,9,3,SYNFlood,559,False,False
5gnidd,0.5,3,9,4,SYNScan,0,False,True
5gnidd,0.5,3,9,5,SlowrateDoS,1100,False,False
5gnidd,0.5,3,9,6,TCPConnectScan,4056,False,False
5gnidd,0.5,3,9,7,UDPFlood,46933,False,False
5gnidd,0.5,3,9,8,UDPScan,217,True,False
1 dataset alpha seed client_id class_id class_name N_ij rare_flag unseen_flag
2 5gnidd 0.5 3 0 0 Benign 12367 False False
3 5gnidd 0.5 3 0 1 HTTPFlood 24330 False False
4 5gnidd 0.5 3 0 2 ICMPFlood 109 True False
5 5gnidd 0.5 3 0 3 SYNFlood 732 False False
6 5gnidd 0.5 3 0 4 SYNScan 1299 False False
7 5gnidd 0.5 3 0 5 SlowrateDoS 2036 False False
8 5gnidd 0.5 3 0 6 TCPConnectScan 261 False False
9 5gnidd 0.5 3 0 7 UDPFlood 107583 False False
10 5gnidd 0.5 3 0 8 UDPScan 129 True False
11 5gnidd 0.5 3 1 0 Benign 86013 False False
12 5gnidd 0.5 3 1 1 HTTPFlood 9922 False False
13 5gnidd 0.5 3 1 2 ICMPFlood 8 True False
14 5gnidd 0.5 3 1 3 SYNFlood 104 False False
15 5gnidd 0.5 3 1 4 SYNScan 31 False False
16 5gnidd 0.5 3 1 5 SlowrateDoS 3166 False False
17 5gnidd 0.5 3 1 6 TCPConnectScan 1928 False False
18 5gnidd 0.5 3 1 7 UDPFlood 6739 False False
19 5gnidd 0.5 3 1 8 UDPScan 5 True False
20 5gnidd 0.5 3 2 0 Benign 112226 False False
21 5gnidd 0.5 3 2 1 HTTPFlood 6136 False False
22 5gnidd 0.5 3 2 2 ICMPFlood 97 True False
23 5gnidd 0.5 3 2 3 SYNFlood 152 False False
24 5gnidd 0.5 3 2 4 SYNScan 143 False False
25 5gnidd 0.5 3 2 5 SlowrateDoS 28 True False
26 5gnidd 0.5 3 2 6 TCPConnectScan 966 False False
27 5gnidd 0.5 3 2 7 UDPFlood 13880 False False
28 5gnidd 0.5 3 2 8 UDPScan 301 False False
29 5gnidd 0.5 3 3 0 Benign 7322 False False
30 5gnidd 0.5 3 3 1 HTTPFlood 29013 False False
31 5gnidd 0.5 3 3 2 ICMPFlood 115 True False
32 5gnidd 0.5 3 3 3 SYNFlood 213 False False
33 5gnidd 0.5 3 3 4 SYNScan 9592 False False
34 5gnidd 0.5 3 3 5 SlowrateDoS 159 True False
35 5gnidd 0.5 3 3 6 TCPConnectScan 373 False False
36 5gnidd 0.5 3 3 7 UDPFlood 2354 False False
37 5gnidd 0.5 3 3 8 UDPScan 503 False False
38 5gnidd 0.5 3 4 0 Benign 28752 False False
39 5gnidd 0.5 3 4 1 HTTPFlood 497 False False
40 5gnidd 0.5 3 4 2 ICMPFlood 15 True False
41 5gnidd 0.5 3 4 3 SYNFlood 1688 False False
42 5gnidd 0.5 3 4 4 SYNScan 71 False False
43 5gnidd 0.5 3 4 5 SlowrateDoS 19630 False False
44 5gnidd 0.5 3 4 6 TCPConnectScan 0 False True
45 5gnidd 0.5 3 4 7 UDPFlood 607 False False
46 5gnidd 0.5 3 4 8 UDPScan 67 True False
47 5gnidd 0.5 3 5 0 Benign 11126 False False
48 5gnidd 0.5 3 5 1 HTTPFlood 366 False False
49 5gnidd 0.5 3 5 2 ICMPFlood 37 True False
50 5gnidd 0.5 3 5 3 SYNFlood 786 False False
51 5gnidd 0.5 3 5 4 SYNScan 519 False False
52 5gnidd 0.5 3 5 5 SlowrateDoS 1272 False False
53 5gnidd 0.5 3 5 6 TCPConnectScan 140 True False
54 5gnidd 0.5 3 5 7 UDPFlood 84223 False False
55 5gnidd 0.5 3 5 8 UDPScan 1613 False False
56 5gnidd 0.5 3 6 0 Benign 7958 False False
57 5gnidd 0.5 3 6 1 HTTPFlood 249 False False
58 5gnidd 0.5 3 6 2 ICMPFlood 40 True False
59 5gnidd 0.5 3 6 3 SYNFlood 912 False False
60 5gnidd 0.5 3 6 4 SYNScan 93 True False
61 5gnidd 0.5 3 6 5 SlowrateDoS 801 False False
62 5gnidd 0.5 3 6 6 TCPConnectScan 3851 False False
63 5gnidd 0.5 3 6 7 UDPFlood 45015 False False
64 5gnidd 0.5 3 6 8 UDPScan 8048 False False
65 5gnidd 0.5 3 7 0 Benign 64584 False False
66 5gnidd 0.5 3 7 1 HTTPFlood 25 False False
67 5gnidd 0.5 3 7 2 ICMPFlood 1 True False
68 5gnidd 0.5 3 7 3 SYNFlood 77 False False
69 5gnidd 0.5 3 7 4 SYNScan 2281 False False
70 5gnidd 0.5 3 7 5 SlowrateDoS 20803 False False
71 5gnidd 0.5 3 7 6 TCPConnectScan 7 True False
72 5gnidd 0.5 3 7 7 UDPFlood 12615 False False
73 5gnidd 0.5 3 7 8 UDPScan 17 False False
74 5gnidd 0.5 3 8 0 Benign 2496 False False
75 5gnidd 0.5 3 8 1 HTTPFlood 14929 False False
76 5gnidd 0.5 3 8 2 ICMPFlood 303 False False
77 5gnidd 0.5 3 8 3 SYNFlood 1582 False False
78 5gnidd 0.5 3 8 4 SYNScan 1 True False
79 5gnidd 0.5 3 8 5 SlowrateDoS 2192 False False
80 5gnidd 0.5 3 8 6 TCPConnectScan 2455 False False
81 5gnidd 0.5 3 8 7 UDPFlood 189 True False
82 5gnidd 0.5 3 8 8 UDPScan 234 False False
83 5gnidd 0.5 3 9 0 Benign 1571 False False
84 5gnidd 0.5 3 9 1 HTTPFlood 13102 False False
85 5gnidd 0.5 3 9 2 ICMPFlood 83 True False
86 5gnidd 0.5 3 9 3 SYNFlood 559 False False
87 5gnidd 0.5 3 9 4 SYNScan 0 False True
88 5gnidd 0.5 3 9 5 SlowrateDoS 1100 False False
89 5gnidd 0.5 3 9 6 TCPConnectScan 4056 False False
90 5gnidd 0.5 3 9 7 UDPFlood 46933 False False
91 5gnidd 0.5 3 9 8 UDPScan 217 True False
exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__characterization.csv
+10
View File
@@ -0,0 +1,10 @@
class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio
0,Benign,334415,10,0,143,159078,33441.5,47131.86755317468,1.409382580122742,1112.4335664335665
1,HTTPFlood,98569,10,0,4,47186,9856.9,14843.031647544243,1.5058519055224506,11796.5
2,ICMPFlood,808,9,1,5,318,89.77777777777777,108.34284858497202,1.206789155030629,63.6
3,SYNFlood,6805,10,0,7,2927,680.5,975.6094761737403,1.4336656519819844,418.14285714285717
4,SYNScan,14030,10,0,45,8239,1403.0,2313.2844183109,1.6488128427019957,183.0888888888889
5,SlowrateDoS,51187,10,0,7,22272,5118.7,6802.114951248619,1.3288754862071657,3181.714285714286
6,TCPConnectScan,14037,10,0,1,5229,1403.7,1552.9292353484752,1.1063113452649962,5229.0
7,UDPFlood,320138,10,0,6152,111407,32013.8,32088.759828949453,1.002341484889312,18.109070221066318
8,UDPScan,11134,9,1,15,3055,1237.111111111111,1139.6573992451947,0.9212247703616627,203.66666666666666
1 class_id class_name total num_clients n_missing_clients min max mean std cv max_min_ratio
2 0 Benign 334415 10 0 143 159078 33441.5 47131.86755317468 1.409382580122742 1112.4335664335665
3 1 HTTPFlood 98569 10 0 4 47186 9856.9 14843.031647544243 1.5058519055224506 11796.5
4 2 ICMPFlood 808 9 1 5 318 89.77777777777777 108.34284858497202 1.206789155030629 63.6
5 3 SYNFlood 6805 10 0 7 2927 680.5 975.6094761737403 1.4336656519819844 418.14285714285717
6 4 SYNScan 14030 10 0 45 8239 1403.0 2313.2844183109 1.6488128427019957 183.0888888888889
7 5 SlowrateDoS 51187 10 0 7 22272 5118.7 6802.114951248619 1.3288754862071657 3181.714285714286
8 6 TCPConnectScan 14037 10 0 1 5229 1403.7 1552.9292353484752 1.1063113452649962 5229.0
9 7 UDPFlood 320138 10 0 6152 111407 32013.8 32088.759828949453 1.002341484889312 18.109070221066318
10 8 UDPScan 11134 9 1 15 3055 1237.111111111111 1139.6573992451947 0.9212247703616627 203.66666666666666
exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__manifest_meta.json
+13
View File
@@ -0,0 +1,13 @@
{
"alpha": 0.5,
"created_utc": "2026-10-31T05:07:28Z",
"dataset": "5gnidd",
"manifest_content_hash": "80fad04f10a0014a58ea97e87ab901d70cf48f29d99f5803510c0cd44b2618b9",
"manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/5gnidd__alpha0.5__seed4.json",
"num_classes": 9,
"num_clients": 10,
"partition_hash": "21a507d9d94f31154c5fb0c6833344738d1a137c78950f5284dec26d727e983a",
"preprocessing_hash": "ffb7b2a1adb6cf8a292ea6e88dca574b44446da2b3ae18f690ae6fc59699e23d",
"procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing",
"seed": 4
}
exploration_v2/analysis/partitions/5gnidd__alpha0.5__seed4__rare_unseen.csv
+91
View File
@@ -0,0 +1,91 @@
dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag
5gnidd,0.5,4,0,0,Benign,159078,False,False
5gnidd,0.5,4,0,1,HTTPFlood,11071,False,False
5gnidd,0.5,4,0,2,ICMPFlood,39,True,False
5gnidd,0.5,4,0,3,SYNFlood,936,False,False
5gnidd,0.5,4,0,4,SYNScan,8239,False,False
5gnidd,0.5,4,0,5,SlowrateDoS,1050,False,False
5gnidd,0.5,4,0,6,TCPConnectScan,33,True,False
5gnidd,0.5,4,0,7,UDPFlood,15599,False,False
5gnidd,0.5,4,0,8,UDPScan,241,False,False
5gnidd,0.5,4,1,0,Benign,15658,False,False
5gnidd,0.5,4,1,1,HTTPFlood,27988,False,False
5gnidd,0.5,4,1,2,ICMPFlood,318,False,False
5gnidd,0.5,4,1,3,SYNFlood,7,True,False
5gnidd,0.5,4,1,4,SYNScan,674,False,False
5gnidd,0.5,4,1,5,SlowrateDoS,7,True,False
5gnidd,0.5,4,1,6,TCPConnectScan,5229,False,False
5gnidd,0.5,4,1,7,UDPFlood,6152,False,False
5gnidd,0.5,4,1,8,UDPScan,93,False,False
5gnidd,0.5,4,2,0,Benign,5874,False,False
5gnidd,0.5,4,2,1,HTTPFlood,2211,False,False
5gnidd,0.5,4,2,2,ICMPFlood,0,False,True
5gnidd,0.5,4,2,3,SYNFlood,2113,False,False
5gnidd,0.5,4,2,4,SYNScan,893,False,False
5gnidd,0.5,4,2,5,SlowrateDoS,1049,False,False
5gnidd,0.5,4,2,6,TCPConnectScan,490,True,False
5gnidd,0.5,4,2,7,UDPFlood,13122,False,False
5gnidd,0.5,4,2,8,UDPScan,131,True,False
5gnidd,0.5,4,3,0,Benign,76962,False,False
5gnidd,0.5,4,3,1,HTTPFlood,851,False,False
5gnidd,0.5,4,3,2,ICMPFlood,10,True,False
5gnidd,0.5,4,3,3,SYNFlood,26,True,False
5gnidd,0.5,4,3,4,SYNScan,1223,False,False
5gnidd,0.5,4,3,5,SlowrateDoS,7546,False,False
5gnidd,0.5,4,3,6,TCPConnectScan,846,False,False
5gnidd,0.5,4,3,7,UDPFlood,57683,False,False
5gnidd,0.5,4,3,8,UDPScan,3055,False,False
5gnidd,0.5,4,4,0,Benign,3588,False,False
5gnidd,0.5,4,4,1,HTTPFlood,47186,False,False
5gnidd,0.5,4,4,2,ICMPFlood,243,True,False
5gnidd,0.5,4,4,3,SYNFlood,26,True,False
5gnidd,0.5,4,4,4,SYNScan,930,False,False
5gnidd,0.5,4,4,5,SlowrateDoS,11733,False,False
5gnidd,0.5,4,4,6,TCPConnectScan,1121,False,False
5gnidd,0.5,4,4,7,UDPFlood,22035,False,False
5gnidd,0.5,4,4,8,UDPScan,1898,False,False
5gnidd,0.5,4,5,0,Benign,26897,False,False
5gnidd,0.5,4,5,1,HTTPFlood,235,False,False
5gnidd,0.5,4,5,2,ICMPFlood,60,True,False
5gnidd,0.5,4,5,3,SYNFlood,2927,False,False
5gnidd,0.5,4,5,4,SYNScan,181,True,False
5gnidd,0.5,4,5,5,SlowrateDoS,263,False,False
5gnidd,0.5,4,5,6,TCPConnectScan,2914,False,False
5gnidd,0.5,4,5,7,UDPFlood,58429,False,False
5gnidd,0.5,4,5,8,UDPScan,2934,False,False
5gnidd,0.5,4,6,0,Benign,143,False,False
5gnidd,0.5,4,6,1,HTTPFlood,5129,False,False
5gnidd,0.5,4,6,2,ICMPFlood,112,False,False
5gnidd,0.5,4,6,3,SYNFlood,237,False,False
5gnidd,0.5,4,6,4,SYNScan,45,True,False
5gnidd,0.5,4,6,5,SlowrateDoS,5427,False,False
5gnidd,0.5,4,6,6,TCPConnectScan,36,True,False
5gnidd,0.5,4,6,7,UDPFlood,7654,False,False
5gnidd,0.5,4,6,8,UDPScan,1444,False,False
5gnidd,0.5,4,7,0,Benign,30083,False,False
5gnidd,0.5,4,7,1,HTTPFlood,3124,False,False
5gnidd,0.5,4,7,2,ICMPFlood,10,True,False
5gnidd,0.5,4,7,3,SYNFlood,8,True,False
5gnidd,0.5,4,7,4,SYNScan,79,False,False
5gnidd,0.5,4,7,5,SlowrateDoS,357,False,False
5gnidd,0.5,4,7,6,TCPConnectScan,1910,False,False
5gnidd,0.5,4,7,7,UDPFlood,111407,False,False
5gnidd,0.5,4,7,8,UDPScan,0,False,True
5gnidd,0.5,4,8,0,Benign,405,False,False
5gnidd,0.5,4,8,1,HTTPFlood,4,True,False
5gnidd,0.5,4,8,2,ICMPFlood,5,False,False
5gnidd,0.5,4,8,3,SYNFlood,413,False,False
5gnidd,0.5,4,8,4,SYNScan,1110,False,False
5gnidd,0.5,4,8,5,SlowrateDoS,1483,False,False
5gnidd,0.5,4,8,6,TCPConnectScan,1,True,False
5gnidd,0.5,4,8,7,UDPFlood,14987,False,False
5gnidd,0.5,4,8,8,UDPScan,1323,False,False
5gnidd,0.5,4,9,0,Benign,15727,False,False
5gnidd,0.5,4,9,1,HTTPFlood,770,False,False
5gnidd,0.5,4,9,2,ICMPFlood,11,True,False
5gnidd,0.5,4,9,3,SYNFlood,112,False,False
5gnidd,0.5,4,9,4,SYNScan,656,False,False
5gnidd,0.5,4,9,5,SlowrateDoS,22272,False,False
5gnidd,0.5,4,9,6,TCPConnectScan,1457,False,False
5gnidd,0.5,4,9,7,UDPFlood,13070,False,False
5gnidd,0.5,4,9,8,UDPScan,15,True,False
1 dataset alpha seed client_id class_id class_name N_ij rare_flag unseen_flag
2 5gnidd 0.5 4 0 0 Benign 159078 False False
3 5gnidd 0.5 4 0 1 HTTPFlood 11071 False False
4 5gnidd 0.5 4 0 2 ICMPFlood 39 True False
5 5gnidd 0.5 4 0 3 SYNFlood 936 False False
6 5gnidd 0.5 4 0 4 SYNScan 8239 False False
7 5gnidd 0.5 4 0 5 SlowrateDoS 1050 False False
8 5gnidd 0.5 4 0 6 TCPConnectScan 33 True False
9 5gnidd 0.5 4 0 7 UDPFlood 15599 False False
10 5gnidd 0.5 4 0 8 UDPScan 241 False False
11 5gnidd 0.5 4 1 0 Benign 15658 False False
12 5gnidd 0.5 4 1 1 HTTPFlood 27988 False False
13 5gnidd 0.5 4 1 2 ICMPFlood 318 False False
14 5gnidd 0.5 4 1 3 SYNFlood 7 True False
15 5gnidd 0.5 4 1 4 SYNScan 674 False False
16 5gnidd 0.5 4 1 5 SlowrateDoS 7 True False
17 5gnidd 0.5 4 1 6 TCPConnectScan 5229 False False
18 5gnidd 0.5 4 1 7 UDPFlood 6152 False False
19 5gnidd 0.5 4 1 8 UDPScan 93 False False
20 5gnidd 0.5 4 2 0 Benign 5874 False False
21 5gnidd 0.5 4 2 1 HTTPFlood 2211 False False
22 5gnidd 0.5 4 2 2 ICMPFlood 0 False True
23 5gnidd 0.5 4 2 3 SYNFlood 2113 False False
24 5gnidd 0.5 4 2 4 SYNScan 893 False False
25 5gnidd 0.5 4 2 5 SlowrateDoS 1049 False False
26 5gnidd 0.5 4 2 6 TCPConnectScan 490 True False
27 5gnidd 0.5 4 2 7 UDPFlood 13122 False False
28 5gnidd 0.5 4 2 8 UDPScan 131 True False
29 5gnidd 0.5 4 3 0 Benign 76962 False False
30 5gnidd 0.5 4 3 1 HTTPFlood 851 False False
31 5gnidd 0.5 4 3 2 ICMPFlood 10 True False
32 5gnidd 0.5 4 3 3 SYNFlood 26 True False
33 5gnidd 0.5 4 3 4 SYNScan 1223 False False
34 5gnidd 0.5 4 3 5 SlowrateDoS 7546 False False
35 5gnidd 0.5 4 3 6 TCPConnectScan 846 False False
36 5gnidd 0.5 4 3 7 UDPFlood 57683 False False
37 5gnidd 0.5 4 3 8 UDPScan 3055 False False
38 5gnidd 0.5 4 4 0 Benign 3588 False False
39 5gnidd 0.5 4 4 1 HTTPFlood 47186 False False
40 5gnidd 0.5 4 4 2 ICMPFlood 243 True False
41 5gnidd 0.5 4 4 3 SYNFlood 26 True False
42 5gnidd 0.5 4 4 4 SYNScan 930 False False
43 5gnidd 0.5 4 4 5 SlowrateDoS 11733 False False
44 5gnidd 0.5 4 4 6 TCPConnectScan 1121 False False
45 5gnidd 0.5 4 4 7 UDPFlood 22035 False False
46 5gnidd 0.5 4 4 8 UDPScan 1898 False False
47 5gnidd 0.5 4 5 0 Benign 26897 False False
48 5gnidd 0.5 4 5 1 HTTPFlood 235 False False
49 5gnidd 0.5 4 5 2 ICMPFlood 60 True False
50 5gnidd 0.5 4 5 3 SYNFlood 2927 False False
51 5gnidd 0.5 4 5 4 SYNScan 181 True False
52 5gnidd 0.5 4 5 5 SlowrateDoS 263 False False
53 5gnidd 0.5 4 5 6 TCPConnectScan 2914 False False
54 5gnidd 0.5 4 5 7 UDPFlood 58429 False False
55 5gnidd 0.5 4 5 8 UDPScan 2934 False False
56 5gnidd 0.5 4 6 0 Benign 143 False False
57 5gnidd 0.5 4 6 1 HTTPFlood 5129 False False
58 5gnidd 0.5 4 6 2 ICMPFlood 112 False False
59 5gnidd 0.5 4 6 3 SYNFlood 237 False False
60 5gnidd 0.5 4 6 4 SYNScan 45 True False
61 5gnidd 0.5 4 6 5 SlowrateDoS 5427 False False
62 5gnidd 0.5 4 6 6 TCPConnectScan 36 True False
63 5gnidd 0.5 4 6 7 UDPFlood 7654 False False
64 5gnidd 0.5 4 6 8 UDPScan 1444 False False
65 5gnidd 0.5 4 7 0 Benign 30083 False False
66 5gnidd 0.5 4 7 1 HTTPFlood 3124 False False
67 5gnidd 0.5 4 7 2 ICMPFlood 10 True False
68 5gnidd 0.5 4 7 3 SYNFlood 8 True False
69 5gnidd 0.5 4 7 4 SYNScan 79 False False
70 5gnidd 0.5 4 7 5 SlowrateDoS 357 False False
71 5gnidd 0.5 4 7 6 TCPConnectScan 1910 False False
72 5gnidd 0.5 4 7 7 UDPFlood 111407 False False
73 5gnidd 0.5 4 7 8 UDPScan 0 False True
74 5gnidd 0.5 4 8 0 Benign 405 False False
75 5gnidd 0.5 4 8 1 HTTPFlood 4 True False
76 5gnidd 0.5 4 8 2 ICMPFlood 5 False False
77 5gnidd 0.5 4 8 3 SYNFlood 413 False False
78 5gnidd 0.5 4 8 4 SYNScan 1110 False False
79 5gnidd 0.5 4 8 5 SlowrateDoS 1483 False False
80 5gnidd 0.5 4 8 6 TCPConnectScan 1 True False
81 5gnidd 0.5 4 8 7 UDPFlood 14987 False False
82 5gnidd 0.5 4 8 8 UDPScan 1323 False False
83 5gnidd 0.5 4 9 0 Benign 15727 False False
84 5gnidd 0.5 4 9 1 HTTPFlood 770 False False
85 5gnidd 0.5 4 9 2 ICMPFlood 11 True False
86 5gnidd 0.5 4 9 3 SYNFlood 112 False False
87 5gnidd 0.5 4 9 4 SYNScan 656 False False
88 5gnidd 0.5 4 9 5 SlowrateDoS 22272 False False
89 5gnidd 0.5 4 9 6 TCPConnectScan 1457 False False
90 5gnidd 0.5 4 9 7 UDPFlood 13070 False False
91 5gnidd 0.5 4 9 8 UDPScan 15 True False
exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__characterization.csv
+10
View File
@@ -0,0 +1,10 @@
class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio
0,C&C,2004,8,2,4,1191,250.5,373.4360721730026,1.490762763165679,297.75
1,Exfiltration,15494,10,0,2,5033,1549.4,1886.1375983739892,1.217334192832057,2516.5
2,Exploitation,793,6,4,7,473,132.16666666666666,167.6011303329691,1.268104390917799,67.57142857142857
3,Lateral_movement,22117,9,1,2,13685,2457.4444444444443,4284.391882457665,1.7434338717782243,6842.5
4,Normal,292777,10,0,549,150706,29277.7,51810.80511254385,1.769633718240977,274.5100182149362
5,RDOS,98883,10,0,9,58544,9888.3,17738.631790811825,1.7939010538527176,6504.888888888889
6,Reconnaissance,89161,10,0,2,35323,8916.1,10550.735959638077,1.183335310240809,17661.5
7,Tampering,3586,7,3,2,2013,512.2857142857143,687.5847406030795,1.3421899565592739,1006.5
8,Weaponization,45094,10,0,31,25866,4509.4,7690.813119560246,1.7055069675700196,834.3870967741935
1 class_id class_name total num_clients n_missing_clients min max mean std cv max_min_ratio
2 0 C&C 2004 8 2 4 1191 250.5 373.4360721730026 1.490762763165679 297.75
3 1 Exfiltration 15494 10 0 2 5033 1549.4 1886.1375983739892 1.217334192832057 2516.5
4 2 Exploitation 793 6 4 7 473 132.16666666666666 167.6011303329691 1.268104390917799 67.57142857142857
5 3 Lateral_movement 22117 9 1 2 13685 2457.4444444444443 4284.391882457665 1.7434338717782243 6842.5
6 4 Normal 292777 10 0 549 150706 29277.7 51810.80511254385 1.769633718240977 274.5100182149362
7 5 RDOS 98883 10 0 9 58544 9888.3 17738.631790811825 1.7939010538527176 6504.888888888889
8 6 Reconnaissance 89161 10 0 2 35323 8916.1 10550.735959638077 1.183335310240809 17661.5
9 7 Tampering 3586 7 3 2 2013 512.2857142857143 687.5847406030795 1.3421899565592739 1006.5
10 8 Weaponization 45094 10 0 31 25866 4509.4 7690.813119560246 1.7055069675700196 834.3870967741935
exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__manifest_meta.json
+13
View File
@@ -0,0 +1,13 @@
{
"alpha": 0.25,
"created_utc": "2026-10-31T05:05:04Z",
"dataset": "xiiotid",
"manifest_content_hash": "1021d6ce77fdb0d4ddf940988f221fd85c8be4c2162d804e1c0da169ed9d7e2f",
"manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/xiiotid__alpha0.25__seed3.json",
"num_classes": 9,
"num_clients": 10,
"partition_hash": "85a8625b81d3624dec9e5322979e2f1547f330c60d5e822e32c12177195ed602",
"preprocessing_hash": "e22db95e5109bfa02701d8ce450453a7c8f7e408c94e81dfe2b95ef4ab2a441f",
"procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing",
"seed": 3
}
exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed3__rare_unseen.csv
+91
View File
@@ -0,0 +1,91 @@
dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag
xiiotid,0.25,3,0,0,C&C,21,False,False
xiiotid,0.25,3,0,1,Exfiltration,1003,False,False
xiiotid,0.25,3,0,2,Exploitation,0,False,True
xiiotid,0.25,3,0,3,Lateral_movement,7,True,False
xiiotid,0.25,3,0,4,Normal,2950,False,False
xiiotid,0.25,3,0,5,RDOS,14,True,False
xiiotid,0.25,3,0,6,Reconnaissance,11735,False,False
xiiotid,0.25,3,0,7,Tampering,0,False,True
xiiotid,0.25,3,0,8,Weaponization,342,False,False
xiiotid,0.25,3,1,0,C&C,27,True,False
xiiotid,0.25,3,1,1,Exfiltration,1029,False,False
xiiotid,0.25,3,1,2,Exploitation,7,True,False
xiiotid,0.25,3,1,3,Lateral_movement,199,False,False
xiiotid,0.25,3,1,4,Normal,549,False,False
xiiotid,0.25,3,1,5,RDOS,1426,False,False
xiiotid,0.25,3,1,6,Reconnaissance,8567,False,False
xiiotid,0.25,3,1,7,Tampering,370,False,False
xiiotid,0.25,3,1,8,Weaponization,1278,False,False
xiiotid,0.25,3,2,0,C&C,1191,False,False
xiiotid,0.25,3,2,1,Exfiltration,483,False,False
xiiotid,0.25,3,2,2,Exploitation,23,True,False
xiiotid,0.25,3,2,3,Lateral_movement,5147,False,False
xiiotid,0.25,3,2,4,Normal,17775,False,False
xiiotid,0.25,3,2,5,RDOS,234,False,False
xiiotid,0.25,3,2,6,Reconnaissance,139,False,False
xiiotid,0.25,3,2,7,Tampering,0,False,True
xiiotid,0.25,3,2,8,Weaponization,31,True,False
xiiotid,0.25,3,3,0,C&C,0,False,True
xiiotid,0.25,3,3,1,Exfiltration,2,True,False
xiiotid,0.25,3,3,2,Exploitation,36,False,False
xiiotid,0.25,3,3,3,Lateral_movement,0,False,True
xiiotid,0.25,3,3,4,Normal,3006,False,False
xiiotid,0.25,3,3,5,RDOS,58,False,False
xiiotid,0.25,3,3,6,Reconnaissance,12,True,False
xiiotid,0.25,3,3,7,Tampering,12,False,False
xiiotid,0.25,3,3,8,Weaponization,1420,False,False
xiiotid,0.25,3,4,0,C&C,15,True,False
xiiotid,0.25,3,4,1,Exfiltration,18,True,False
xiiotid,0.25,3,4,2,Exploitation,39,False,False
xiiotid,0.25,3,4,3,Lateral_movement,2326,False,False
xiiotid,0.25,3,4,4,Normal,2692,False,False
xiiotid,0.25,3,4,5,RDOS,58544,False,False
xiiotid,0.25,3,4,6,Reconnaissance,773,False,False
xiiotid,0.25,3,4,7,Tampering,0,False,True
xiiotid,0.25,3,4,8,Weaponization,25866,False,False
xiiotid,0.25,3,5,0,C&C,320,True,False
xiiotid,0.25,3,5,1,Exfiltration,4958,False,False
xiiotid,0.25,3,5,2,Exploitation,0,False,True
xiiotid,0.25,3,5,3,Lateral_movement,31,True,False
xiiotid,0.25,3,5,4,Normal,966,False,False
xiiotid,0.25,3,5,5,RDOS,18218,False,False
xiiotid,0.25,3,5,6,Reconnaissance,3006,False,False
xiiotid,0.25,3,5,7,Tampering,949,False,False
xiiotid,0.25,3,5,8,Weaponization,8009,False,False
xiiotid,0.25,3,6,0,C&C,0,False,True
xiiotid,0.25,3,6,1,Exfiltration,2705,False,False
xiiotid,0.25,3,6,2,Exploitation,215,False,False
xiiotid,0.25,3,6,3,Lateral_movement,167,True,False
xiiotid,0.25,3,6,4,Normal,1157,False,False
xiiotid,0.25,3,6,5,RDOS,1247,False,False
xiiotid,0.25,3,6,6,Reconnaissance,13942,False,False
xiiotid,0.25,3,6,7,Tampering,2,True,False
xiiotid,0.25,3,6,8,Weaponization,347,False,False
xiiotid,0.25,3,7,0,C&C,165,False,False
xiiotid,0.25,3,7,1,Exfiltration,35,True,False
xiiotid,0.25,3,7,2,Exploitation,473,False,False
xiiotid,0.25,3,7,3,Lateral_movement,13685,False,False
xiiotid,0.25,3,7,4,Normal,111159,False,False
xiiotid,0.25,3,7,5,RDOS,19011,False,False
xiiotid,0.25,3,7,6,Reconnaissance,15662,False,False
xiiotid,0.25,3,7,7,Tampering,2013,False,False
xiiotid,0.25,3,7,8,Weaponization,120,True,False
xiiotid,0.25,3,8,0,C&C,4,False,False
xiiotid,0.25,3,8,1,Exfiltration,228,False,False
xiiotid,0.25,3,8,2,Exploitation,0,False,True
xiiotid,0.25,3,8,3,Lateral_movement,2,True,False
xiiotid,0.25,3,8,4,Normal,150706,False,False
xiiotid,0.25,3,8,5,RDOS,122,False,False
xiiotid,0.25,3,8,6,Reconnaissance,35323,False,False
xiiotid,0.25,3,8,7,Tampering,2,True,False
xiiotid,0.25,3,8,8,Weaponization,83,False,False
xiiotid,0.25,3,9,0,C&C,261,False,False
xiiotid,0.25,3,9,1,Exfiltration,5033,False,False
xiiotid,0.25,3,9,2,Exploitation,0,False,True
xiiotid,0.25,3,9,3,Lateral_movement,553,False,False
xiiotid,0.25,3,9,4,Normal,1817,False,False
xiiotid,0.25,3,9,5,RDOS,9,True,False
xiiotid,0.25,3,9,6,Reconnaissance,2,True,False
xiiotid,0.25,3,9,7,Tampering,238,False,False
xiiotid,0.25,3,9,8,Weaponization,7598,False,False
1 dataset alpha seed client_id class_id class_name N_ij rare_flag unseen_flag
2 xiiotid 0.25 3 0 0 C&C 21 False False
3 xiiotid 0.25 3 0 1 Exfiltration 1003 False False
4 xiiotid 0.25 3 0 2 Exploitation 0 False True
5 xiiotid 0.25 3 0 3 Lateral_movement 7 True False
6 xiiotid 0.25 3 0 4 Normal 2950 False False
7 xiiotid 0.25 3 0 5 RDOS 14 True False
8 xiiotid 0.25 3 0 6 Reconnaissance 11735 False False
9 xiiotid 0.25 3 0 7 Tampering 0 False True
10 xiiotid 0.25 3 0 8 Weaponization 342 False False
11 xiiotid 0.25 3 1 0 C&C 27 True False
12 xiiotid 0.25 3 1 1 Exfiltration 1029 False False
13 xiiotid 0.25 3 1 2 Exploitation 7 True False
14 xiiotid 0.25 3 1 3 Lateral_movement 199 False False
15 xiiotid 0.25 3 1 4 Normal 549 False False
16 xiiotid 0.25 3 1 5 RDOS 1426 False False
17 xiiotid 0.25 3 1 6 Reconnaissance 8567 False False
18 xiiotid 0.25 3 1 7 Tampering 370 False False
19 xiiotid 0.25 3 1 8 Weaponization 1278 False False
20 xiiotid 0.25 3 2 0 C&C 1191 False False
21 xiiotid 0.25 3 2 1 Exfiltration 483 False False
22 xiiotid 0.25 3 2 2 Exploitation 23 True False
23 xiiotid 0.25 3 2 3 Lateral_movement 5147 False False
24 xiiotid 0.25 3 2 4 Normal 17775 False False
25 xiiotid 0.25 3 2 5 RDOS 234 False False
26 xiiotid 0.25 3 2 6 Reconnaissance 139 False False
27 xiiotid 0.25 3 2 7 Tampering 0 False True
28 xiiotid 0.25 3 2 8 Weaponization 31 True False
29 xiiotid 0.25 3 3 0 C&C 0 False True
30 xiiotid 0.25 3 3 1 Exfiltration 2 True False
31 xiiotid 0.25 3 3 2 Exploitation 36 False False
32 xiiotid 0.25 3 3 3 Lateral_movement 0 False True
33 xiiotid 0.25 3 3 4 Normal 3006 False False
34 xiiotid 0.25 3 3 5 RDOS 58 False False
35 xiiotid 0.25 3 3 6 Reconnaissance 12 True False
36 xiiotid 0.25 3 3 7 Tampering 12 False False
37 xiiotid 0.25 3 3 8 Weaponization 1420 False False
38 xiiotid 0.25 3 4 0 C&C 15 True False
39 xiiotid 0.25 3 4 1 Exfiltration 18 True False
40 xiiotid 0.25 3 4 2 Exploitation 39 False False
41 xiiotid 0.25 3 4 3 Lateral_movement 2326 False False
42 xiiotid 0.25 3 4 4 Normal 2692 False False
43 xiiotid 0.25 3 4 5 RDOS 58544 False False
44 xiiotid 0.25 3 4 6 Reconnaissance 773 False False
45 xiiotid 0.25 3 4 7 Tampering 0 False True
46 xiiotid 0.25 3 4 8 Weaponization 25866 False False
47 xiiotid 0.25 3 5 0 C&C 320 True False
48 xiiotid 0.25 3 5 1 Exfiltration 4958 False False
49 xiiotid 0.25 3 5 2 Exploitation 0 False True
50 xiiotid 0.25 3 5 3 Lateral_movement 31 True False
51 xiiotid 0.25 3 5 4 Normal 966 False False
52 xiiotid 0.25 3 5 5 RDOS 18218 False False
53 xiiotid 0.25 3 5 6 Reconnaissance 3006 False False
54 xiiotid 0.25 3 5 7 Tampering 949 False False
55 xiiotid 0.25 3 5 8 Weaponization 8009 False False
56 xiiotid 0.25 3 6 0 C&C 0 False True
57 xiiotid 0.25 3 6 1 Exfiltration 2705 False False
58 xiiotid 0.25 3 6 2 Exploitation 215 False False
59 xiiotid 0.25 3 6 3 Lateral_movement 167 True False
60 xiiotid 0.25 3 6 4 Normal 1157 False False
61 xiiotid 0.25 3 6 5 RDOS 1247 False False
62 xiiotid 0.25 3 6 6 Reconnaissance 13942 False False
63 xiiotid 0.25 3 6 7 Tampering 2 True False
64 xiiotid 0.25 3 6 8 Weaponization 347 False False
65 xiiotid 0.25 3 7 0 C&C 165 False False
66 xiiotid 0.25 3 7 1 Exfiltration 35 True False
67 xiiotid 0.25 3 7 2 Exploitation 473 False False
68 xiiotid 0.25 3 7 3 Lateral_movement 13685 False False
69 xiiotid 0.25 3 7 4 Normal 111159 False False
70 xiiotid 0.25 3 7 5 RDOS 19011 False False
71 xiiotid 0.25 3 7 6 Reconnaissance 15662 False False
72 xiiotid 0.25 3 7 7 Tampering 2013 False False
73 xiiotid 0.25 3 7 8 Weaponization 120 True False
74 xiiotid 0.25 3 8 0 C&C 4 False False
75 xiiotid 0.25 3 8 1 Exfiltration 228 False False
76 xiiotid 0.25 3 8 2 Exploitation 0 False True
77 xiiotid 0.25 3 8 3 Lateral_movement 2 True False
78 xiiotid 0.25 3 8 4 Normal 150706 False False
79 xiiotid 0.25 3 8 5 RDOS 122 False False
80 xiiotid 0.25 3 8 6 Reconnaissance 35323 False False
81 xiiotid 0.25 3 8 7 Tampering 2 True False
82 xiiotid 0.25 3 8 8 Weaponization 83 False False
83 xiiotid 0.25 3 9 0 C&C 261 False False
84 xiiotid 0.25 3 9 1 Exfiltration 5033 False False
85 xiiotid 0.25 3 9 2 Exploitation 0 False True
86 xiiotid 0.25 3 9 3 Lateral_movement 553 False False
87 xiiotid 0.25 3 9 4 Normal 1817 False False
88 xiiotid 0.25 3 9 5 RDOS 9 True False
89 xiiotid 0.25 3 9 6 Reconnaissance 2 True False
90 xiiotid 0.25 3 9 7 Tampering 238 False False
91 xiiotid 0.25 3 9 8 Weaponization 7598 False False
exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__characterization.csv
+10
View File
@@ -0,0 +1,10 @@
class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio
0,C&C,2004,9,1,1,1110,222.66666666666666,338.2435020316971,1.5190576438549273,1110.0
1,Exfiltration,15494,10,0,1,7839,1549.4,2701.0935637256257,1.743315840793614,7839.0
2,Exploitation,793,8,2,3,667,99.125,215.40568556795338,2.1730712289326948,222.33333333333334
3,Lateral_movement,22117,9,1,11,9367,2457.4444444444443,3120.737203187417,1.2699115987107996,851.5454545454545
4,Normal,292777,9,1,1,193448,32530.777777777777,58303.312956712805,1.7922508141364084,193448.0
5,RDOS,98883,9,1,24,36163,10987.0,13856.277750936977,1.2611520661633728,1506.7916666666667
6,Reconnaissance,89161,10,0,25,32287,8916.1,10687.687022457198,1.1986952840880203,1291.48
7,Tampering,3586,10,0,3,2959,358.6,870.6838921215897,2.4280086227595916,986.3333333333334
8,Weaponization,45094,10,0,4,18287,4509.4,6073.340385652693,1.3468178439820582,4571.75
1 class_id class_name total num_clients n_missing_clients min max mean std cv max_min_ratio
2 0 C&C 2004 9 1 1 1110 222.66666666666666 338.2435020316971 1.5190576438549273 1110.0
3 1 Exfiltration 15494 10 0 1 7839 1549.4 2701.0935637256257 1.743315840793614 7839.0
4 2 Exploitation 793 8 2 3 667 99.125 215.40568556795338 2.1730712289326948 222.33333333333334
5 3 Lateral_movement 22117 9 1 11 9367 2457.4444444444443 3120.737203187417 1.2699115987107996 851.5454545454545
6 4 Normal 292777 9 1 1 193448 32530.777777777777 58303.312956712805 1.7922508141364084 193448.0
7 5 RDOS 98883 9 1 24 36163 10987.0 13856.277750936977 1.2611520661633728 1506.7916666666667
8 6 Reconnaissance 89161 10 0 25 32287 8916.1 10687.687022457198 1.1986952840880203 1291.48
9 7 Tampering 3586 10 0 3 2959 358.6 870.6838921215897 2.4280086227595916 986.3333333333334
10 8 Weaponization 45094 10 0 4 18287 4509.4 6073.340385652693 1.3468178439820582 4571.75
exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__manifest_meta.json
+13
View File
@@ -0,0 +1,13 @@
{
"alpha": 0.25,
"created_utc": "2026-10-31T05:06:13Z",
"dataset": "xiiotid",
"manifest_content_hash": "1ea850f1be9ddda24a69d192ef678cb337e62edb60978985615271a15d46d2c8",
"manifest_path": "/home/willtanoe/project/prototype-agregation/exploration_v2/partitions/xiiotid__alpha0.25__seed4.json",
"num_classes": 9,
"num_clients": 10,
"partition_hash": "ff47f504d5b604935333ccbfddd060c18d44d672104e5455f86dcefe48315887",
"preprocessing_hash": "4837f4ab5939fe7bf7f83168d915e67d570086d8534a521a9a9827f10aa41086",
"procedure": "protean_research.data.pipeline.create_partition (per_class_dirichlet_multinomial), frozen split/preprocessing",
"seed": 4
}
exploration_v2/analysis/partitions/xiiotid__alpha0.25__seed4__rare_unseen.csv
+91
View File
@@ -0,0 +1,91 @@
dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag
xiiotid,0.25,4,0,0,C&C,1,True,False
xiiotid,0.25,4,0,1,Exfiltration,109,False,False
xiiotid,0.25,4,0,2,Exploitation,667,False,False
xiiotid,0.25,4,0,3,Lateral_movement,156,False,False
xiiotid,0.25,4,0,4,Normal,193448,False,False
xiiotid,0.25,4,0,5,RDOS,29952,False,False
xiiotid,0.25,4,0,6,Reconnaissance,1314,False,False
xiiotid,0.25,4,0,7,Tampering,3,True,False
xiiotid,0.25,4,0,8,Weaponization,9802,False,False
xiiotid,0.25,4,1,0,C&C,20,False,False
xiiotid,0.25,4,1,1,Exfiltration,368,False,False
xiiotid,0.25,4,1,2,Exploitation,4,True,False
xiiotid,0.25,4,1,3,Lateral_movement,1852,False,False
xiiotid,0.25,4,1,4,Normal,13384,False,False
xiiotid,0.25,4,1,5,RDOS,262,False,False
xiiotid,0.25,4,1,6,Reconnaissance,1102,False,False
xiiotid,0.25,4,1,7,Tampering,42,False,False
xiiotid,0.25,4,1,8,Weaponization,4,True,False
xiiotid,0.25,4,2,0,C&C,58,False,False
xiiotid,0.25,4,2,1,Exfiltration,336,False,False
xiiotid,0.25,4,2,2,Exploitation,0,False,True
xiiotid,0.25,4,2,3,Lateral_movement,1331,False,False
xiiotid,0.25,4,2,4,Normal,0,False,False
xiiotid,0.25,4,2,5,RDOS,218,False,False
xiiotid,0.25,4,2,6,Reconnaissance,32287,False,False
xiiotid,0.25,4,2,7,Tampering,5,True,False
xiiotid,0.25,4,2,8,Weaponization,27,True,False
xiiotid,0.25,4,3,0,C&C,0,False,True
xiiotid,0.25,4,3,1,Exfiltration,7839,False,False
xiiotid,0.25,4,3,2,Exploitation,10,True,False
xiiotid,0.25,4,3,3,Lateral_movement,11,True,False
xiiotid,0.25,4,3,4,Normal,22325,False,False
xiiotid,0.25,4,3,5,RDOS,619,False,False
xiiotid,0.25,4,3,6,Reconnaissance,12186,False,False
xiiotid,0.25,4,3,7,Tampering,2959,False,False
xiiotid,0.25,4,3,8,Weaponization,7624,False,False
xiiotid,0.25,4,4,0,C&C,413,False,False
xiiotid,0.25,4,4,1,Exfiltration,94,False,False
xiiotid,0.25,4,4,2,Exploitation,4,True,False
xiiotid,0.25,4,4,3,Lateral_movement,9367,False,False
xiiotid,0.25,4,4,4,Normal,13997,False,False
xiiotid,0.25,4,4,5,RDOS,0,False,True
xiiotid,0.25,4,4,6,Reconnaissance,1924,False,False
xiiotid,0.25,4,4,7,Tampering,6,True,False
xiiotid,0.25,4,4,8,Weaponization,47,False,False
xiiotid,0.25,4,5,0,C&C,238,False,False
xiiotid,0.25,4,5,1,Exfiltration,5,True,False
xiiotid,0.25,4,5,2,Exploitation,3,True,False
xiiotid,0.25,4,5,3,Lateral_movement,0,False,True
xiiotid,0.25,4,5,4,Normal,1,False,False
xiiotid,0.25,4,5,5,RDOS,24028,False,False
xiiotid,0.25,4,5,6,Reconnaissance,22974,False,False
xiiotid,0.25,4,5,7,Tampering,62,False,False
xiiotid,0.25,4,5,8,Weaponization,5,False,False
xiiotid,0.25,4,6,0,C&C,4,True,False
xiiotid,0.25,4,6,1,Exfiltration,1,True,False
xiiotid,0.25,4,6,2,Exploitation,50,False,False
xiiotid,0.25,4,6,3,Lateral_movement,121,False,False
xiiotid,0.25,4,6,4,Normal,167,False,False
xiiotid,0.25,4,6,5,RDOS,36163,False,False
xiiotid,0.25,4,6,6,Reconnaissance,14248,False,False
xiiotid,0.25,4,6,7,Tampering,75,False,False
xiiotid,0.25,4,6,8,Weaponization,18287,False,False
xiiotid,0.25,4,7,0,C&C,1110,False,False
xiiotid,0.25,4,7,1,Exfiltration,508,False,False
xiiotid,0.25,4,7,2,Exploitation,47,True,False
xiiotid,0.25,4,7,3,Lateral_movement,6770,False,False
xiiotid,0.25,4,7,4,Normal,61,False,False
xiiotid,0.25,4,7,5,RDOS,2656,False,False
xiiotid,0.25,4,7,6,Reconnaissance,25,True,False
xiiotid,0.25,4,7,7,Tampering,88,False,False
xiiotid,0.25,4,7,8,Weaponization,147,False,False
xiiotid,0.25,4,8,0,C&C,78,False,False
xiiotid,0.25,4,8,1,Exfiltration,328,False,False
xiiotid,0.25,4,8,2,Exploitation,0,False,True
xiiotid,0.25,4,8,3,Lateral_movement,1112,False,False
xiiotid,0.25,4,8,4,Normal,42050,False,False
xiiotid,0.25,4,8,5,RDOS,24,True,False
xiiotid,0.25,4,8,6,Reconnaissance,132,False,False
xiiotid,0.25,4,8,7,Tampering,46,False,False
xiiotid,0.25,4,8,8,Weaponization,10,True,False
xiiotid,0.25,4,9,0,C&C,82,True,False
xiiotid,0.25,4,9,1,Exfiltration,5906,False,False
xiiotid,0.25,4,9,2,Exploitation,8,True,False
xiiotid,0.25,4,9,3,Lateral_movement,1397,False,False
xiiotid,0.25,4,9,4,Normal,7344,False,False
xiiotid,0.25,4,9,5,RDOS,4961,False,False
xiiotid,0.25,4,9,6,Reconnaissance,2969,False,False
xiiotid,0.25,4,9,7,Tampering,300,False,False
xiiotid,0.25,4,9,8,Weaponization,9141,False,False
1 dataset alpha seed client_id class_id class_name N_ij rare_flag unseen_flag
2 xiiotid 0.25 4 0 0 C&C 1 True False
3 xiiotid 0.25 4 0 1 Exfiltration 109 False False
4 xiiotid 0.25 4 0 2 Exploitation 667 False False
5 xiiotid 0.25 4 0 3 Lateral_movement 156 False False
6 xiiotid 0.25 4 0 4 Normal 193448 False False
7 xiiotid 0.25 4 0 5 RDOS 29952 False False
8 xiiotid 0.25 4 0 6 Reconnaissance 1314 False False
9 xiiotid 0.25 4 0 7 Tampering 3 True False
10 xiiotid 0.25 4 0 8 Weaponization 9802 False False
11 xiiotid 0.25 4 1 0 C&C 20 False False
12 xiiotid 0.25 4 1 1 Exfiltration 368 False False
13 xiiotid 0.25 4 1 2 Exploitation 4 True False
14 xiiotid 0.25 4 1 3 Lateral_movement 1852 False False
15 xiiotid 0.25 4 1 4 Normal 13384 False False
16 xiiotid 0.25 4 1 5 RDOS 262 False False
17 xiiotid 0.25 4 1 6 Reconnaissance 1102 False False
18 xiiotid 0.25 4 1 7 Tampering 42 False False
19 xiiotid 0.25 4 1 8 Weaponization 4 True False
20 xiiotid 0.25 4 2 0 C&C 58 False False
21 xiiotid 0.25 4 2 1 Exfiltration 336 False False
22 xiiotid 0.25 4 2 2 Exploitation 0 False True
23 xiiotid 0.25 4 2 3 Lateral_movement 1331 False False
24 xiiotid 0.25 4 2 4 Normal 0 False False
25 xiiotid 0.25 4 2 5 RDOS 218 False False
26 xiiotid 0.25 4 2 6 Reconnaissance 32287 False False
27 xiiotid 0.25 4 2 7 Tampering 5 True False
28 xiiotid 0.25 4 2 8 Weaponization 27 True False
29 xiiotid 0.25 4 3 0 C&C 0 False True
30 xiiotid 0.25 4 3 1 Exfiltration 7839 False False
31 xiiotid 0.25 4 3 2 Exploitation 10 True False
32 xiiotid 0.25 4 3 3 Lateral_movement 11 True False
33 xiiotid 0.25 4 3 4 Normal 22325 False False
34 xiiotid 0.25 4 3 5 RDOS 619 False False
35 xiiotid 0.25 4 3 6 Reconnaissance 12186 False False
36 xiiotid 0.25 4 3 7 Tampering 2959 False False
37 xiiotid 0.25 4 3 8 Weaponization 7624 False False
38 xiiotid 0.25 4 4 0 C&C 413 False False
39 xiiotid 0.25 4 4 1 Exfiltration 94 False False
40 xiiotid 0.25 4 4 2 Exploitation 4 True False
41 xiiotid 0.25 4 4 3 Lateral_movement 9367 False False
42 xiiotid 0.25 4 4 4 Normal 13997 False False
43 xiiotid 0.25 4 4 5 RDOS 0 False True
44 xiiotid 0.25 4 4 6 Reconnaissance 1924 False False
45 xiiotid 0.25 4 4 7 Tampering 6 True False
46 xiiotid 0.25 4 4 8 Weaponization 47 False False
47 xiiotid 0.25 4 5 0 C&C 238 False False
48 xiiotid 0.25 4 5 1 Exfiltration 5 True False
49 xiiotid 0.25 4 5 2 Exploitation 3 True False
50 xiiotid 0.25 4 5 3 Lateral_movement 0 False True
51 xiiotid 0.25 4 5 4 Normal 1 False False
52 xiiotid 0.25 4 5 5 RDOS 24028 False False
53 xiiotid 0.25 4 5 6 Reconnaissance 22974 False False
54 xiiotid 0.25 4 5 7 Tampering 62 False False
55 xiiotid 0.25 4 5 8 Weaponization 5 False False
56 xiiotid 0.25 4 6 0 C&C 4 True False
57 xiiotid 0.25 4 6 1 Exfiltration 1 True False
58 xiiotid 0.25 4 6 2 Exploitation 50 False False
59 xiiotid 0.25 4 6 3 Lateral_movement 121 False False
60 xiiotid 0.25 4 6 4 Normal 167 False False
61 xiiotid 0.25 4 6 5 RDOS 36163 False False
62 xiiotid 0.25 4 6 6 Reconnaissance 14248 False False
63 xiiotid 0.25 4 6 7 Tampering 75 False False
64 xiiotid 0.25 4 6 8 Weaponization 18287 False False
65 xiiotid 0.25 4 7 0 C&C 1110 False False
66 xiiotid 0.25 4 7 1 Exfiltration 508 False False
67 xiiotid 0.25 4 7 2 Exploitation 47 True False
68 xiiotid 0.25 4 7 3 Lateral_movement 6770 False False
69 xiiotid 0.25 4 7 4 Normal 61 False False
70 xiiotid 0.25 4 7 5 RDOS 2656 False False
71 xiiotid 0.25 4 7 6 Reconnaissance 25 True False
72 xiiotid 0.25 4 7 7 Tampering 88 False False
73 xiiotid 0.25 4 7 8 Weaponization 147 False False
74 xiiotid 0.25 4 8 0 C&C 78 False False
75 xiiotid 0.25 4 8 1 Exfiltration 328 False False
76 xiiotid 0.25 4 8 2 Exploitation 0 False True
77 xiiotid 0.25 4 8 3 Lateral_movement 1112 False False
78 xiiotid 0.25 4 8 4 Normal 42050 False False
79 xiiotid 0.25 4 8 5 RDOS 24 True False
80 xiiotid 0.25 4 8 6 Reconnaissance 132 False False
81 xiiotid 0.25 4 8 7 Tampering 46 False False
82 xiiotid 0.25 4 8 8 Weaponization 10 True False
83 xiiotid 0.25 4 9 0 C&C 82 True False
84 xiiotid 0.25 4 9 1 Exfiltration 5906 False False
85 xiiotid 0.25 4 9 2 Exploitation 8 True False
86 xiiotid 0.25 4 9 3 Lateral_movement 1397 False False
87 xiiotid 0.25 4 9 4 Normal 7344 False False
88 xiiotid 0.25 4 9 5 RDOS 4961 False False
89 xiiotid 0.25 4 9 6 Reconnaissance 2969 False False
90 xiiotid 0.25 4 9 7 Tampering 300 False False
91 xiiotid 0.25 4 9 8 Weaponization 9141 False False
exploration_v2/analysis/reference_consistency.csv
+3
View File
@@ -0,0 +1,3 @@
condition,dataset,alpha,new_original_val_global,restored_original_val_global,sweep_beta0_val_global,new_minus_restored,new_minus_sweep_beta0
S1,xiiotid,0.25,0.6380388187784778,0.6415812303524282,0.643275274917486,-0.0035424115739504103,-0.00523645613900825
S2,5gnidd,0.5,0.8615257707605136,0.8694027416468965,0.8676517373672202,-0.00787697088638284,-0.006125966606706612
1 condition dataset alpha new_original_val_global restored_original_val_global sweep_beta0_val_global new_minus_restored new_minus_sweep_beta0
2 S1 xiiotid 0.25 0.6380388187784778 0.6415812303524282 0.643275274917486 -0.0035424115739504103 -0.00523645613900825
3 S2 5gnidd 0.5 0.8615257707605136 0.8694027416468965 0.8676517373672202 -0.00787697088638284 -0.006125966606706612
exploration_v2/analysis/run_inventory.csv
+35
View File
@@ -0,0 +1,35 @@
phase,run_id,dataset,alpha,seed,mode,beta,shrink_lambda,cap_factor,min_support,label,status,wall_s,peak_rss_kb,exit_code,notes
screening,xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,0,original_protean,,,,,original_protean,done,10879.00,8788432,0,
screening,5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0,original_protean,,,,,original_protean,done,16807.00,10800928,0,
screening,xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,0,v_beta,1.5,,,,v_beta__b1.5,done,11048.00,8788696,0,
screening,xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,0,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,10965.00,8788104,0,
screening,xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,0,v_cap,1.0,,4,,v_cap__b1__c4,done,11022.00,8789080,0,
screening,xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,0,v_cap,1.0,,2,,v_cap__b1__c2,done,10960.00,8787984,0,
screening,xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,0,v_minsup,0.0,,,10,v_minsup__t10,done,10904.00,8788056,0,
screening,xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,0,v_logweight,0.0,,,,v_logweight,done,10645.00,8787588,0,
screening,5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,0,v_beta,1.5,,,,v_beta__b1.5,done,17571.00,10787996,0,
screening,5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,0,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,17467.00,10791372,0,
screening,5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,0,v_cap,1.0,,4,,v_cap__b1__c4,done,16374.00,10795492,0,
screening,5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,0,v_cap,1.0,,2,,v_cap__b1__c2,done,15875.00,10794444,0,
screening,5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0,v_minsup,0.0,,,10,v_minsup__t10,done,16232.00,10787860,0,
screening,5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0,v_logweight,0.0,,,,v_logweight,done,17342.00,10806700,0,
confirm,xiiotid__alpha0.25__seed3__original_protean,xiiotid,0.25,3,original_protean,,,,,original_protean,done,11102.00,10986852,0,
confirm,xiiotid__alpha0.25__seed3__support_weighted__beta0.5,xiiotid,0.25,3,support_weighted,0.5,,,,support_weighted__beta0.5,done,11105.00,10986972,0,
confirm,xiiotid__alpha0.25__seed3__support_weighted__beta1,xiiotid,0.25,3,support_weighted,1,,,,support_weighted__beta1,done,11247.00,10986812,0,
confirm,xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,xiiotid,0.25,3,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,10678.00,10986572,0,
confirm,xiiotid__alpha0.25__seed3__v_cap__b1__c4,xiiotid,0.25,3,v_cap,1.0,,4,,v_cap__b1__c4,done,10621.00,10986660,0,
confirm,xiiotid__alpha0.25__seed4__original_protean,xiiotid,0.25,4,original_protean,,,,,original_protean,done,11200.00,13373900,0,
confirm,xiiotid__alpha0.25__seed4__support_weighted__beta0.5,xiiotid,0.25,4,support_weighted,0.5,,,,support_weighted__beta0.5,done,11147.00,13376792,0,
confirm,xiiotid__alpha0.25__seed4__support_weighted__beta1,xiiotid,0.25,4,support_weighted,1,,,,support_weighted__beta1,done,11529.00,13377304,0,
confirm,xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,xiiotid,0.25,4,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,11448.00,13377596,0,
confirm,xiiotid__alpha0.25__seed4__v_cap__b1__c4,xiiotid,0.25,4,v_cap,1.0,,4,,v_cap__b1__c4,done,10168.00,13377208,0,
confirm,5gnidd__alpha0.5__seed3__original_protean,5gnidd,0.5,3,original_protean,,,,,original_protean,done,15875.00,9482756,0,
confirm,5gnidd__alpha0.5__seed3__support_weighted__beta0.5,5gnidd,0.5,3,support_weighted,0.5,,,,support_weighted__beta0.5,done,16116.00,9507836,0,
confirm,5gnidd__alpha0.5__seed3__support_weighted__beta1,5gnidd,0.5,3,support_weighted,1,,,,support_weighted__beta1,done,16782.00,9480524,0,
confirm,5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,5gnidd,0.5,3,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,16459.00,9501568,0,
confirm,5gnidd__alpha0.5__seed3__v_cap__b1__c4,5gnidd,0.5,3,v_cap,1.0,,4,,v_cap__b1__c4,done,17023.00,9487356,0,
confirm,5gnidd__alpha0.5__seed4__original_protean,5gnidd,0.5,4,original_protean,,,,,original_protean,done,17324.00,11657012,0,
confirm,5gnidd__alpha0.5__seed4__support_weighted__beta0.5,5gnidd,0.5,4,support_weighted,0.5,,,,support_weighted__beta0.5,done,17181.00,11697428,0,
confirm,5gnidd__alpha0.5__seed4__support_weighted__beta1,5gnidd,0.5,4,support_weighted,1,,,,support_weighted__beta1,done,17582.00,11669892,0,
confirm,5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,5gnidd,0.5,4,v_shrink,1.0,0.5,,,v_shrink__b1__lam0.5,done,17234.00,11688404,0,
confirm,5gnidd__alpha0.5__seed4__v_cap__b1__c4,5gnidd,0.5,4,v_cap,1.0,,4,,v_cap__b1__c4,done,16415.00,11679488,0,
1 phase run_id dataset alpha seed mode beta shrink_lambda cap_factor min_support label status wall_s peak_rss_kb exit_code notes
2 screening xiiotid__alpha0.25__seed0__original_protean xiiotid 0.25 0 original_protean original_protean done 10879.00 8788432 0
3 screening 5gnidd__alpha0.5__seed0__original_protean 5gnidd 0.5 0 original_protean original_protean done 16807.00 10800928 0
4 screening xiiotid__alpha0.25__seed0__v_beta__b1.5 xiiotid 0.25 0 v_beta 1.5 v_beta__b1.5 done 11048.00 8788696 0
5 screening xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5 xiiotid 0.25 0 v_shrink 1.0 0.5 v_shrink__b1__lam0.5 done 10965.00 8788104 0
6 screening xiiotid__alpha0.25__seed0__v_cap__b1__c4 xiiotid 0.25 0 v_cap 1.0 4 v_cap__b1__c4 done 11022.00 8789080 0
7 screening xiiotid__alpha0.25__seed0__v_cap__b1__c2 xiiotid 0.25 0 v_cap 1.0 2 v_cap__b1__c2 done 10960.00 8787984 0
8 screening xiiotid__alpha0.25__seed0__v_minsup__t10 xiiotid 0.25 0 v_minsup 0.0 10 v_minsup__t10 done 10904.00 8788056 0
9 screening xiiotid__alpha0.25__seed0__v_logweight xiiotid 0.25 0 v_logweight 0.0 v_logweight done 10645.00 8787588 0
10 screening 5gnidd__alpha0.5__seed0__v_beta__b1.5 5gnidd 0.5 0 v_beta 1.5 v_beta__b1.5 done 17571.00 10787996 0
11 screening 5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5 5gnidd 0.5 0 v_shrink 1.0 0.5 v_shrink__b1__lam0.5 done 17467.00 10791372 0
12 screening 5gnidd__alpha0.5__seed0__v_cap__b1__c4 5gnidd 0.5 0 v_cap 1.0 4 v_cap__b1__c4 done 16374.00 10795492 0
13 screening 5gnidd__alpha0.5__seed0__v_cap__b1__c2 5gnidd 0.5 0 v_cap 1.0 2 v_cap__b1__c2 done 15875.00 10794444 0
14 screening 5gnidd__alpha0.5__seed0__v_minsup__t10 5gnidd 0.5 0 v_minsup 0.0 10 v_minsup__t10 done 16232.00 10787860 0
15 screening 5gnidd__alpha0.5__seed0__v_logweight 5gnidd 0.5 0 v_logweight 0.0 v_logweight done 17342.00 10806700 0
16 confirm xiiotid__alpha0.25__seed3__original_protean xiiotid 0.25 3 original_protean original_protean done 11102.00 10986852 0
17 confirm xiiotid__alpha0.25__seed3__support_weighted__beta0.5 xiiotid 0.25 3 support_weighted 0.5 support_weighted__beta0.5 done 11105.00 10986972 0
18 confirm xiiotid__alpha0.25__seed3__support_weighted__beta1 xiiotid 0.25 3 support_weighted 1 support_weighted__beta1 done 11247.00 10986812 0
19 confirm xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5 xiiotid 0.25 3 v_shrink 1.0 0.5 v_shrink__b1__lam0.5 done 10678.00 10986572 0
20 confirm xiiotid__alpha0.25__seed3__v_cap__b1__c4 xiiotid 0.25 3 v_cap 1.0 4 v_cap__b1__c4 done 10621.00 10986660 0
21 confirm xiiotid__alpha0.25__seed4__original_protean xiiotid 0.25 4 original_protean original_protean done 11200.00 13373900 0
22 confirm xiiotid__alpha0.25__seed4__support_weighted__beta0.5 xiiotid 0.25 4 support_weighted 0.5 support_weighted__beta0.5 done 11147.00 13376792 0
23 confirm xiiotid__alpha0.25__seed4__support_weighted__beta1 xiiotid 0.25 4 support_weighted 1 support_weighted__beta1 done 11529.00 13377304 0
24 confirm xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5 xiiotid 0.25 4 v_shrink 1.0 0.5 v_shrink__b1__lam0.5 done 11448.00 13377596 0
25 confirm xiiotid__alpha0.25__seed4__v_cap__b1__c4 xiiotid 0.25 4 v_cap 1.0 4 v_cap__b1__c4 done 10168.00 13377208 0
26 confirm 5gnidd__alpha0.5__seed3__original_protean 5gnidd 0.5 3 original_protean original_protean done 15875.00 9482756 0
27 confirm 5gnidd__alpha0.5__seed3__support_weighted__beta0.5 5gnidd 0.5 3 support_weighted 0.5 support_weighted__beta0.5 done 16116.00 9507836 0
28 confirm 5gnidd__alpha0.5__seed3__support_weighted__beta1 5gnidd 0.5 3 support_weighted 1 support_weighted__beta1 done 16782.00 9480524 0
29 confirm 5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5 5gnidd 0.5 3 v_shrink 1.0 0.5 v_shrink__b1__lam0.5 done 16459.00 9501568 0
30 confirm 5gnidd__alpha0.5__seed3__v_cap__b1__c4 5gnidd 0.5 3 v_cap 1.0 4 v_cap__b1__c4 done 17023.00 9487356 0
31 confirm 5gnidd__alpha0.5__seed4__original_protean 5gnidd 0.5 4 original_protean original_protean done 17324.00 11657012 0
32 confirm 5gnidd__alpha0.5__seed4__support_weighted__beta0.5 5gnidd 0.5 4 support_weighted 0.5 support_weighted__beta0.5 done 17181.00 11697428 0
33 confirm 5gnidd__alpha0.5__seed4__support_weighted__beta1 5gnidd 0.5 4 support_weighted 1 support_weighted__beta1 done 17582.00 11669892 0
34 confirm 5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5 5gnidd 0.5 4 v_shrink 1.0 0.5 v_shrink__b1__lam0.5 done 17234.00 11688404 0
35 confirm 5gnidd__alpha0.5__seed4__v_cap__b1__c4 5gnidd 0.5 4 v_cap 1.0 4 v_cap__b1__c4 done 16415.00 11679488 0
exploration_v2/analysis/screening_pairing_check.csv
+15
View File
@@ -0,0 +1,15 @@
run_id,dataset,alpha,manifest_content_hash,preprocessing_hash,support_sha256,snapshot0_sha256
5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a,a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346,6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a,dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349,5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
1 run_id dataset alpha manifest_content_hash preprocessing_hash support_sha256 snapshot0_sha256
2 5gnidd__alpha0.5__seed0__original_protean 5gnidd 0.5 0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6 ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346 6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
3 5gnidd__alpha0.5__seed0__v_beta__b1.5 5gnidd 0.5 0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6 ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346 6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
4 5gnidd__alpha0.5__seed0__v_cap__b1__c2 5gnidd 0.5 0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6 ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346 6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
5 5gnidd__alpha0.5__seed0__v_cap__b1__c4 5gnidd 0.5 0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6 ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346 6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
6 5gnidd__alpha0.5__seed0__v_logweight 5gnidd 0.5 0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6 ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346 6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
7 5gnidd__alpha0.5__seed0__v_minsup__t10 5gnidd 0.5 0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6 ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346 6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
8 5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5 5gnidd 0.5 0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6 ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a a11f2d89f27753692268ebea60abca47e8ef2900f622b1d371ddc3817238b346 6143d1f99515b903c5602afcc776613db658b3b2e6af35272f57e858c4e73fef
9 xiiotid__alpha0.25__seed0__original_protean xiiotid 0.25 839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da 8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349 5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
10 xiiotid__alpha0.25__seed0__v_beta__b1.5 xiiotid 0.25 839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da 8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349 5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
11 xiiotid__alpha0.25__seed0__v_cap__b1__c2 xiiotid 0.25 839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da 8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349 5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
12 xiiotid__alpha0.25__seed0__v_cap__b1__c4 xiiotid 0.25 839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da 8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349 5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
13 xiiotid__alpha0.25__seed0__v_logweight xiiotid 0.25 839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da 8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349 5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
14 xiiotid__alpha0.25__seed0__v_minsup__t10 xiiotid 0.25 839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da 8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349 5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
15 xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5 xiiotid 0.25 839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da 8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a dcd8864b78dc46d2f727be6596903c6944d0e9acd6cce994b61f0689339c3349 5aaa79e7c7266b9ea0331b890bddac5794033f452a18971c192c627e6f4c1f00
exploration_v2/analysis/screening_ranking.csv
+7
View File
@@ -0,0 +1,7 @@
rank,rank_secondary,label,mode,beta,shrink_lambda,cap_factor,min_support,score_mean_delta_val_global,min_condition_delta,secondary_mean_delta_val_per_client,S1_val_global,S1_baseline,S1_delta,S1_val_per_client,S1_delta_per_client,S2_val_global,S2_baseline,S2_delta,S2_val_per_client,S2_delta_per_client
1,1,v_shrink__b1__lam0.5,v_shrink,1.0,0.5,,,0.02666418343657906,0.01418010949105697,0.02904392091826946,0.6771870761605789,0.6380388187784778,0.03914825738210115,0.6394200514807982,0.0314475437388545,0.8757058802515706,0.8615257707605136,0.01418010949105697,0.7519195999208417,0.026640298097684423
2,2,v_cap__b1__c4,v_cap,1.0,,4.0,,0.006386658832843839,0.0034615578813887993,0.014884565739555988,0.6415003766598666,0.6380388187784778,0.0034615578813887993,0.6043797897480034,-0.003592717993940231,0.8708375305448125,0.8615257707605136,0.009311759784298879,0.7586411512962095,0.03336184947305221
3,3,v_cap__b1__c2,v_cap,1.0,,2.0,,-0.0024310752295784166,-0.0055917994582352915,-0.013098515805364142,0.6324470193202425,0.6380388187784778,-0.0055917994582352915,0.5746261281359909,-0.03334637960595277,0.8622554197595921,0.8615257707605136,0.0007296489990784583,0.7324286498183817,0.0071493479952244865
4,5,v_logweight,v_logweight,0.0,,,,-0.013930127799255287,-0.034591638342140296,-0.030092848616604162,0.6447702015221075,0.6380388187784778,0.006731382743629721,0.6126459310493659,0.004673423307422264,0.8269341324183733,0.8615257707605136,-0.034591638342140296,0.6604201812825267,-0.06485912054063059
5,6,v_minsup__t10,v_minsup,0.0,,,10.0,-0.03017797764203456,-0.04375397523698199,-0.05005934390012601,0.6214368387313907,0.6380388187784778,-0.016601980047087128,0.581752384882787,-0.026220122859156647,0.8177717955235316,0.8615257707605136,-0.04375397523698199,0.6513807368820619,-0.07389856494109537
6,4,v_beta__b1.5,v_beta,1.5,,,,-0.04850986532623136,-0.09180491842510796,-0.026127080317370543,0.5462339003533698,0.6380388187784778,-0.09180491842510796,0.554598371814126,-0.0533741359278177,0.8563109585331589,0.8615257707605136,-0.005214812227354759,0.7263992771162339,0.0011199752930766138
1 rank rank_secondary label mode beta shrink_lambda cap_factor min_support score_mean_delta_val_global min_condition_delta secondary_mean_delta_val_per_client S1_val_global S1_baseline S1_delta S1_val_per_client S1_delta_per_client S2_val_global S2_baseline S2_delta S2_val_per_client S2_delta_per_client
2 1 1 v_shrink__b1__lam0.5 v_shrink 1.0 0.5 0.02666418343657906 0.01418010949105697 0.02904392091826946 0.6771870761605789 0.6380388187784778 0.03914825738210115 0.6394200514807982 0.0314475437388545 0.8757058802515706 0.8615257707605136 0.01418010949105697 0.7519195999208417 0.026640298097684423
3 2 2 v_cap__b1__c4 v_cap 1.0 4.0 0.006386658832843839 0.0034615578813887993 0.014884565739555988 0.6415003766598666 0.6380388187784778 0.0034615578813887993 0.6043797897480034 -0.003592717993940231 0.8708375305448125 0.8615257707605136 0.009311759784298879 0.7586411512962095 0.03336184947305221
4 3 3 v_cap__b1__c2 v_cap 1.0 2.0 -0.0024310752295784166 -0.0055917994582352915 -0.013098515805364142 0.6324470193202425 0.6380388187784778 -0.0055917994582352915 0.5746261281359909 -0.03334637960595277 0.8622554197595921 0.8615257707605136 0.0007296489990784583 0.7324286498183817 0.0071493479952244865
5 4 5 v_logweight v_logweight 0.0 -0.013930127799255287 -0.034591638342140296 -0.030092848616604162 0.6447702015221075 0.6380388187784778 0.006731382743629721 0.6126459310493659 0.004673423307422264 0.8269341324183733 0.8615257707605136 -0.034591638342140296 0.6604201812825267 -0.06485912054063059
6 5 6 v_minsup__t10 v_minsup 0.0 10.0 -0.03017797764203456 -0.04375397523698199 -0.05005934390012601 0.6214368387313907 0.6380388187784778 -0.016601980047087128 0.581752384882787 -0.026220122859156647 0.8177717955235316 0.8615257707605136 -0.04375397523698199 0.6513807368820619 -0.07389856494109537
7 6 4 v_beta__b1.5 v_beta 1.5 -0.04850986532623136 -0.09180491842510796 -0.026127080317370543 0.5462339003533698 0.6380388187784778 -0.09180491842510796 0.554598371814126 -0.0533741359278177 0.8563109585331589 0.8615257707605136 -0.005214812227354759 0.7263992771162339 0.0011199752930766138
exploration_v2/analysis/screening_references.csv
+3
View File
@@ -0,0 +1,3 @@
condition,baseline_run,baseline_val_global,best_beta_by_val_global,best_val_global,best_delta_global,best_beta_by_val_per_client,best_val_per_client,best_delta_per_client
S1,xiiotid__alpha0.25__seed0__original_protean,0.6380388187784778,0.75,0.6690666806848056,0.03102786190632778,0.75,0.6564371328240081,0.048464625082064394
S2,5gnidd__alpha0.5__seed0__original_protean,0.8615257707605136,1.0,0.8762664919755059,0.014740721214992258,0.75,0.7719208198370514,0.046641518013894157
1 condition baseline_run baseline_val_global best_beta_by_val_global best_val_global best_delta_global best_beta_by_val_per_client best_val_per_client best_delta_per_client
2 S1 xiiotid__alpha0.25__seed0__original_protean 0.6380388187784778 0.75 0.6690666806848056 0.03102786190632778 0.75 0.6564371328240081 0.048464625082064394
3 S2 5gnidd__alpha0.5__seed0__original_protean 0.8615257707605136 1.0 0.8762664919755059 0.014740721214992258 0.75 0.7719208198370514 0.046641518013894157
exploration_v2/analysis/walltime.csv
+35
View File
@@ -0,0 +1,35 @@
phase,run_id,status,wall_s,wall_h,peak_rss_gib
screening,xiiotid__alpha0.25__seed0__original_protean,done,10879.00,3.022,8.381
screening,5gnidd__alpha0.5__seed0__original_protean,done,16807.00,4.669,10.301
screening,xiiotid__alpha0.25__seed0__v_beta__b1.5,done,11048.00,3.069,8.382
screening,xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,done,10965.00,3.046,8.381
screening,xiiotid__alpha0.25__seed0__v_cap__b1__c4,done,11022.00,3.062,8.382
screening,xiiotid__alpha0.25__seed0__v_cap__b1__c2,done,10960.00,3.044,8.381
screening,xiiotid__alpha0.25__seed0__v_minsup__t10,done,10904.00,3.029,8.381
screening,xiiotid__alpha0.25__seed0__v_logweight,done,10645.00,2.957,8.38
screening,5gnidd__alpha0.5__seed0__v_beta__b1.5,done,17571.00,4.881,10.288
screening,5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,done,17467.00,4.852,10.291
screening,5gnidd__alpha0.5__seed0__v_cap__b1__c4,done,16374.00,4.548,10.295
screening,5gnidd__alpha0.5__seed0__v_cap__b1__c2,done,15875.00,4.41,10.294
screening,5gnidd__alpha0.5__seed0__v_minsup__t10,done,16232.00,4.509,10.288
screening,5gnidd__alpha0.5__seed0__v_logweight,done,17342.00,4.817,10.306
confirm,xiiotid__alpha0.25__seed3__original_protean,done,11102.00,3.084,10.478
confirm,xiiotid__alpha0.25__seed3__support_weighted__beta0.5,done,11105.00,3.085,10.478
confirm,xiiotid__alpha0.25__seed3__support_weighted__beta1,done,11247.00,3.124,10.478
confirm,xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,done,10678.00,2.966,10.478
confirm,xiiotid__alpha0.25__seed3__v_cap__b1__c4,done,10621.00,2.95,10.478
confirm,xiiotid__alpha0.25__seed4__original_protean,done,11200.00,3.111,12.754
confirm,xiiotid__alpha0.25__seed4__support_weighted__beta0.5,done,11147.00,3.096,12.757
confirm,xiiotid__alpha0.25__seed4__support_weighted__beta1,done,11529.00,3.203,12.758
confirm,xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,done,11448.00,3.18,12.758
confirm,xiiotid__alpha0.25__seed4__v_cap__b1__c4,done,10168.00,2.824,12.757
confirm,5gnidd__alpha0.5__seed3__original_protean,done,15875.00,4.41,9.043
confirm,5gnidd__alpha0.5__seed3__support_weighted__beta0.5,done,16116.00,4.477,9.067
confirm,5gnidd__alpha0.5__seed3__support_weighted__beta1,done,16782.00,4.662,9.041
confirm,5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,done,16459.00,4.572,9.061
confirm,5gnidd__alpha0.5__seed3__v_cap__b1__c4,done,17023.00,4.729,9.048
confirm,5gnidd__alpha0.5__seed4__original_protean,done,17324.00,4.812,11.117
confirm,5gnidd__alpha0.5__seed4__support_weighted__beta0.5,done,17181.00,4.772,11.156
confirm,5gnidd__alpha0.5__seed4__support_weighted__beta1,done,17582.00,4.884,11.129
confirm,5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,done,17234.00,4.787,11.147
confirm,5gnidd__alpha0.5__seed4__v_cap__b1__c4,done,16415.00,4.56,11.138
1 phase run_id status wall_s wall_h peak_rss_gib
2 screening xiiotid__alpha0.25__seed0__original_protean done 10879.00 3.022 8.381
3 screening 5gnidd__alpha0.5__seed0__original_protean done 16807.00 4.669 10.301
4 screening xiiotid__alpha0.25__seed0__v_beta__b1.5 done 11048.00 3.069 8.382
5 screening xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5 done 10965.00 3.046 8.381
6 screening xiiotid__alpha0.25__seed0__v_cap__b1__c4 done 11022.00 3.062 8.382
7 screening xiiotid__alpha0.25__seed0__v_cap__b1__c2 done 10960.00 3.044 8.381
8 screening xiiotid__alpha0.25__seed0__v_minsup__t10 done 10904.00 3.029 8.381
9 screening xiiotid__alpha0.25__seed0__v_logweight done 10645.00 2.957 8.38
10 screening 5gnidd__alpha0.5__seed0__v_beta__b1.5 done 17571.00 4.881 10.288
11 screening 5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5 done 17467.00 4.852 10.291
12 screening 5gnidd__alpha0.5__seed0__v_cap__b1__c4 done 16374.00 4.548 10.295
13 screening 5gnidd__alpha0.5__seed0__v_cap__b1__c2 done 15875.00 4.41 10.294
14 screening 5gnidd__alpha0.5__seed0__v_minsup__t10 done 16232.00 4.509 10.288
15 screening 5gnidd__alpha0.5__seed0__v_logweight done 17342.00 4.817 10.306
16 confirm xiiotid__alpha0.25__seed3__original_protean done 11102.00 3.084 10.478
17 confirm xiiotid__alpha0.25__seed3__support_weighted__beta0.5 done 11105.00 3.085 10.478
18 confirm xiiotid__alpha0.25__seed3__support_weighted__beta1 done 11247.00 3.124 10.478
19 confirm xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5 done 10678.00 2.966 10.478
20 confirm xiiotid__alpha0.25__seed3__v_cap__b1__c4 done 10621.00 2.95 10.478
21 confirm xiiotid__alpha0.25__seed4__original_protean done 11200.00 3.111 12.754
22 confirm xiiotid__alpha0.25__seed4__support_weighted__beta0.5 done 11147.00 3.096 12.757
23 confirm xiiotid__alpha0.25__seed4__support_weighted__beta1 done 11529.00 3.203 12.758
24 confirm xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5 done 11448.00 3.18 12.758
25 confirm xiiotid__alpha0.25__seed4__v_cap__b1__c4 done 10168.00 2.824 12.757
26 confirm 5gnidd__alpha0.5__seed3__original_protean done 15875.00 4.41 9.043
27 confirm 5gnidd__alpha0.5__seed3__support_weighted__beta0.5 done 16116.00 4.477 9.067
28 confirm 5gnidd__alpha0.5__seed3__support_weighted__beta1 done 16782.00 4.662 9.041
29 confirm 5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5 done 16459.00 4.572 9.061
30 confirm 5gnidd__alpha0.5__seed3__v_cap__b1__c4 done 17023.00 4.729 9.048
31 confirm 5gnidd__alpha0.5__seed4__original_protean done 17324.00 4.812 11.117
32 confirm 5gnidd__alpha0.5__seed4__support_weighted__beta0.5 done 17181.00 4.772 11.156
33 confirm 5gnidd__alpha0.5__seed4__support_weighted__beta1 done 17582.00 4.884 11.129
34 confirm 5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5 done 17234.00 4.787 11.147
35 confirm 5gnidd__alpha0.5__seed4__v_cap__b1__c4 done 16415.00 4.56 11.138
exploration_v2/driver.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# exploration_v2 driver entry point.
#
# Holds an exclusive lock so two drivers can never run at once. The actual
# scheduler and heartbeat live in scripts/pool_driver.py. This script is safe
# to re-invoke; resume.sh does exactly that.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/.." && pwd)"
PY="$REPO_ROOT/.venv/bin/python"
export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1
export VECLIB_MAXIMUM_THREADS=1
export PYTHONDONTWRITEBYTECODE=1
export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=/run/user/$(id -u)/bus}"
mkdir -p "$HERE/logs"
exec 9>"$HERE/driver.lock"
if ! flock -n 9; then
echo "[exploration_v2] another driver already holds driver.lock; exiting" >&2
exit 0
fi
cd "$REPO_ROOT" || exit 3
exec "$PY" "$HERE/scripts/pool_driver.py" "$@"
exploration_v2/partitions/5gnidd__alpha0.5__seed3.json
+154
View File
@@ -0,0 +1,154 @@
{
"algorithm": "per_class_dirichlet_multinomial",
"alpha": 0.5,
"benign_class": "Benign",
"class_names": [
"Benign",
"HTTPFlood",
"ICMPFlood",
"SYNFlood",
"SYNScan",
"SlowrateDoS",
"TCPConnectScan",
"UDPFlood",
"UDPScan"
],
"config_hash": null,
"created_utc": "2026-10-30T04:32:10Z",
"dataset": "5gnidd",
"dataset_name": "5G-NIDD",
"manifest_version": "1",
"num_classes": 9,
"num_clients": 10,
"num_train": 851123,
"partition_hash": "0739f15c51fe4fa2250f717e8f52548706cb7093581447cd43a4b2364f6ec058",
"preprocessing_hash": "60b5682b39184fd7d24e117430a00c160c5c7e95d1bc1fcaa99745108f514d81",
"python": "3.12.3",
"rng_seed": 2052809991,
"seed": 3,
"split": {
"hash": "178e69cfacc40a1651246d0c33a26d16f76c8e19254d033c8c04cbbfbb86246a",
"seed": 3,
"sizes": {
"test": 243177,
"train": 851123,
"validation": 121589
},
"strategy": "stratified_by_class",
"test_fraction": 0.2,
"train_fraction": 0.7,
"validation_fraction": 0.1
},
"support": [
[
12367,
24330,
109,
732,
1299,
2036,
261,
107583,
129
],
[
86013,
9922,
8,
104,
31,
3166,
1928,
6739,
5
],
[
112226,
6136,
97,
152,
143,
28,
966,
13880,
301
],
[
7322,
29013,
115,
213,
9592,
159,
373,
2354,
503
],
[
28752,
497,
15,
1688,
71,
19630,
0,
607,
67
],
[
11126,
366,
37,
786,
519,
1272,
140,
84223,
1613
],
[
7958,
249,
40,
912,
93,
801,
3851,
45015,
8048
],
[
64584,
25,
1,
77,
2281,
20803,
7,
12615,
17
],
[
2496,
14929,
303,
1582,
1,
2192,
2455,
189,
234
],
[
1571,
13102,
83,
559,
0,
1100,
4056,
46933,
217
]
]
}
exploration_v2/partitions/5gnidd__alpha0.5__seed4.json
+154
View File
@@ -0,0 +1,154 @@
{
"algorithm": "per_class_dirichlet_multinomial",
"alpha": 0.5,
"benign_class": "Benign",
"class_names": [
"Benign",
"HTTPFlood",
"ICMPFlood",
"SYNFlood",
"SYNScan",
"SlowrateDoS",
"TCPConnectScan",
"UDPFlood",
"UDPScan"
],
"config_hash": null,
"created_utc": "2026-10-30T04:32:41Z",
"dataset": "5gnidd",
"dataset_name": "5G-NIDD",
"manifest_version": "1",
"num_classes": 9,
"num_clients": 10,
"num_train": 851123,
"partition_hash": "21a507d9d94f31154c5fb0c6833344738d1a137c78950f5284dec26d727e983a",
"preprocessing_hash": "ffb7b2a1adb6cf8a292ea6e88dca574b44446da2b3ae18f690ae6fc59699e23d",
"python": "3.12.3",
"rng_seed": 1792011051,
"seed": 4,
"split": {
"hash": "0b80242a3ac045217b73a89e5a7d03cb61abacefcc050e11b93ac83c831479ce",
"seed": 4,
"sizes": {
"test": 243177,
"train": 851123,
"validation": 121589
},
"strategy": "stratified_by_class",
"test_fraction": 0.2,
"train_fraction": 0.7,
"validation_fraction": 0.1
},
"support": [
[
159078,
11071,
39,
936,
8239,
1050,
33,
15599,
241
],
[
15658,
27988,
318,
7,
674,
7,
5229,
6152,
93
],
[
5874,
2211,
0,
2113,
893,
1049,
490,
13122,
131
],
[
76962,
851,
10,
26,
1223,
7546,
846,
57683,
3055
],
[
3588,
47186,
243,
26,
930,
11733,
1121,
22035,
1898
],
[
26897,
235,
60,
2927,
181,
263,
2914,
58429,
2934
],
[
143,
5129,
112,
237,
45,
5427,
36,
7654,
1444
],
[
30083,
3124,
10,
8,
79,
357,
1910,
111407,
0
],
[
405,
4,
5,
413,
1110,
1483,
1,
14987,
1323
],
[
15727,
770,
11,
112,
656,
22272,
1457,
13070,
15
]
]
}
exploration_v2/partitions/xiiotid__alpha0.25__seed3.json
+154
View File
@@ -0,0 +1,154 @@
{
"algorithm": "per_class_dirichlet_multinomial",
"alpha": 0.25,
"benign_class": "Normal",
"class_names": [
"C&C",
"Exfiltration",
"Exploitation",
"Lateral_movement",
"Normal",
"RDOS",
"Reconnaissance",
"Tampering",
"Weaponization"
],
"config_hash": null,
"created_utc": "2026-10-30T04:00:15Z",
"dataset": "xiiotid",
"dataset_name": "X-IIoTID",
"manifest_version": "1",
"num_classes": 9,
"num_clients": 10,
"num_train": 569909,
"partition_hash": "85a8625b81d3624dec9e5322979e2f1547f330c60d5e822e32c12177195ed602",
"preprocessing_hash": "e22db95e5109bfa02701d8ce450453a7c8f7e408c94e81dfe2b95ef4ab2a441f",
"python": "3.12.3",
"rng_seed": 1031130567,
"seed": 3,
"split": {
"hash": "8e66d08e71225567f8da977e5b927c476882c1d1ac768c1b490cdbc8de2945e9",
"seed": 3,
"sizes": {
"test": 162831,
"train": 569909,
"validation": 81414
},
"strategy": "stratified_by_class",
"test_fraction": 0.2,
"train_fraction": 0.7,
"validation_fraction": 0.1
},
"support": [
[
21,
1003,
0,
7,
2950,
14,
11735,
0,
342
],
[
27,
1029,
7,
199,
549,
1426,
8567,
370,
1278
],
[
1191,
483,
23,
5147,
17775,
234,
139,
0,
31
],
[
0,
2,
36,
0,
3006,
58,
12,
12,
1420
],
[
15,
18,
39,
2326,
2692,
58544,
773,
0,
25866
],
[
320,
4958,
0,
31,
966,
18218,
3006,
949,
8009
],
[
0,
2705,
215,
167,
1157,
1247,
13942,
2,
347
],
[
165,
35,
473,
13685,
111159,
19011,
15662,
2013,
120
],
[
4,
228,
0,
2,
150706,
122,
35323,
2,
83
],
[
261,
5033,
0,
553,
1817,
9,
2,
238,
7598
]
]
}
exploration_v2/partitions/xiiotid__alpha0.25__seed4.json
+154
View File
@@ -0,0 +1,154 @@
{
"algorithm": "per_class_dirichlet_multinomial",
"alpha": 0.25,
"benign_class": "Normal",
"class_names": [
"C&C",
"Exfiltration",
"Exploitation",
"Lateral_movement",
"Normal",
"RDOS",
"Reconnaissance",
"Tampering",
"Weaponization"
],
"config_hash": null,
"created_utc": "2026-10-30T04:31:39Z",
"dataset": "xiiotid",
"dataset_name": "X-IIoTID",
"manifest_version": "1",
"num_classes": 9,
"num_clients": 10,
"num_train": 569909,
"partition_hash": "ff47f504d5b604935333ccbfddd060c18d44d672104e5455f86dcefe48315887",
"preprocessing_hash": "4837f4ab5939fe7bf7f83168d915e67d570086d8534a521a9a9827f10aa41086",
"python": "3.12.3",
"rng_seed": 1148101028,
"seed": 4,
"split": {
"hash": "5bc9100e5cbb4dfccf92e1b0fe64da544a2966c81d82dfab2efa2367173ca807",
"seed": 4,
"sizes": {
"test": 162831,
"train": 569909,
"validation": 81414
},
"strategy": "stratified_by_class",
"test_fraction": 0.2,
"train_fraction": 0.7,
"validation_fraction": 0.1
},
"support": [
[
1,
109,
667,
156,
193448,
29952,
1314,
3,
9802
],
[
20,
368,
4,
1852,
13384,
262,
1102,
42,
4
],
[
58,
336,
0,
1331,
0,
218,
32287,
5,
27
],
[
0,
7839,
10,
11,
22325,
619,
12186,
2959,
7624
],
[
413,
94,
4,
9367,
13997,
0,
1924,
6,
47
],
[
238,
5,
3,
0,
1,
24028,
22974,
62,
5
],
[
4,
1,
50,
121,
167,
36163,
14248,
75,
18287
],
[
1110,
508,
47,
6770,
61,
2656,
25,
88,
147
],
[
78,
328,
0,
1112,
42050,
24,
132,
46,
10
],
[
82,
5906,
8,
1397,
7344,
4961,
2969,
300,
9141
]
]
}
exploration_v2/queue_confirm.csv
+21
View File
@@ -0,0 +1,21 @@
run_id,dataset,alpha,seed,mode,beta,shrink,cap,minsup,part_dir,label,role
xiiotid__alpha0.25__seed3__original_protean,xiiotid,0.25,3,original_protean,,,,,exploration_v2/partitions,original_protean,reference
xiiotid__alpha0.25__seed3__support_weighted__beta0.5,xiiotid,0.25,3,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference
xiiotid__alpha0.25__seed3__support_weighted__beta1,xiiotid,0.25,3,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference
xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,xiiotid,0.25,3,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant
xiiotid__alpha0.25__seed3__v_cap__b1__c4,xiiotid,0.25,3,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant
xiiotid__alpha0.25__seed4__original_protean,xiiotid,0.25,4,original_protean,,,,,exploration_v2/partitions,original_protean,reference
xiiotid__alpha0.25__seed4__support_weighted__beta0.5,xiiotid,0.25,4,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference
xiiotid__alpha0.25__seed4__support_weighted__beta1,xiiotid,0.25,4,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference
xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,xiiotid,0.25,4,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant
xiiotid__alpha0.25__seed4__v_cap__b1__c4,xiiotid,0.25,4,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant
5gnidd__alpha0.5__seed3__original_protean,5gnidd,0.5,3,original_protean,,,,,exploration_v2/partitions,original_protean,reference
5gnidd__alpha0.5__seed3__support_weighted__beta0.5,5gnidd,0.5,3,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference
5gnidd__alpha0.5__seed3__support_weighted__beta1,5gnidd,0.5,3,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference
5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,5gnidd,0.5,3,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant
5gnidd__alpha0.5__seed3__v_cap__b1__c4,5gnidd,0.5,3,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant
5gnidd__alpha0.5__seed4__original_protean,5gnidd,0.5,4,original_protean,,,,,exploration_v2/partitions,original_protean,reference
5gnidd__alpha0.5__seed4__support_weighted__beta0.5,5gnidd,0.5,4,support_weighted,0.5,,,,exploration_v2/partitions,support_weighted__beta0.5,reference
5gnidd__alpha0.5__seed4__support_weighted__beta1,5gnidd,0.5,4,support_weighted,1,,,,exploration_v2/partitions,support_weighted__beta1,reference
5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,5gnidd,0.5,4,v_shrink,1.0,0.5,,,exploration_v2/partitions,v_shrink__b1__lam0.5,confirm_variant
5gnidd__alpha0.5__seed4__v_cap__b1__c4,5gnidd,0.5,4,v_cap,1.0,,4,,exploration_v2/partitions,v_cap__b1__c4,confirm_variant
1 run_id dataset alpha seed mode beta shrink cap minsup part_dir label role
2 xiiotid__alpha0.25__seed3__original_protean xiiotid 0.25 3 original_protean exploration_v2/partitions original_protean reference
3 xiiotid__alpha0.25__seed3__support_weighted__beta0.5 xiiotid 0.25 3 support_weighted 0.5 exploration_v2/partitions support_weighted__beta0.5 reference
4 xiiotid__alpha0.25__seed3__support_weighted__beta1 xiiotid 0.25 3 support_weighted 1 exploration_v2/partitions support_weighted__beta1 reference
5 xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5 xiiotid 0.25 3 v_shrink 1.0 0.5 exploration_v2/partitions v_shrink__b1__lam0.5 confirm_variant
6 xiiotid__alpha0.25__seed3__v_cap__b1__c4 xiiotid 0.25 3 v_cap 1.0 4 exploration_v2/partitions v_cap__b1__c4 confirm_variant
7 xiiotid__alpha0.25__seed4__original_protean xiiotid 0.25 4 original_protean exploration_v2/partitions original_protean reference
8 xiiotid__alpha0.25__seed4__support_weighted__beta0.5 xiiotid 0.25 4 support_weighted 0.5 exploration_v2/partitions support_weighted__beta0.5 reference
9 xiiotid__alpha0.25__seed4__support_weighted__beta1 xiiotid 0.25 4 support_weighted 1 exploration_v2/partitions support_weighted__beta1 reference
10 xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5 xiiotid 0.25 4 v_shrink 1.0 0.5 exploration_v2/partitions v_shrink__b1__lam0.5 confirm_variant
11 xiiotid__alpha0.25__seed4__v_cap__b1__c4 xiiotid 0.25 4 v_cap 1.0 4 exploration_v2/partitions v_cap__b1__c4 confirm_variant
12 5gnidd__alpha0.5__seed3__original_protean 5gnidd 0.5 3 original_protean exploration_v2/partitions original_protean reference
13 5gnidd__alpha0.5__seed3__support_weighted__beta0.5 5gnidd 0.5 3 support_weighted 0.5 exploration_v2/partitions support_weighted__beta0.5 reference
14 5gnidd__alpha0.5__seed3__support_weighted__beta1 5gnidd 0.5 3 support_weighted 1 exploration_v2/partitions support_weighted__beta1 reference
15 5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5 5gnidd 0.5 3 v_shrink 1.0 0.5 exploration_v2/partitions v_shrink__b1__lam0.5 confirm_variant
16 5gnidd__alpha0.5__seed3__v_cap__b1__c4 5gnidd 0.5 3 v_cap 1.0 4 exploration_v2/partitions v_cap__b1__c4 confirm_variant
17 5gnidd__alpha0.5__seed4__original_protean 5gnidd 0.5 4 original_protean exploration_v2/partitions original_protean reference
18 5gnidd__alpha0.5__seed4__support_weighted__beta0.5 5gnidd 0.5 4 support_weighted 0.5 exploration_v2/partitions support_weighted__beta0.5 reference
19 5gnidd__alpha0.5__seed4__support_weighted__beta1 5gnidd 0.5 4 support_weighted 1 exploration_v2/partitions support_weighted__beta1 reference
20 5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5 5gnidd 0.5 4 v_shrink 1.0 0.5 exploration_v2/partitions v_shrink__b1__lam0.5 confirm_variant
21 5gnidd__alpha0.5__seed4__v_cap__b1__c4 5gnidd 0.5 4 v_cap 1.0 4 exploration_v2/partitions v_cap__b1__c4 confirm_variant
exploration_v2/queue_screen.csv
+15
View File
@@ -0,0 +1,15 @@
run_id,dataset,alpha,seed,mode,beta,shrink,cap,minsup,part_dir,label
xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,0,original_protean,,,,,artifacts/partitions,original_protean
5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0,original_protean,,,,,artifacts/partitions,original_protean
xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,0,v_beta,1.5,,,,artifacts/partitions,v_beta__b1.5
xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,0,v_shrink,1.0,0.5,,,artifacts/partitions,v_shrink__b1__lam0.5
xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,0,v_cap,1.0,,4,,artifacts/partitions,v_cap__b1__c4
xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,0,v_cap,1.0,,2,,artifacts/partitions,v_cap__b1__c2
xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,0,v_minsup,0.0,,,10,artifacts/partitions,v_minsup__t10
xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,0,v_logweight,0.0,,,,artifacts/partitions,v_logweight
5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,0,v_beta,1.5,,,,artifacts/partitions,v_beta__b1.5
5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,0,v_shrink,1.0,0.5,,,artifacts/partitions,v_shrink__b1__lam0.5
5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,0,v_cap,1.0,,4,,artifacts/partitions,v_cap__b1__c4
5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,0,v_cap,1.0,,2,,artifacts/partitions,v_cap__b1__c2
5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0,v_minsup,0.0,,,10,artifacts/partitions,v_minsup__t10
5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0,v_logweight,0.0,,,,artifacts/partitions,v_logweight
1 run_id dataset alpha seed mode beta shrink cap minsup part_dir label
2 xiiotid__alpha0.25__seed0__original_protean xiiotid 0.25 0 original_protean artifacts/partitions original_protean
3 5gnidd__alpha0.5__seed0__original_protean 5gnidd 0.5 0 original_protean artifacts/partitions original_protean
4 xiiotid__alpha0.25__seed0__v_beta__b1.5 xiiotid 0.25 0 v_beta 1.5 artifacts/partitions v_beta__b1.5
5 xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5 xiiotid 0.25 0 v_shrink 1.0 0.5 artifacts/partitions v_shrink__b1__lam0.5
6 xiiotid__alpha0.25__seed0__v_cap__b1__c4 xiiotid 0.25 0 v_cap 1.0 4 artifacts/partitions v_cap__b1__c4
7 xiiotid__alpha0.25__seed0__v_cap__b1__c2 xiiotid 0.25 0 v_cap 1.0 2 artifacts/partitions v_cap__b1__c2
8 xiiotid__alpha0.25__seed0__v_minsup__t10 xiiotid 0.25 0 v_minsup 0.0 10 artifacts/partitions v_minsup__t10
9 xiiotid__alpha0.25__seed0__v_logweight xiiotid 0.25 0 v_logweight 0.0 artifacts/partitions v_logweight
10 5gnidd__alpha0.5__seed0__v_beta__b1.5 5gnidd 0.5 0 v_beta 1.5 artifacts/partitions v_beta__b1.5
11 5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5 5gnidd 0.5 0 v_shrink 1.0 0.5 artifacts/partitions v_shrink__b1__lam0.5
12 5gnidd__alpha0.5__seed0__v_cap__b1__c4 5gnidd 0.5 0 v_cap 1.0 4 artifacts/partitions v_cap__b1__c4
13 5gnidd__alpha0.5__seed0__v_cap__b1__c2 5gnidd 0.5 0 v_cap 1.0 2 artifacts/partitions v_cap__b1__c2
14 5gnidd__alpha0.5__seed0__v_minsup__t10 5gnidd 0.5 0 v_minsup 0.0 10 artifacts/partitions v_minsup__t10
15 5gnidd__alpha0.5__seed0__v_logweight 5gnidd 0.5 0 v_logweight 0.0 artifacts/partitions v_logweight
exploration_v2/ranking.json
+513
View File
@@ -0,0 +1,513 @@
{
"generated_utc": "2026-10-31T05:03:55Z",
"stage": "screening",
"ranking_metric": "mean over S1,S2 of val_global_rare_f1(candidate) - val_global_rare_f1(original)",
"secondary_metric": "same with val_rare_class_macro_f1_per_client_mean",
"conditions": {
"S1": {
"dataset": "xiiotid",
"alpha": 0.25
},
"S2": {
"dataset": "5gnidd",
"alpha": 0.5
}
},
"screening_seed": 0,
"baseline_source": {
"S1": "in_environment_seed0",
"S2": "in_environment_seed0"
},
"original": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__original_protean",
"source": "in_environment_seed0",
"val_global_rare_f1": 0.6380388187784778,
"val_global_rare_recall": 0.840600287361644,
"val_global_macro_f1": 0.6716435746566032,
"val_per_client_rare_f1": 0.6079725077419437,
"val_sel_file_present": true
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__original_protean",
"source": "in_environment_seed0",
"val_global_rare_f1": 0.8615257707605136,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8292070866942571,
"val_per_client_rare_f1": 0.7252793018231573,
"val_sel_file_present": true
}
},
"current_method_reference": {
"S1": {
"original": {
"run_id": "xiiotid__alpha0.25__seed0__original_protean",
"source": "in_environment_seed0",
"val_global_rare_f1": 0.6380388187784778,
"val_global_rare_recall": 0.840600287361644,
"val_global_macro_f1": 0.6716435746566032,
"val_per_client_rare_f1": 0.6079725077419437,
"val_sel_file_present": true
},
"best_beta_by_val_global": {
"beta": 0.75,
"val_global_rare_f1": 0.6690666806848056,
"delta_vs_original": 0.03102786190632778
},
"best_beta_by_val_per_client": {
"beta": 0.75,
"val_per_client_rare_f1": 0.6564371328240081,
"delta_vs_original": 0.048464625082064394
},
"all_betas": {
"0": {
"run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.643275274917486,
"val_global_rare_recall": 0.8414156040624469,
"val_global_macro_f1": 0.6762678752194305,
"val_per_client_rare_f1": 0.6151143983374613,
"val_sel_file_present": true
},
"0.25": {
"run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0.25",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.6474224331223045,
"val_global_rare_recall": 0.8369281236929922,
"val_global_macro_f1": 0.6800358336787912,
"val_per_client_rare_f1": 0.612295413922644,
"val_sel_file_present": true
},
"0.5": {
"run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0.5",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.6328331530314106,
"val_global_rare_recall": 0.8055422411499216,
"val_global_macro_f1": 0.6670453405769549,
"val_per_client_rare_f1": 0.5909962766172367,
"val_sel_file_present": true
},
"0.75": {
"run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta0.75",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.6690666806848056,
"val_global_rare_recall": 0.8048039361797941,
"val_global_macro_f1": 0.699686253411971,
"val_per_client_rare_f1": 0.6564371328240081,
"val_sel_file_present": true
},
"1": {
"run_id": "xiiotid__alpha0.25__seed0__support_weighted__beta1",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.5937608319568202,
"val_global_rare_recall": 0.7156752311559107,
"val_global_macro_f1": 0.6331165546843429,
"val_per_client_rare_f1": 0.5797652799748645,
"val_sel_file_present": true
}
}
},
"S2": {
"original": {
"run_id": "5gnidd__alpha0.5__seed0__original_protean",
"source": "in_environment_seed0",
"val_global_rare_f1": 0.8615257707605136,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8292070866942571,
"val_per_client_rare_f1": 0.7252793018231573,
"val_sel_file_present": true
},
"best_beta_by_val_global": {
"beta": 1.0,
"val_global_rare_f1": 0.8762664919755059,
"delta_vs_original": 0.014740721214992258
},
"best_beta_by_val_per_client": {
"beta": 0.75,
"val_per_client_rare_f1": 0.7719208198370514,
"delta_vs_original": 0.046641518013894157
},
"all_betas": {
"0": {
"run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.8676517373672202,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8354514554793465,
"val_per_client_rare_f1": 0.7394716750551001,
"val_sel_file_present": true
},
"0.25": {
"run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0.25",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.8262070288650597,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.799036115730399,
"val_per_client_rare_f1": 0.6579489943732403,
"val_sel_file_present": true
},
"0.5": {
"run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0.5",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.8627083270087127,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8363882297413696,
"val_per_client_rare_f1": 0.7316622118992844,
"val_sel_file_present": true
},
"0.75": {
"run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta0.75",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.8751527301478877,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8498739964533271,
"val_per_client_rare_f1": 0.7719208198370514,
"val_sel_file_present": true
},
"1": {
"run_id": "5gnidd__alpha0.5__seed0__support_weighted__beta1",
"source": "sweep_seed0_1thread",
"val_global_rare_f1": 0.8762664919755059,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8461375258117143,
"val_per_client_rare_f1": 0.7666551346534329,
"val_sel_file_present": true
}
}
}
},
"candidates": [
{
"label": "v_shrink__b1__lam0.5",
"mode": "v_shrink",
"params": {
"beta": 1.0,
"shrink_lambda": 0.5,
"cap_factor": null,
"min_support": null
},
"conditions": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5",
"val_global_rare_f1": 0.6771870761605789,
"val_global_rare_recall": 0.8372489152448566,
"val_global_macro_f1": 0.7085312157748022,
"val_per_client_rare_f1": 0.6394200514807982,
"val_sel_file_present": true
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5",
"val_global_rare_f1": 0.8757058802515706,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.839727653421119,
"val_per_client_rare_f1": 0.7519195999208417,
"val_sel_file_present": true
}
},
"per_condition": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5",
"val_global_rare_f1": 0.6771870761605789,
"baseline_val_global_rare_f1": 0.6380388187784778,
"delta_vs_original": 0.03914825738210115,
"val_per_client_rare_f1": 0.6394200514807982,
"baseline_val_per_client_rare_f1": 0.6079725077419437,
"delta_per_client_vs_original": 0.0314475437388545
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5",
"val_global_rare_f1": 0.8757058802515706,
"baseline_val_global_rare_f1": 0.8615257707605136,
"delta_vs_original": 0.01418010949105697,
"val_per_client_rare_f1": 0.7519195999208417,
"baseline_val_per_client_rare_f1": 0.7252793018231573,
"delta_per_client_vs_original": 0.026640298097684423
}
},
"score": 0.02666418343657906,
"min_condition_delta": 0.01418010949105697,
"secondary_score_per_client": 0.02904392091826946,
"rank": 1,
"rank_secondary": 1
},
{
"label": "v_cap__b1__c4",
"mode": "v_cap",
"params": {
"beta": 1.0,
"shrink_lambda": null,
"cap_factor": 4.0,
"min_support": null
},
"conditions": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c4",
"val_global_rare_f1": 0.6415003766598666,
"val_global_rare_recall": 0.7994428549956025,
"val_global_macro_f1": 0.6751242688482385,
"val_per_client_rare_f1": 0.6043797897480034,
"val_sel_file_present": true
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c4",
"val_global_rare_f1": 0.8708375305448125,
"val_global_rare_recall": 0.9719002398444734,
"val_global_macro_f1": 0.8324391611800375,
"val_per_client_rare_f1": 0.7586411512962095,
"val_sel_file_present": true
}
},
"per_condition": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c4",
"val_global_rare_f1": 0.6415003766598666,
"baseline_val_global_rare_f1": 0.6380388187784778,
"delta_vs_original": 0.0034615578813887993,
"val_per_client_rare_f1": 0.6043797897480034,
"baseline_val_per_client_rare_f1": 0.6079725077419437,
"delta_per_client_vs_original": -0.003592717993940231
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c4",
"val_global_rare_f1": 0.8708375305448125,
"baseline_val_global_rare_f1": 0.8615257707605136,
"delta_vs_original": 0.009311759784298879,
"val_per_client_rare_f1": 0.7586411512962095,
"baseline_val_per_client_rare_f1": 0.7252793018231573,
"delta_per_client_vs_original": 0.03336184947305221
}
},
"score": 0.006386658832843839,
"min_condition_delta": 0.0034615578813887993,
"secondary_score_per_client": 0.014884565739555988,
"rank": 2,
"rank_secondary": 2
},
{
"label": "v_cap__b1__c2",
"mode": "v_cap",
"params": {
"beta": 1.0,
"shrink_lambda": null,
"cap_factor": 2.0,
"min_support": null
},
"conditions": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c2",
"val_global_rare_f1": 0.6324470193202425,
"val_global_rare_recall": 0.8098457414207068,
"val_global_macro_f1": 0.6670829111135634,
"val_per_client_rare_f1": 0.5746261281359909,
"val_sel_file_present": true
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c2",
"val_global_rare_f1": 0.8622554197595921,
"val_global_rare_recall": 0.9719002398444734,
"val_global_macro_f1": 0.8250473485396072,
"val_per_client_rare_f1": 0.7324286498183817,
"val_sel_file_present": true
}
},
"per_condition": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_cap__b1__c2",
"val_global_rare_f1": 0.6324470193202425,
"baseline_val_global_rare_f1": 0.6380388187784778,
"delta_vs_original": -0.0055917994582352915,
"val_per_client_rare_f1": 0.5746261281359909,
"baseline_val_per_client_rare_f1": 0.6079725077419437,
"delta_per_client_vs_original": -0.03334637960595277
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_cap__b1__c2",
"val_global_rare_f1": 0.8622554197595921,
"baseline_val_global_rare_f1": 0.8615257707605136,
"delta_vs_original": 0.0007296489990784583,
"val_per_client_rare_f1": 0.7324286498183817,
"baseline_val_per_client_rare_f1": 0.7252793018231573,
"delta_per_client_vs_original": 0.0071493479952244865
}
},
"score": -0.0024310752295784166,
"min_condition_delta": -0.0055917994582352915,
"secondary_score_per_client": -0.013098515805364142,
"rank": 3,
"rank_secondary": 3
},
{
"label": "v_logweight",
"mode": "v_logweight",
"params": {
"beta": 0.0,
"shrink_lambda": null,
"cap_factor": null,
"min_support": null
},
"conditions": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_logweight",
"val_global_rare_f1": 0.6447702015221075,
"val_global_rare_recall": 0.8413731264629913,
"val_global_macro_f1": 0.6777965963838619,
"val_per_client_rare_f1": 0.6126459310493659,
"val_sel_file_present": true
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_logweight",
"val_global_rare_f1": 0.8269341324183733,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8025209368110234,
"val_per_client_rare_f1": 0.6604201812825267,
"val_sel_file_present": true
}
},
"per_condition": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_logweight",
"val_global_rare_f1": 0.6447702015221075,
"baseline_val_global_rare_f1": 0.6380388187784778,
"delta_vs_original": 0.006731382743629721,
"val_per_client_rare_f1": 0.6126459310493659,
"baseline_val_per_client_rare_f1": 0.6079725077419437,
"delta_per_client_vs_original": 0.004673423307422264
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_logweight",
"val_global_rare_f1": 0.8269341324183733,
"baseline_val_global_rare_f1": 0.8615257707605136,
"delta_vs_original": -0.034591638342140296,
"val_per_client_rare_f1": 0.6604201812825267,
"baseline_val_per_client_rare_f1": 0.7252793018231573,
"delta_per_client_vs_original": -0.06485912054063059
}
},
"score": -0.013930127799255287,
"min_condition_delta": -0.034591638342140296,
"secondary_score_per_client": -0.030092848616604162,
"rank": 4,
"rank_secondary": 5
},
{
"label": "v_minsup__t10",
"mode": "v_minsup",
"params": {
"beta": 0.0,
"shrink_lambda": null,
"cap_factor": null,
"min_support": 10.0
},
"conditions": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_minsup__t10",
"val_global_rare_f1": 0.6214368387313907,
"val_global_rare_recall": 0.8322773169160314,
"val_global_macro_f1": 0.6563272217955932,
"val_per_client_rare_f1": 0.581752384882787,
"val_sel_file_present": true
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_minsup__t10",
"val_global_rare_f1": 0.8177717955235316,
"val_global_rare_recall": 0.9716507803262696,
"val_global_macro_f1": 0.8013311345491211,
"val_per_client_rare_f1": 0.6513807368820619,
"val_sel_file_present": true
}
},
"per_condition": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_minsup__t10",
"val_global_rare_f1": 0.6214368387313907,
"baseline_val_global_rare_f1": 0.6380388187784778,
"delta_vs_original": -0.016601980047087128,
"val_per_client_rare_f1": 0.581752384882787,
"baseline_val_per_client_rare_f1": 0.6079725077419437,
"delta_per_client_vs_original": -0.026220122859156647
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_minsup__t10",
"val_global_rare_f1": 0.8177717955235316,
"baseline_val_global_rare_f1": 0.8615257707605136,
"delta_vs_original": -0.04375397523698199,
"val_per_client_rare_f1": 0.6513807368820619,
"baseline_val_per_client_rare_f1": 0.7252793018231573,
"delta_per_client_vs_original": -0.07389856494109537
}
},
"score": -0.03017797764203456,
"min_condition_delta": -0.04375397523698199,
"secondary_score_per_client": -0.05005934390012601,
"rank": 5,
"rank_secondary": 6
},
{
"label": "v_beta__b1.5",
"mode": "v_beta",
"params": {
"beta": 1.5,
"shrink_lambda": null,
"cap_factor": null,
"min_support": null
},
"conditions": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_beta__b1.5",
"val_global_rare_f1": 0.5462339003533698,
"val_global_rare_recall": 0.6920487487732283,
"val_global_macro_f1": 0.5912778236542747,
"val_per_client_rare_f1": 0.554598371814126,
"val_sel_file_present": true
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_beta__b1.5",
"val_global_rare_f1": 0.8563109585331589,
"val_global_rare_recall": 0.971733905845804,
"val_global_macro_f1": 0.8386791194607972,
"val_per_client_rare_f1": 0.7263992771162339,
"val_sel_file_present": true
}
},
"per_condition": {
"S1": {
"run_id": "xiiotid__alpha0.25__seed0__v_beta__b1.5",
"val_global_rare_f1": 0.5462339003533698,
"baseline_val_global_rare_f1": 0.6380388187784778,
"delta_vs_original": -0.09180491842510796,
"val_per_client_rare_f1": 0.554598371814126,
"baseline_val_per_client_rare_f1": 0.6079725077419437,
"delta_per_client_vs_original": -0.0533741359278177
},
"S2": {
"run_id": "5gnidd__alpha0.5__seed0__v_beta__b1.5",
"val_global_rare_f1": 0.8563109585331589,
"baseline_val_global_rare_f1": 0.8615257707605136,
"delta_vs_original": -0.005214812227354759,
"val_per_client_rare_f1": 0.7263992771162339,
"baseline_val_per_client_rare_f1": 0.7252793018231573,
"delta_per_client_vs_original": 0.0011199752930766138
}
},
"score": -0.04850986532623136,
"min_condition_delta": -0.09180491842510796,
"secondary_score_per_client": -0.026127080317370543,
"rank": 6,
"rank_secondary": 4
}
],
"secondary_ranking_order": [
"v_shrink__b1__lam0.5",
"v_cap__b1__c4",
"v_cap__b1__c2",
"v_beta__b1.5",
"v_logweight",
"v_minsup__t10"
],
"top2": [
"v_shrink__b1__lam0.5",
"v_cap__b1__c4"
],
"selection_margin_top1_minus_top2": 0.02027752460373522,
"note": "Validation only. No test metric of any new run was read before this file was frozen."
}
exploration_v2/ranking.sha256
+2
View File
@@ -0,0 +1,2 @@
f43ebc762d1512c5decfda51f31bef7f1d089a13156e8e3c3d7eae2982b054d3 ranking.json
2026-10-31T05:03:55Z
exploration_v2/resume.sh
Executable
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
# Resume the exploration_v2 driver in its dedicated tmux session.
#
# Safe to call repeatedly. Never kills a run that is already active; the
# scheduler re-attaches to live systemd scopes by name.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/.." && pwd)"
SESSION="explore_v2"
if [ -f "$HERE/DONE" ]; then
echo "[resume] DONE already exists; nothing to resume."
cat "$HERE/DONE"
exit 0
fi
# Is a driver already holding the lock?
if ! flock -n "$HERE/driver.lock" true 2>/dev/null; then
echo "[resume] driver.lock is held: a driver is already running."
exit 0
fi
# Also check for a live driver process (belt and braces).
if pgrep -f '[p]ool_driver.py' >/dev/null 2>&1; then
echo "[resume] pool_driver.py is already alive."
exit 0
fi
if tmux has-session -t "$SESSION" 2>/dev/null; then
echo "[resume] respawning pane in existing tmux session $SESSION"
tmux respawn-pane -k -t "$SESSION" "bash '$HERE/driver.sh'"
else
echo "[resume] creating tmux session $SESSION"
tmux new-session -d -s "$SESSION" -c "$REPO_ROOT" "bash '$HERE/driver.sh'"
fi
tmux list-sessions 2>/dev/null | grep "^$SESSION" || true
exploration_v2/run_one_inner_v2.sh
Executable
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Inner command executed inside the systemd MemoryMax scope by run_one_v2.sh.
# Usage: run_one_inner_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL
set -uo pipefail
if [ "$#" -ne 11 ]; then
echo "usage: run_one_inner_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL" >&2
exit 2
fi
RUN_ID="$1"; DATASET="$2"; ALPHA="$3"; SEED="$4"; MODE="$5"; BETA="$6"
SHRINK="$7"; CAP="$8"; MINSUP="$9"; PART_DIR="${10}"; LABEL="${11}"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/.." && pwd)"
RUN_DIR="$HERE/results/runs/$RUN_ID"
cd "$REPO_ROOT" || exit 3
EXTRA=(--variant-label "$LABEL")
case "$MODE" in
original_protean) ;;
support_weighted) EXTRA+=(--beta "$BETA") ;;
v_beta) EXTRA+=(--beta "$BETA") ;;
v_shrink) EXTRA+=(--beta "$BETA" --shrink-lambda "$SHRINK") ;;
v_cap) EXTRA+=(--beta "$BETA" --cap-factor "$CAP") ;;
v_minsup) EXTRA+=(--beta "$BETA" --min-support "$MINSUP") ;;
v_logweight) EXTRA+=(--beta "$BETA") ;;
*) echo "unknown mode $MODE" >&2; exit 2 ;;
esac
exec "$REPO_ROOT/.venv/bin/python" "$HERE/scripts/thread1_launcher.py" \
"$HERE/scripts/run_variant.py" \
--dataset "$DATASET" \
--alpha "$ALPHA" \
--seed "$SEED" \
--clients 10 \
--rounds 10 \
--local-epochs 3 \
--batch-size 256 \
--lr 0.001 \
--mu 0.1 \
--lam 1.0 \
--aggregation-mode "$MODE" \
"${EXTRA[@]}" \
--partitions-dir "$PART_DIR" \
--results-dir "$RUN_DIR/results" \
--analysis-dir "$RUN_DIR/analysis" \
--processed-dir "$RUN_DIR/processed"
exploration_v2/run_one_v2.sh
Executable
+182
View File
@@ -0,0 +1,182 @@
#!/usr/bin/env bash
# One resumable exploration_v2 run.
#
# run_one_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL
#
# Mirrors results/sweep/run_one.sh:
# * one process, one thread (env pinned here and in thread1_launcher.py)
# * one systemd user scope with MemoryMax=11G (xiiotid) / 15G (5gnidd)
# and MemorySwapMax=0 (the sweep's per-dataset caps)
# * logs under <run>/logs/, COMPLETE only after exit 0 and every primary
# artifact exists and is non-empty
# * summary.tsv upsert through run_record_v2.py
# * .failed marker written on any non-zero exit so the driver can retry
set -uo pipefail
if [ "$#" -ne 11 ]; then
echo "usage: run_one_v2.sh RUN_ID DATASET ALPHA SEED MODE BETA SHRINK CAP MINSUP PART_DIR LABEL" >&2
exit 2
fi
RUN_ID="$1"; DATASET="$2"; ALPHA="$3"; SEED="$4"; MODE="$5"; BETA="$6"
SHRINK="$7"; CAP="$8"; MINSUP="$9"; PART_DIR="${10}"; LABEL="${11}"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/.." && pwd)"
PY="$REPO_ROOT/.venv/bin/python"
RUN_DIR="$HERE/results/runs/$RUN_ID"
RESULTS_DIR="$RUN_DIR/results"
ANALYSIS_DIR="$RUN_DIR/analysis"
PROCESSED_DIR="$RUN_DIR/processed"
LOG_DIR="$RUN_DIR/logs"
SUMMARY="$HERE/summary.csv"
LOCK="$HERE/summary.lock"
STOP_FILE="$HERE/STOP"
# Per-dataset systemd cgroup caps. 5G-NIDD keeps the sweep's 15G. X-IIoTID is
# raised from the sweep's 11G to 13G after three cgroup OOM kills at 11G, while
# two concurrent X-IIoTID processes still stay at 26G <= the 36G budget.
case "$DATASET" in
5gnidd) MEMORY_MAX="15G" ;;
*) MEMORY_MAX="13G" ;;
esac
RUN_TIMEOUT_S="${EXPLORE_RUN_TIMEOUT_S:-43200}"
mkdir -p "$RESULTS_DIR" "$ANALYSIS_DIR" "$PROCESSED_DIR" "$LOG_DIR"
export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=/run/user/$(id -u)/bus}"
export PYTHONDONTWRITEBYTECODE=1
utc() { date -u +%Y-%m-%dT%H:%M:%SZ; }
log() { echo "[$(utc)] $*" >> "$LOG_DIR/run.log"; }
summary_row() {
local status="$1" rc="$2" notes="${3:-}"
local time_file=""
[ -s "$LOG_DIR/time.txt" ] && time_file="$LOG_DIR/time.txt"
"$PY" "$HERE/scripts/run_record_v2.py" record \
--summary "$SUMMARY" --lock "$LOCK" \
--run-name "$RUN_ID" --dataset "$DATASET" --alpha "$ALPHA" \
--beta "$BETA" --seed "$SEED" --aggregation-mode "$MODE" \
--status "$status" --exit-code "$rc" \
--time-file "$time_file" --started "${STARTED:-}" --ended "$(utc)" \
--run-dir "$RUN_DIR" --log-dir "$LOG_DIR" --notes "$notes" \
|| log "WARNING: could not update summary.csv"
}
primary_ok() {
local p
for p in \
"$RESULTS_DIR/$RUN_ID/metadata.json" \
"$RESULTS_DIR/$RUN_ID/history.json" \
"$RESULTS_DIR/$RUN_ID/test_global_metrics.json" \
"$RESULTS_DIR/$RUN_ID/test_confusion_matrix.json" \
"$RESULTS_DIR/$RUN_ID/test_per_class.csv" \
"$RESULTS_DIR/$RUN_ID/val_global_metrics.json" \
"$RESULTS_DIR/$RUN_ID/val_selection.json" \
"$RESULTS_DIR/$RUN_ID/per_client.csv" \
"$RESULTS_DIR/$RUN_ID/val_per_client.csv" \
"$RESULTS_DIR/$RUN_ID/train_support_Nij.csv" \
"$RESULTS_DIR/$RUN_ID/characterization.csv" \
"$RESULTS_DIR/$RUN_ID/rare_unseen.csv" \
"$ANALYSIS_DIR/${RUN_ID}__prototype_snapshot.json" ; do
[ -s "$p" ] || return 1
done
return 0
}
# ---- resumability -------------------------------------------------------
if [ -f "$RUN_DIR/COMPLETE" ] && primary_ok; then
log "SKIP: COMPLETE marker and all primary artifacts present"
summary_row "skipped_complete" 0 "resumed: already complete"
exit 0
fi
if [ -f "$RUN_DIR/COMPLETE" ]; then
mv "$RUN_DIR/COMPLETE" "$RUN_DIR/COMPLETE.invalid.$(date -u +%Y%m%dT%H%M%SZ)"
log "WARNING: moved invalid COMPLETE marker aside"
fi
if [ -e "$STOP_FILE" ]; then
log "STOP present: refusing to start $RUN_ID"
summary_row "stopped" 0 "not started because exploration_v2/STOP exists"
exit 0
fi
# ---- scope unit ---------------------------------------------------------
UNIT="explorev2-$(printf '%s' "$RUN_ID" | sha1sum | cut -c1-12)"
if systemctl --user is-active --quiet "$UNIT.scope" 2>/dev/null; then
log "REFUSE: scope $UNIT already active"
exit 0
fi
# A failed or OOM-killed transient scope keeps its unit name loaded, so a retry
# with the same deterministic name is rejected by systemd-run. Unload it first
# (and give systemd a moment) so retries can actually start.
systemctl --user reset-failed "$UNIT.scope" >/dev/null 2>&1 || true
sleep 1
STARTED="$(utc)"
date -u +%s > "$LOG_DIR/start.epoch"
echo "{\"run_id\": \"$RUN_ID\", \"unit\": \"$UNIT\", \"started_utc\": \"$STARTED\", \"pid\": $$}" > "$RUN_DIR/.running"
log "START dataset=$DATASET alpha=$ALPHA seed=$SEED mode=$MODE beta=$BETA label=$LABEL"
log "command: systemd-run --user --scope -u $UNIT -p MemoryMax=$MEMORY_MAX -p MemorySwapMax=0 ..."
systemd-run --user --scope -q --unit="$UNIT" \
-p "MemoryMax=$MEMORY_MAX" \
-p "MemorySwapMax=0" \
--setenv=OMP_NUM_THREADS=1 \
--setenv=MKL_NUM_THREADS=1 \
--setenv=OPENBLAS_NUM_THREADS=1 \
--setenv=NUMEXPR_NUM_THREADS=1 \
--setenv=VECLIB_MAXIMUM_THREADS=1 \
--setenv=PYTHONDONTWRITEBYTECODE=1 \
/usr/bin/time -v -o "$LOG_DIR/time.txt" \
timeout --signal=TERM --kill-after=60s "$RUN_TIMEOUT_S" \
bash "$HERE/run_one_inner_v2.sh" "$RUN_ID" "$DATASET" "$ALPHA" "$SEED" "$MODE" "$BETA" "$SHRINK" "$CAP" "$MINSUP" "$PART_DIR" "$LABEL" \
> "$LOG_DIR/stdout.log" 2> "$LOG_DIR/stderr.log"
RC=$?
ENDED="$(utc)"
date -u +%s > "$LOG_DIR/end.epoch"
# A systemd cgroup OOM kill can surface as 143 (the timeout process is
# terminated with the rest of the cgroup) rather than 137. Query the transient
# scope's Result so the driver's memory-kill throttle sees it correctly.
SCOPE_RESULT="$(systemctl --user show -p Result --value "$UNIT.scope" 2>/dev/null || true)"
if grep -q "Failed to start transient scope unit" "$LOG_DIR/stderr.log" 2>/dev/null; then
log "scope did not start (unit-name collision); not treating as OOM"
elif [ "$SCOPE_RESULT" = "oom-kill" ]; then
log "scope result=oom-kill (exit was $RC); recording as memory kill 137"
RC=137
fi
log "END exit_code=$RC"
rm -f "$RUN_DIR/.running"
# ---- completion marker / summary ---------------------------------------
if [ "$RC" -eq 0 ] && primary_ok; then
if "$PY" "$HERE/scripts/run_record_v2.py" marker \
--run-dir "$RUN_DIR" --run-name "$RUN_ID" --dataset "$DATASET" \
--alpha "$ALPHA" --beta "$BETA" --seed "$SEED" \
--aggregation-mode "$MODE" --exit-code "$RC" \
--time-file "$LOG_DIR/time.txt" --finished "$ENDED"; then
summary_row "completed" "$RC" ""
log "COMPLETE marker written"
exit 0
else
summary_row "failed" "$RC" "artifacts missing; COMPLETE not written"
printf '{"exit_code": %s, "reason": "artifacts_missing", "ended_utc": "%s"}\n' "$RC" "$ENDED" > "$RUN_DIR/.failed"
exit 1
fi
fi
note="exit=$RC"
[ "$RC" -eq 124 ] && note="timeout after ${RUN_TIMEOUT_S}s"
[ "$RC" -eq 137 ] && note="killed (137): likely MemoryMax=$MEMORY_MAX or OOM"
[ "$RC" -eq 0 ] && note="exit=0 but one or more primary artifacts are missing/empty"
summary_row "failed" "$RC" "$note"
printf '{"exit_code": %s, "reason": "%s", "ended_utc": "%s"}\n' "$RC" "$note" "$ENDED" > "$RUN_DIR/.failed"
log "ERROR: run not marked complete ($note)"
exit 1
exploration_v2/scripts/analyze_results.py
+564
View File
@@ -0,0 +1,564 @@
#!/usr/bin/env python
"""Final analysis for exploration_v2 — all tables and REPORT_DATA.md.
Runs only after the ranking has been frozen and confirmation has finished (or
the driver stopped). It may read test metrics of confirmation runs; it never
uses them to re-rank or re-select anything.
"""
from __future__ import annotations
import csv
import json
import statistics as st
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
EXPL = REPO / "exploration_v2"
RUNS = EXPL / "results" / "runs"
OUT = EXPL / "analysis"
OUT.mkdir(parents=True, exist_ok=True)
def read_csv(p):
if not Path(p).is_file():
return []
with Path(p).open(newline="", encoding="utf-8") as fh:
return list(csv.DictReader(fh))
def write_csv(name, rows):
p = OUT / name
if not rows:
p.write_text("", encoding="utf-8")
return
with p.open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
def jload(p):
p = Path(p)
if not p.is_file():
return {}
return json.loads(p.read_text(encoding="utf-8"))
def resolve(rid):
"""Return the directory containing a run's metric files."""
a = RUNS / rid / "results" / rid
if (a / "val_global_metrics.json").is_file() or (a / "test_global_metrics.json").is_file():
return a
b = REPO / "results" / rid
if (b / "val_global_metrics.json").is_file() or (b / "test_global_metrics.json").is_file():
return b
return None
def metrics(rid):
d = resolve(rid)
if d is None:
return None
vg = jload(d / "val_global_metrics.json")
vs = jload(d / "val_selection.json")
tg = jload(d / "test_global_metrics.json")
return {
"val_global_rare_f1": vg.get("rare_class_macro_f1"),
"val_global_rare_recall": vg.get("rare_class_recall"),
"val_global_macro_f1": vg.get("macro_f1"),
"val_per_client_rare_f1": vs.get("val_rare_class_macro_f1_per_client_mean"),
"test_rare_f1": tg.get("rare_class_macro_f1"),
"test_rare_recall": tg.get("rare_class_recall"),
"test_macro_f1": tg.get("macro_f1"),
}
def summary_rows():
rows = read_csv(EXPL / "summary.csv")
return {r["config"]: r for r in rows}
def pair_stats(diffs):
diffs = [d for d in diffs if d is not None]
if not diffs:
return None, None, 0, 0
return (st.fmean(diffs),
st.stdev(diffs) if len(diffs) > 1 else None,
sum(1 for d in diffs if d > 0),
len(diffs))
def main() -> int:
OUT.mkdir(parents=True, exist_ok=True)
# Enforce the pre-registration gate: never analyse confirmation test
# metrics unless the ranking was frozen first.
rank_file = EXPL / "ranking.json"
freeze_file = EXPL / "ranking.sha256"
if not rank_file.is_file() or not freeze_file.is_file():
raise SystemExit("ranking.json/ranking.sha256 missing: refusing to run "
"the post-confirmation analysis")
import hashlib
recorded = freeze_file.read_text(encoding="utf-8").splitlines()[0].split()[0]
actual = hashlib.sha256(rank_file.read_bytes()).hexdigest()
if recorded != actual:
raise SystemExit("ranking.json hash does not match the frozen record; "
"refusing to re-rank or analyse")
ranking = jload(rank_file)
screen_q = read_csv(EXPL / "queue_screen.csv")
confirm_q = read_csv(EXPL / "queue_confirm.csv")
summ = summary_rows()
top2 = ranking.get("top2", [])
# ---------------- inventory ----------------
inv = []
for phase, queue in (("screening", screen_q), ("confirm", confirm_q)):
for r in queue:
rid = r["run_id"]
d = RUNS / rid
if (d / "COMPLETE").is_file():
status = "done"
elif (d / ".failed").is_file():
status = "failed"
elif (d / ".running").is_file():
status = "running"
elif d.is_dir():
status = "incomplete"
else:
status = "queued"
s = summ.get(rid, {})
inv.append({
"phase": phase, "run_id": rid, "dataset": r["dataset"],
"alpha": r["alpha"], "seed": r["seed"], "mode": r["mode"],
"beta": r.get("beta", ""), "shrink_lambda": r.get("shrink", ""),
"cap_factor": r.get("cap", ""), "min_support": r.get("minsup", ""),
"label": r.get("label", ""), "status": status,
"wall_s": s.get("wall_s", ""), "peak_rss_kb": s.get("peak_rss_kb", ""),
"exit_code": s.get("exit_code", ""), "notes": s.get("notes", ""),
})
write_csv("run_inventory.csv", inv)
# ---------------- screening ranking ----------------
screen_rows = []
for c in ranking.get("candidates", []):
row = {
"rank": c["rank"], "rank_secondary": c.get("rank_secondary", ""),
"label": c["label"], "mode": c["mode"],
"beta": c["params"].get("beta"),
"shrink_lambda": c["params"].get("shrink_lambda"),
"cap_factor": c["params"].get("cap_factor"),
"min_support": c["params"].get("min_support"),
"score_mean_delta_val_global": c["score"],
"min_condition_delta": c["min_condition_delta"],
"secondary_mean_delta_val_per_client": c["secondary_score_per_client"],
}
for cid, per in c["per_condition"].items():
row[f"{cid}_val_global"] = per["val_global_rare_f1"]
row[f"{cid}_baseline"] = per["baseline_val_global_rare_f1"]
row[f"{cid}_delta"] = per["delta_vs_original"]
row[f"{cid}_val_per_client"] = per["val_per_client_rare_f1"]
row[f"{cid}_delta_per_client"] = per["delta_per_client_vs_original"]
screen_rows.append(row)
write_csv("screening_ranking.csv", screen_rows)
# reference rows for the same conditions
ref_rows = []
for cid, ref in ranking.get("current_method_reference", {}).items():
ref_rows.append({
"condition": cid,
"baseline_run": ref["original"]["run_id"],
"baseline_val_global": ref["original"]["val_global_rare_f1"],
"best_beta_by_val_global": ref["best_beta_by_val_global"]["beta"],
"best_val_global": ref["best_beta_by_val_global"]["val_global_rare_f1"],
"best_delta_global": ref["best_beta_by_val_global"]["delta_vs_original"],
"best_beta_by_val_per_client": ref["best_beta_by_val_per_client"]["beta"],
"best_val_per_client": ref["best_beta_by_val_per_client"]["val_per_client_rare_f1"],
"best_delta_per_client": ref["best_beta_by_val_per_client"]["delta_vs_original"],
})
write_csv("screening_references.csv", ref_rows)
# ---------------- confirmation ----------------
conditions = {"S1": ("xiiotid", 0.25), "S2": ("5gnidd", 0.5)}
def rid_for(ds, alpha, seed, label):
return f"{ds}__alpha{alpha:g}__seed{seed}__{label}"
confirm_rows = []
for ds, alpha in conditions.values():
for seed in (3, 4):
base_id = rid_for(ds, alpha, seed, "original_protean")
base = metrics(base_id)
b05 = metrics(rid_for(ds, alpha, seed, "support_weighted__beta0.5"))
b1 = metrics(rid_for(ds, alpha, seed, "support_weighted__beta1"))
if base is None:
continue
cur_candidates = [("original_protean", base),
("support_weighted__beta0.5", b05),
("support_weighted__beta1", b1)]
cur_candidates = [(n, m) for n, m in cur_candidates
if m is not None and m["val_global_rare_f1"] is not None]
cur_name, cur_best = max(
cur_candidates, key=lambda nm: nm[1]["val_global_rare_f1"])
for label in top2 + ["support_weighted__beta0.5", "support_weighted__beta1"]:
rid = rid_for(ds, alpha, seed, label)
m = metrics(rid)
if m is None:
continue
confirm_rows.append({
"dataset": ds, "alpha": alpha, "seed": seed,
"config": label, "run_id": rid,
"val_global_rare_f1": m["val_global_rare_f1"],
"val_per_client_rare_f1": m["val_per_client_rare_f1"],
"test_rare_f1": m["test_rare_f1"],
"test_rare_recall": m["test_rare_recall"],
"baseline_val_global": base["val_global_rare_f1"],
"baseline_test_rare_f1": base["test_rare_f1"],
"delta_val_vs_original": (m["val_global_rare_f1"] - base["val_global_rare_f1"]),
"delta_test_vs_original": (m["test_rare_f1"] - base["test_rare_f1"]),
"delta_val_vs_beta1": ((m["val_global_rare_f1"] - b1["val_global_rare_f1"])
if b1 and b1["val_global_rare_f1"] is not None else None),
"delta_test_vs_beta1": ((m["test_rare_f1"] - b1["test_rare_f1"])
if b1 and b1["test_rare_f1"] is not None else None),
"delta_val_vs_beta05": ((m["val_global_rare_f1"] - b05["val_global_rare_f1"])
if b05 and b05["val_global_rare_f1"] is not None else None),
"delta_test_vs_beta05": ((m["test_rare_f1"] - b05["test_rare_f1"])
if b05 and b05["test_rare_f1"] is not None else None),
"current_best_config": cur_name,
"current_best_val_global": cur_best["val_global_rare_f1"],
"delta_val_vs_current_best": (m["val_global_rare_f1"]
- cur_best["val_global_rare_f1"]),
"delta_test_vs_current_best": ((m["test_rare_f1"] - cur_best["test_rare_f1"])
if m["test_rare_f1"] is not None
and cur_best["test_rare_f1"] is not None
else None),
})
write_csv("confirm_paired.csv", confirm_rows)
summary = []
for label in top2:
rows = [r for r in confirm_rows if r["config"] == label]
dv = [r["delta_val_vs_original"] for r in rows]
dt = [r["delta_test_vs_original"] for r in rows]
d1v = [r["delta_val_vs_beta1"] for r in rows]
d1t = [r["delta_test_vs_beta1"] for r in rows]
d05v = [r["delta_val_vs_beta05"] for r in rows]
d05t = [r["delta_test_vs_beta05"] for r in rows]
dcbv = [r["delta_val_vs_current_best"] for r in rows]
dcbt = [r["delta_test_vs_current_best"] for r in rows]
mcbv, scbv, poscbv, _ = pair_stats(dcbv)
mcbt, scbt, poscbt, _ = pair_stats(dcbt)
mv, sv, pos, n = pair_stats(dv)
mt, stt, post, _ = pair_stats(dt)
m1v, s1v, _, _ = pair_stats(d1v)
m1t, s1t, _, _ = pair_stats(d1t)
m05v, s05v, _, _ = pair_stats(d05v)
m05t, s05t, _, _ = pair_stats(d05t)
per_cond = {}
for cid, (ds, alpha) in conditions.items():
sub = [r for r in rows if r["dataset"] == ds and r["alpha"] == alpha]
per_cond[cid] = [r["delta_val_vs_original"] for r in sub]
improved = bool(mv is not None and mv > 0 and pos == n and n == 4)
summary.append({
"variant": label,
"n_pairs": n,
"mean_delta_val_vs_original": mv, "sd_delta_val_vs_original": sv,
"positive_pairs_val_vs_original": pos,
"mean_delta_test_vs_original": mt, "sd_delta_test_vs_original": stt,
"positive_pairs_test_vs_original": post,
"mean_delta_val_vs_beta1": m1v, "sd_delta_val_vs_beta1": s1v,
"mean_delta_test_vs_beta1": m1t, "sd_delta_test_vs_beta1": s1t,
"mean_delta_val_vs_beta05": m05v, "sd_delta_val_vs_beta05": s05v,
"mean_delta_test_vs_beta05": m05t, "sd_delta_test_vs_beta05": s05t,
"mean_delta_val_vs_current_best": mcbv,
"sd_delta_val_vs_current_best": scbv,
"positive_pairs_val_vs_current_best": poscbv,
"mean_delta_test_vs_current_best": mcbt,
"sd_delta_test_vs_current_best": scbt,
"positive_pairs_test_vs_current_best": poscbt,
"improved_vs_original_per_plan": improved,
"S1_val_deltas": "|".join(f"{x:.6f}" for x in per_cond.get("S1", [])),
"S2_val_deltas": "|".join(f"{x:.6f}" for x in per_cond.get("S2", [])),
})
write_csv("confirm_summary.csv", summary)
# reference configurations (beta1, beta05) summary over the same seeds
ref_summary = []
for label in ("support_weighted__beta0.5", "support_weighted__beta1"):
rows = [r for r in confirm_rows if r["config"] == label]
dv = [r["delta_val_vs_original"] for r in rows]
dt = [r["delta_test_vs_original"] for r in rows]
mv, sv, pos, n = pair_stats(dv)
mt, stt, post, _ = pair_stats(dt)
ref_summary.append({"config": label, "n_pairs": n,
"mean_delta_val_vs_original": mv,
"sd_delta_val_vs_original": sv,
"positive_pairs_val": pos,
"mean_delta_test_vs_original": mt,
"sd_delta_test_vs_original": stt,
"positive_pairs_test": post})
write_csv("confirm_reference_summary.csv", ref_summary)
# ---------------- reference consistency (environment check) ----------
ref_cons = []
for cid, (ds, alpha) in conditions.items():
base = f"{ds}__alpha{alpha:g}__seed0"
new_id = f"{base}__original_protean"
new_dir = RUNS / new_id / "results" / new_id
new_val = (jload(new_dir / "val_global_metrics.json").get("rare_class_macro_f1")
if (new_dir / "val_global_metrics.json").is_file()
else None)
old_id = new_id
old_val = jload(REPO / "results" / old_id / "val_global_metrics.json").get(
"rare_class_macro_f1")
sweep_id = f"{base}__support_weighted__beta0"
sweep_dir = REPO / "results" / "sweep" / "runs" / sweep_id / "results" / sweep_id
sweep_val = (jload(sweep_dir / "val_global_metrics.json").get("rare_class_macro_f1")
if (sweep_dir / "val_global_metrics.json").is_file() else None)
ref_cons.append({
"condition": cid, "dataset": ds, "alpha": alpha,
"new_original_val_global": new_val,
"restored_original_val_global": old_val,
"sweep_beta0_val_global": sweep_val,
"new_minus_restored": (new_val - old_val) if new_val is not None and old_val is not None else None,
"new_minus_sweep_beta0": (new_val - sweep_val) if new_val is not None and sweep_val is not None else None,
})
write_csv("reference_consistency.csv", ref_cons)
# ---------------- wall time / memory ----------------
wall = []
for r in inv:
wall.append({"phase": r["phase"], "run_id": r["run_id"],
"status": r["status"], "wall_s": r["wall_s"],
"wall_h": (round(float(r["wall_s"]) / 3600, 3)
if r["wall_s"] not in ("", None) else None),
"peak_rss_gib": (round(float(r["peak_rss_kb"]) / 1048576, 3)
if r["peak_rss_kb"] not in ("", None) else None)})
write_csv("walltime.csv", wall)
# ---------------- REPORT_DATA.md ----------------
def fmt(x, nd=4):
if x is None or x == "":
return ""
try:
return f"{float(x):.{nd}f}"
except Exception:
return str(x)
lines = []
lines.append("# exploration_v2 — report data\n")
from datetime import datetime, timezone
lines.append(f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}\n")
lines.append("This file is generated by `exploration_v2/scripts/analyze_results.py`. "
"All numbers come from files on disk. Missing rows mean a run did not "
"produce the metric.\n")
lines.append("## Plan and ranking freeze\n")
lines.append("| item | value |\n|---|---|\n")
for name in ("PLAN.sha256", "ranking.sha256"):
p = EXPL / name
if p.is_file():
content = p.read_text(encoding="utf-8").splitlines()
lines.append(f"| `{name}` | `{content[0] if content else ''}`"
f"{(' at ' + content[1]) if len(content) > 1 else ''} |\n")
lines.append("\n")
lines.append("## Screening ranking (validation only)\n")
if ranking:
lines.append(f"Top-1 minus top-2 validation score (selection margin): "
f"{fmt(ranking.get('selection_margin_top1_minus_top2'), 6)} "
f"(top2 = {ranking.get('top2')}).\n\n")
lines.append("Score = mean over S1,S2 of the paired validation difference in "
"global rare-class macro-F1 versus the original baseline.\n\n")
lines.append("| rank | variant | β | λ | cap | min N | score | min cond Δ | "
"S1 Δ | S2 Δ | secondary rank |\n|---|---|---|---|---|---|---|---|---|---|---|\n")
for r in screen_rows:
lines.append(f"| {r['rank']} | `{r['label']}` | {fmt(r['beta'],2)} | "
f"{fmt(r['shrink_lambda'],2)} | {fmt(r['cap_factor'],1)} | "
f"{fmt(r['min_support'],0)} | {fmt(r['score_mean_delta_val_global'])} | "
f"{fmt(r['min_condition_delta'])} | {fmt(r.get('S1_delta'))} | "
f"{fmt(r.get('S2_delta'))} | {r['rank_secondary']} |\n")
lines.append("\n")
lines.append("## Confirmatory results (per seed)\n")
lines.append("| dataset | α | seed | config | val rare-F1 | test rare-F1 | "
"Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 |\n"
"|---|---|---|---|---|---|---|---|---|---|\n")
for r in confirm_rows:
lines.append(f"| {r['dataset']} | {fmt(r['alpha'],2)} | {r['seed']} | "
f"`{r['config']}` | {fmt(r['val_global_rare_f1'])} | "
f"{fmt(r['test_rare_f1'])} | {fmt(r['delta_val_vs_original'])} | "
f"{fmt(r['delta_test_vs_original'])} | {fmt(r['delta_val_vs_beta1'])} | "
f"{fmt(r['delta_test_vs_beta1'])} |\n")
lines.append("\n")
lines.append("## Confirmatory summary\n")
lines.append("| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | "
"sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | "
"mean Δtest vs current best | improved per plan |\n"
"|---|---|---|---|---|---|---|---|---|\n")
for r in summary:
lines.append(f"| `{r['variant']}` | {fmt(r['mean_delta_val_vs_original'])} | "
f"{fmt(r['sd_delta_val_vs_original'])} | "
f"{r['positive_pairs_val_vs_original']}/{r['n_pairs']} | "
f"{fmt(r['mean_delta_test_vs_original'])} | "
f"{fmt(r['sd_delta_test_vs_original'])} | "
f"{fmt(r['mean_delta_val_vs_beta1'])} | "
f"{fmt(r['mean_delta_test_vs_beta1'])} | "
f"{fmt(r.get('mean_delta_val_vs_current_best'))} | "
f"{fmt(r.get('mean_delta_test_vs_current_best'))} | "
f"{r['improved_vs_original_per_plan']} |\n")
lines.append("\n")
lines.append("## Reference configurations on the confirmation seeds\n")
lines.append("| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive |\n"
"|---|---|---|---|---|---|---|\n")
for r in ref_summary:
lines.append(f"| `{r['config']}` | {fmt(r['mean_delta_val_vs_original'])} | "
f"{fmt(r['sd_delta_val_vs_original'])} | {r['positive_pairs_val']} | "
f"{fmt(r['mean_delta_test_vs_original'])} | "
f"{fmt(r['sd_delta_test_vs_original'])} | {r['positive_pairs_test']} |\n")
lines.append("\n")
lines.append("## Reference consistency (environment check)\n")
lines.append("| condition | new original val | restored original val | sweep β=0 val | "
"new − restored | new − sweep β=0 |\n|---|---|---|---|---|---|\n")
for r in ref_cons:
lines.append(f"| {r['condition']} | {fmt(r['new_original_val_global'])} | "
f"{fmt(r['restored_original_val_global'])} | "
f"{fmt(r['sweep_beta0_val_global'])} | "
f"{fmt(r['new_minus_restored'])} | "
f"{fmt(r['new_minus_sweep_beta0'])} |\n")
lines.append("\n")
lines.append("## Wall time and memory (new runs)\n")
lines.append("| phase | run | status | wall h | peak GiB |\n|---|---|---|---|---|\n")
for r in wall:
lines.append(f"| {r['phase']} | `{r['run_id']}` | {r['status']} | "
f"{fmt(r['wall_h'],2)} | {fmt(r['peak_rss_gib'],2)} |\n")
lines.append("\n")
lines.append("## Failed or missing runs\n")
bad = [r for r in inv if r["status"] in ("failed", "incomplete", "queued")]
if not bad:
lines.append("None.\n")
else:
lines.append("| phase | run | status | exit | notes |\n|---|---|---|---|---|\n")
for r in bad:
lines.append(f"| {r['phase']} | `{r['run_id']}` | {r['status']} | "
f"{r['exit_code']} | {r['notes']} |\n")
lines.append("\n")
(EXPL / "REPORT_DATA.md").write_text("".join(lines), encoding="utf-8")
# ---- inject the same tables into REPORT.md between the markers -------
def f(x, nd=4):
if x in (None, ""):
return ""
try:
return f"{float(x):.{nd}f}"
except Exception:
return str(x)
auto = []
auto.append("## 5. Results of the new runs\n\n")
auto.append("### 5.1 Screening ranking (validation only)\n\n")
if ranking:
auto.append(f"Top-1 minus top-2 validation score (selection margin): "
f"{f(ranking.get('selection_margin_top1_minus_top2'), 6)} "
f"(top2 = {ranking.get('top2')}).\n\n")
auto.append("Score = mean over S1,S2 of the paired validation difference in "
"global rare-class macro-F1 versus the in-environment original "
"baseline. `rank_secondary` uses the deployed per-client-mean "
"metric.\n\n")
auto.append("| rank | variant | β | λ | cap | min N | score | min cond Δ | "
"S1 Δ | S2 Δ | rank_secondary |\n|---|---|---|---|---|---|---|---|---|---|\n")
for r in read_csv(OUT / "screening_ranking.csv"):
auto.append(f"| {r['rank']} | `{r['label']}` | {f(r['beta'],2)} | "
f"{f(r['shrink_lambda'],2)} | {f(r['cap_factor'],1)} | "
f"{f(r['min_support'],0)} | {f(r['score_mean_delta_val_global'])} | "
f"{f(r['min_condition_delta'])} | {f(r.get('S1_delta'))} | "
f"{f(r.get('S2_delta'))} | {r['rank_secondary']} |\n")
auto.append("\n")
auto.append("### 5.2 Confirmatory runs, per seed\n\n")
auto.append("| dataset | α | seed | config | val rare-F1 | test rare-F1 | "
"Δval vs orig | Δtest vs orig | Δval vs β=1 | Δtest vs β=1 |\n"
"|---|---|---|---|---|---|---|---|---|---|\n")
for r in read_csv(OUT / "confirm_paired.csv"):
auto.append(f"| {r['dataset']} | {f(r['alpha'],2)} | {r['seed']} | "
f"`{r['config']}` | {f(r['val_global_rare_f1'])} | "
f"{f(r['test_rare_f1'])} | {f(r['delta_val_vs_original'])} | "
f"{f(r['delta_test_vs_original'])} | {f(r['delta_val_vs_beta1'])} | "
f"{f(r['delta_test_vs_beta1'])} |\n")
auto.append("\n")
auto.append("### 5.3 Confirmatory summary\n\n")
auto.append("A variant “improves” under the pre-registered rule only if the "
"mean validation difference over the four pairs is positive **and** "
"all four are positive.\n\n")
auto.append("| variant | mean Δval vs orig | sd | positive/4 | mean Δtest vs orig | "
"sd | mean Δval vs β=1 | mean Δtest vs β=1 | mean Δval vs current best | "
"mean Δtest vs current best | improved |\n"
"|---|---|---|---|---|---|---|---|---|\n")
for r in read_csv(OUT / "confirm_summary.csv"):
auto.append(f"| `{r['variant']}` | {f(r['mean_delta_val_vs_original'])} | "
f"{f(r['sd_delta_val_vs_original'])} | "
f"{r['positive_pairs_val_vs_original']}/{r['n_pairs']} | "
f"{f(r['mean_delta_test_vs_original'])} | "
f"{f(r['sd_delta_test_vs_original'])} | "
f"{f(r['mean_delta_val_vs_beta1'])} | "
f"{f(r['mean_delta_test_vs_beta1'])} | "
f"{f(r.get('mean_delta_val_vs_current_best'))} | "
f"{f(r.get('mean_delta_test_vs_current_best'))} | "
f"{r['improved_vs_original_per_plan']} |\n")
auto.append("\n")
auto.append("### 5.4 Reference configurations on the confirmation seeds\n\n")
auto.append("| config | mean Δval vs orig | sd | positive | mean Δtest vs orig | sd | positive |\n"
"|---|---|---|---|---|---|---|\n")
for r in read_csv(OUT / "confirm_reference_summary.csv"):
auto.append(f"| `{r['config']}` | {f(r['mean_delta_val_vs_original'])} | "
f"{f(r['sd_delta_val_vs_original'])} | {r['positive_pairs_val']} | "
f"{f(r['mean_delta_test_vs_original'])} | "
f"{f(r['sd_delta_test_vs_original'])} | {r['positive_pairs_test']} |\n")
auto.append("\n")
auto.append("### 5.5 Reference consistency (environment check)\n\n")
auto.append("| condition | new original val | restored original val | sweep β=0 val | "
"new − restored | new − sweep β=0 |\n|---|---|---|---|---|---|\n")
for r in read_csv(OUT / "reference_consistency.csv"):
auto.append(f"| {r['condition']} | {f(r['new_original_val_global'])} | "
f"{f(r['restored_original_val_global'])} | "
f"{f(r['sweep_beta0_val_global'])} | {f(r['new_minus_restored'])} | "
f"{f(r['new_minus_sweep_beta0'])} |\n")
auto.append("\n")
auto.append("### 5.6 Wall time and memory\n\n")
auto.append("| phase | run | status | wall h | peak GiB |\n|---|---|---|---|---|\n")
for r in read_csv(OUT / "walltime.csv"):
auto.append(f"| {r['phase']} | `{r['run_id']}` | {r['status']} | "
f"{f(r['wall_h'],2)} | {f(r['peak_rss_gib'],2)} |\n")
auto.append("\n")
bad = [r for r in inv if r["status"] in ("failed", "incomplete", "queued")]
auto.append("### 5.7 Failed or missing runs\n\n")
if not bad:
auto.append("None.\n")
else:
for r in bad:
auto.append(f"- `{r['run_id']}` ({r['phase']}): {r['status']}, "
f"exit {r['exit_code']}, {r['notes']}\n")
report = EXPL / "REPORT.md"
if report.is_file():
text = report.read_text(encoding="utf-8")
start = "<!-- AUTO_RESULTS_START -->"
end = "<!-- AUTO_RESULTS_END -->"
if start in text and end in text:
pre = text.split(start, 1)[0] + start + "\n"
post = "\n" + end + text.split(end, 1)[1]
report.write_text(pre + "".join(auto) + post, encoding="utf-8")
print(json.dumps({"screened": len(screen_rows), "confirm_rows": len(confirm_rows),
"summary_rows": len(summary),
"failed": [r["run_id"] for r in inv if r["status"] == "failed"]},
indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/confirm_effects.py
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python
"""Per-class and per-client effects of the confirmed variants (post-freeze).
Reads the 20 confirmation runs and the matching original baseline runs. Writes
two CSVs in exploration_v2/analysis/. Descriptive only; no selection.
"""
from __future__ import annotations
import csv
import statistics as st
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
RUNS = REPO / "exploration_v2" / "results" / "runs"
OUT = REPO / "exploration_v2" / "analysis"
PAIRS = [("xiiotid", 0.25, 3), ("xiiotid", 0.25, 4),
("5gnidd", 0.5, 3), ("5gnidd", 0.5, 4)]
VARIANTS = ["v_shrink__b1__lam0.5", "v_cap__b1__c4",
"support_weighted__beta0.5", "support_weighted__beta1"]
def rows(p):
with p.open(newline="", encoding="utf-8") as fh:
return list(csv.DictReader(fh))
def base(rid):
return RUNS / rid / "results" / rid
def write(name, data):
if not data:
return
with (OUT / name).open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(data[0]))
w.writeheader()
w.writerows(data)
def main() -> int:
OUT.mkdir(parents=True, exist_ok=True)
cls_rows, cli_rows = [], []
for ds, alpha, seed in PAIRS:
b = f"{ds}__alpha{alpha:g}__seed{seed}__original_protean"
bp = {int(r["class_id"]): r for r in rows(base(b) / "test_per_class.csv")}
bm = {int(r["client_id"]): r for r in rows(base(b) / "val_per_client.csv")}
for label in VARIANTS:
c = f"{ds}__alpha{alpha:g}__seed{seed}__{label}"
cp = {int(r["class_id"]): r for r in rows(base(c) / "test_per_class.csv")}
for cid, br in bp.items():
if float(br["support"]) <= 0:
continue
cr = cp.get(cid)
if cr is None:
continue
cls_rows.append({
"dataset": ds, "alpha": alpha, "seed": seed, "variant": label,
"class_id": cid, "class_name": br["class_name"],
"test_support": int(float(br["support"])),
"base_f1": br["f1"], "variant_f1": cr["f1"],
"delta_f1": float(cr["f1"]) - float(br["f1"]),
"base_recall": br["recall"], "variant_recall": cr["recall"],
"delta_recall": float(cr["recall"]) - float(br["recall"]),
})
cm = {int(r["client_id"]): r for r in rows(base(c) / "val_per_client.csv")}
for cid, br in bm.items():
cr = cm.get(cid)
if cr is None or br["rare_class_macro_f1"] in ("", None):
continue
if cr["rare_class_macro_f1"] in ("", None):
continue
cli_rows.append({
"dataset": ds, "alpha": alpha, "seed": seed, "variant": label,
"client_id": cid, "n_samples": cr["n_samples"],
"base_val_rare_f1": br["rare_class_macro_f1"],
"variant_val_rare_f1": cr["rare_class_macro_f1"],
"delta_val_rare_f1": (float(cr["rare_class_macro_f1"])
- float(br["rare_class_macro_f1"])),
})
write("confirm_per_class_variant.csv", cls_rows)
write("confirm_per_client_variant.csv", cli_rows)
# compact pooled summary for the two confirmed variants
for label in VARIANTS:
print("VAR", label)
for ds in ("xiiotid", "5gnidd"):
sub = [r for r in cls_rows if r["variant"] == label and r["dataset"] == ds]
by = {}
for r in sub:
by.setdefault(r["class_name"], []).append(r["delta_f1"])
order = sorted(by, key=lambda k: -st.fmean(by[k]))
print(" ", ds, [(k, round(st.fmean(by[k]), 3)) for k in order])
cc = [r for r in cli_rows if r["variant"] == label and r["dataset"] == ds]
d = [r["delta_val_rare_f1"] for r in cc]
print(" per-client mean %.4f improve/degrade %d/%d" % (
st.fmean(d), sum(1 for x in d if x > 0), sum(1 for x in d if x < 0)))
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/make_confirm_queue.py
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python
"""Build exploration_v2/queue_confirm.csv from the frozen ranking.json.
Confirmation design (PLAN.md §6): top-2 variants plus three reference configs
(original, support_weighted beta=0.5, support_weighted beta=1.0) on the two
screening conditions and the two new seeds 3 and 4.
"""
from __future__ import annotations
import csv
import json
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
EXPL = REPO / "exploration_v2"
def main() -> int:
ranking = json.loads((EXPL / "ranking.json").read_text(encoding="utf-8"))
top2 = ranking["top2"]
if len(top2) != 2:
raise SystemExit(f"ranking.json does not contain 2 candidates: {top2}")
specs = {}
with (EXPL / "queue_screen.csv").open(newline="", encoding="utf-8") as fh:
for row in csv.DictReader(fh):
specs[row["label"]] = row
for label in top2:
if label not in specs:
raise SystemExit(f"top candidate {label} has no screening spec")
rows = []
for ds, alpha in (("xiiotid", 0.25), ("5gnidd", 0.5)):
for seed in (3, 4):
base = f"{ds}__alpha{alpha:g}__seed{seed}"
rows.append(dict(run_id=f"{base}__original_protean", dataset=ds,
alpha=alpha, seed=seed, mode="original_protean",
beta="", shrink="", cap="", minsup="",
part_dir="exploration_v2/partitions",
label="original_protean", role="reference"))
for b in ("0.5", "1"):
rows.append(dict(run_id=f"{base}__support_weighted__beta{b}",
dataset=ds, alpha=alpha, seed=seed,
mode="support_weighted", beta=b, shrink="", cap="",
minsup="", part_dir="exploration_v2/partitions",
label=f"support_weighted__beta{b}",
role="reference"))
for label in top2:
s = specs[label]
rows.append(dict(run_id=f"{base}__{label}", dataset=ds,
alpha=alpha, seed=seed, mode=s["mode"],
beta=s["beta"], shrink=s["shrink"], cap=s["cap"],
minsup=s["minsup"],
part_dir="exploration_v2/partitions",
label=label, role="confirm_variant"))
out = EXPL / "queue_confirm.csv"
with out.open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
print(f"wrote {out} ({len(rows)} rows) top2={top2}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/make_manifest_v2.py
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env python
"""Create one new-seed partition manifest inside exploration_v2/.
This reuses the frozen project partition procedure
(``protean_research.data.pipeline.create_partition`` ->
``per_class_dirichlet_multinomial``) and the frozen split/preprocessing code.
It never writes into ``artifacts/partitions``.
Usage:
python exploration_v2/scripts/make_manifest_v2.py --dataset xiiotid --alpha 0.25 --seed 3
"""
from __future__ import annotations
import argparse
import csv
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO / "src"))
from protean_research.data import manifest as manifest_mod # noqa: E402
from protean_research.data import rare_unseen # noqa: E402
from protean_research.data.characterization import characterize_support # noqa: E402
from protean_research.data.pipeline import ( # noqa: E402
create_partition,
prepare_dataset,
preprocessing_content_hash,
)
from protean_research.utils.config import load_datasets_config # noqa: E402
def write_csv(path: Path, rows: list[dict]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if not rows:
path.write_text("", encoding="utf-8")
return
with path.open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--dataset", required=True)
ap.add_argument("--alpha", type=float, required=True)
ap.add_argument("--seed", type=int, required=True)
ap.add_argument("--clients", type=int, default=10)
ap.add_argument("--partitions-dir", default=None)
ap.add_argument("--analysis-dir", default=None)
args = ap.parse_args(argv)
partitions_dir = Path(args.partitions_dir or REPO / "exploration_v2" / "partitions")
analysis_dir = Path(args.analysis_dir or REPO / "exploration_v2" / "analysis" / "partitions")
partitions_dir.mkdir(parents=True, exist_ok=True)
analysis_dir.mkdir(parents=True, exist_ok=True)
datasets_cfg = load_datasets_config()
bundle = prepare_dataset(args.dataset, seed=args.seed, datasets_cfg=datasets_cfg)
pre_hash = preprocessing_content_hash(bundle.preprocessor)
path = manifest_mod.manifest_path(partitions_dir, args.dataset, args.alpha, args.seed)
if path.exists():
manifest = manifest_mod.load_manifest(path)
print(f"exists: {path}")
else:
manifest, assignments, support = create_partition(
bundle,
alpha=args.alpha,
seed=args.seed,
num_clients=args.clients,
partition_dir=partitions_dir,
preprocessing_hash=pre_hash,
)
print(f"created: {path}")
# Exact reload check against the stored recipe.
split, assignments, support = manifest_mod.reproduce_train_assignments(
manifest, bundle.labels)
tag = f"{args.dataset}__alpha{args.alpha:g}__seed{args.seed}"
write_csv(analysis_dir / f"{tag}__characterization.csv",
characterize_support(support, bundle.class_names))
write_csv(analysis_dir / f"{tag}__rare_unseen.csv",
rare_unseen.rare_unseen_rows(
support, bundle.class_names, bundle.cleaned.benign_index,
args.dataset, args.alpha, args.seed))
meta = {
"dataset": args.dataset,
"alpha": args.alpha,
"seed": args.seed,
"manifest_path": str(path),
"partition_hash": manifest["partition_hash"],
"manifest_content_hash": manifest_mod.manifest_content_hash(manifest),
"preprocessing_hash": pre_hash,
"num_clients": args.clients,
"num_classes": len(bundle.class_names),
"created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"procedure": "protean_research.data.pipeline.create_partition "
"(per_class_dirichlet_multinomial), frozen split/preprocessing",
}
(analysis_dir / f"{tag}__manifest_meta.json").write_text(
json.dumps(meta, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({"manifest": str(path), "partition_hash": meta["partition_hash"]}))
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/phase2_diagnose.py
+421
View File
@@ -0,0 +1,421 @@
#!/usr/bin/env python
"""Phase 2 read-only diagnosis of the existing 108-run grid.
Reads only already-restored result directories (results/<run>/) plus the
committed summary tables under artifacts/analysis/. Writes only inside
exploration_v2/results/. Does not import project code, does not train.
"""
from __future__ import annotations
import csv
import json
import math
import statistics as st
from itertools import product
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
OUT = REPO / "exploration_v2" / "results"
OUT.mkdir(parents=True, exist_ok=True)
DATASETS = ["xiiotid", "5gnidd"]
ALPHAS = [0.25, 0.5, 0.75]
SEEDS = [0, 1, 2]
BETAS = [0.0, 0.25, 0.5, 0.75, 1.0]
def read_json(path):
with path.open(encoding="utf-8") as fh:
return json.load(fh)
def read_csv(path):
with path.open(newline="", encoding="utf-8") as fh:
return list(csv.DictReader(fh))
def fnum(v):
if v is None or v == "":
return None
try:
return float(v)
except (TypeError, ValueError):
return None
def run_dir(dataset, alpha, seed, tag):
return REPO / "results" / f"{dataset}__alpha{alpha:g}__seed{seed}__{tag}"
def tag_for(beta):
if beta is None:
return "original_protean"
return f"support_weighted__beta{beta:g}"
def load_run(dataset, alpha, seed, beta):
d = run_dir(dataset, alpha, seed, tag_for(beta))
if not d.is_dir():
return None
rec = {
"dataset": dataset, "alpha": alpha, "seed": seed,
"beta": beta, "mode": "original" if beta is None else "support_weighted",
"dir": str(d.relative_to(REPO)),
}
for name in ["metadata", "val_selection", "val_global_metrics",
"test_global_metrics", "history"]:
p = d / f"{name}.json"
rec[name] = read_json(p) if p.exists() else None
for name in ["val_per_client", "per_client", "val_per_class", "test_per_class",
"train_support_Nij", "rare_unseen"]:
p = d / f"{name}.csv"
rec[name] = read_csv(p) if p.exists() else None
return rec
def support_stats(rec, class_names):
rows = rec.get("train_support_Nij") or []
if not rows:
return {}
support = []
for r in rows:
support.append([int(float(r[c])) for c in class_names])
n_clients = len(support)
n_classes = len(class_names)
per_class = []
for j in range(n_classes):
vals = [support[i][j] for i in range(n_clients)]
pos = [v for v in vals if v > 0]
total = sum(vals)
if not pos:
per_class.append({"class_id": j, "n_contrib": 0, "cv": None,
"max_share": 0.0, "min_pos": 0, "max_pos": 0})
continue
mean = sum(pos) / len(pos)
sd = st.pstdev(pos) if len(pos) > 1 else 0.0
beta1 = [v ** 1.0 for v in pos]
per_class.append({
"class_id": j,
"n_contrib": len(pos),
"cv": (sd / mean) if mean else None,
"max_share": max(beta1) / sum(beta1) if sum(beta1) else None,
"min_pos": min(pos),
"max_pos": max(pos),
"max_over_min": (max(pos) / min(pos)) if min(pos) else None,
})
cvs = [p["cv"] for p in per_class if p["cv"] is not None]
return {
"client_sizes": [sum(row) for row in support],
"mean_support_cv": st.fmean(cvs) if cvs else None,
"max_support_cv": max(cvs) if cvs else None,
"n_classes_lt_M": sum(1 for p in per_class if p["n_contrib"] < n_clients),
"n_classes_singleton": sum(1 for p in per_class if p["n_contrib"] == 1),
"max_weight_share_beta1": max(
(p["max_share"] for p in per_class if p["max_share"] is not None), default=None),
"mean_weight_share_beta1": st.fmean(
[p["max_share"] for p in per_class if p["max_share"] is not None])
if any(p["max_share"] is not None for p in per_class) else None,
"per_class": per_class,
}
def main() -> int:
runs = {}
for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS):
ref = load_run(dataset, alpha, seed, None)
if ref is None:
print(f"MISSING original {dataset} {alpha} {seed}")
continue
class_names = ref["metadata"]["class_names"]
runs[(dataset, alpha, seed, None)] = ref
for beta in BETAS:
r = load_run(dataset, alpha, seed, beta)
if r is None:
print(f"MISSING beta {dataset} {alpha} {seed} {beta}")
continue
runs[(dataset, alpha, seed, beta)] = r
# ---------- per-run metric table ----------
rows = []
for (dataset, alpha, seed, beta), rec in runs.items():
vs = rec.get("val_selection") or {}
vg = rec.get("val_global_metrics") or {}
tg = rec.get("test_global_metrics") or {}
md = rec.get("metadata") or {}
class_names = md.get("class_names") or []
ss = support_stats(rec, class_names)
rows.append({
"dataset": dataset, "alpha": alpha, "seed": seed,
"mode": rec["mode"], "beta": "" if beta is None else beta,
"val_sel_per_client_rare_f1": vs.get("val_rare_class_macro_f1_per_client_mean"),
"val_rare_f1_global": vg.get("rare_class_macro_f1"),
"val_rare_recall_global": vg.get("rare_class_recall"),
"val_macro_f1": vg.get("macro_f1"),
"val_accuracy": vg.get("accuracy"),
"val_weighted_f1": vg.get("weighted_f1"),
"test_rare_f1_global": tg.get("rare_class_macro_f1"),
"test_rare_recall_global": tg.get("rare_class_recall"),
"test_macro_f1": tg.get("macro_f1"),
"test_accuracy": tg.get("accuracy"),
"test_weighted_f1": tg.get("weighted_f1"),
"rare_support_test": tg.get("rare_support"),
"val_rare_support": vg.get("rare_support"),
"mean_support_cv": ss.get("mean_support_cv"),
"max_support_cv": ss.get("max_support_cv"),
"n_classes_lt_M": ss.get("n_classes_lt_M"),
"n_classes_singleton": ss.get("n_classes_singleton"),
"max_weight_share_beta1": ss.get("max_weight_share_beta1"),
"mean_weight_share_beta1": ss.get("mean_weight_share_beta1"),
"client_size_ratio_mean": (
st.fmean(ss["client_sizes"]) / (sum(ss["client_sizes"]) / len(ss["client_sizes"]))
if ss.get("client_sizes") else None),
})
# client size ratio proper: mean client size / smallest? Use max/mean computed below.
for r in rows:
rec = runs[(r["dataset"], r["alpha"], r["seed"],
None if r["mode"] == "original" else r["beta"])]
ss = support_stats(rec, (rec["metadata"] or {}).get("class_names") or [])
sizes = ss.get("client_sizes")
r["client_size_ratio_mean"] = (max(sizes) / st.fmean(sizes)) if sizes else None
field_order = list(rows[0].keys())
with (OUT / "phase2_per_run.csv").open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=field_order)
w.writeheader()
w.writerows(rows)
# ---------- paired deltas vs original and vs beta=0 ----------
by_key = {(r["dataset"], r["alpha"], r["seed"], r["mode"], r["beta"]): r for r in rows}
deltas = []
for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS):
base = by_key.get((dataset, alpha, seed, "original", ""))
b0 = by_key.get((dataset, alpha, seed, "support_weighted", 0.0))
if not base:
continue
for beta in BETAS:
cur = by_key.get((dataset, alpha, seed, "support_weighted", beta))
if not cur:
continue
for refname, ref in (("vs_original", base), ("vs_beta0", b0)):
if not ref:
continue
deltas.append({
"dataset": dataset, "alpha": alpha, "seed": seed, "beta": beta,
"reference": refname,
"d_val_sel": (cur["val_sel_per_client_rare_f1"] or 0) - (ref["val_sel_per_client_rare_f1"] or 0),
"d_val_rare_f1_global": (cur["val_rare_f1_global"] or 0) - (ref["val_rare_f1_global"] or 0),
"d_test_rare_f1": (cur["test_rare_f1_global"] or 0) - (ref["test_rare_f1_global"] or 0),
"d_val_rare_recall": (cur["val_rare_recall_global"] or 0) - (ref["val_rare_recall_global"] or 0),
"d_test_rare_recall": (cur["test_rare_recall_global"] or 0) - (ref["test_rare_recall_global"] or 0),
"d_val_macro_f1": (cur["val_macro_f1"] or 0) - (ref["val_macro_f1"] or 0),
"d_test_macro_f1": (cur["test_macro_f1"] or 0) - (ref["test_macro_f1"] or 0),
"d_val_accuracy": (cur["val_accuracy"] or 0) - (ref["val_accuracy"] or 0),
"d_test_accuracy": (cur["test_accuracy"] or 0) - (ref["test_accuracy"] or 0),
"val_test_gap_cur": (cur["test_rare_f1_global"] or 0) - (cur["val_rare_f1_global"] or 0),
"val_test_gap_ref": (ref["test_rare_f1_global"] or 0) - (ref["val_rare_f1_global"] or 0),
})
with (OUT / "phase2_paired_deltas.csv").open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(deltas[0].keys()))
w.writeheader()
w.writerows(deltas)
# ---------- pooled per condition x beta ----------
pooled = []
for dataset, alpha in product(DATASETS, ALPHAS):
for beta in [None] + BETAS:
vals = []
for seed in SEEDS:
r = by_key.get((dataset, alpha, seed,
"original" if beta is None else "support_weighted",
"" if beta is None else beta))
if r:
vals.append(r)
if not vals:
continue
def mean(key):
xs = [v[key] for v in vals if v[key] is not None]
return st.fmean(xs) if xs else None
def sd(key):
xs = [v[key] for v in vals if v[key] is not None]
return st.stdev(xs) if len(xs) > 1 else None
pooled.append({
"dataset": dataset, "alpha": alpha,
"beta": "" if beta is None else beta,
"n_seeds": len(vals),
"val_sel_mean": mean("val_sel_per_client_rare_f1"),
"val_sel_sd": sd("val_sel_per_client_rare_f1"),
"val_rare_f1_global_mean": mean("val_rare_f1_global"),
"val_rare_f1_global_sd": sd("val_rare_f1_global"),
"test_rare_f1_mean": mean("test_rare_f1_global"),
"test_rare_f1_sd": sd("test_rare_f1_global"),
"val_rare_recall_mean": mean("val_rare_recall_global"),
"test_rare_recall_mean": mean("test_rare_recall_global"),
"val_macro_f1_mean": mean("val_macro_f1"),
"test_macro_f1_mean": mean("test_macro_f1"),
"val_accuracy_mean": mean("val_accuracy"),
"test_accuracy_mean": mean("test_accuracy"),
"mean_support_cv": mean("mean_support_cv"),
"mean_weight_share_beta1": mean("mean_weight_share_beta1"),
"n_classes_lt_M": mean("n_classes_lt_M"),
})
with (OUT / "phase2_pooled_by_beta.csv").open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(pooled[0].keys()))
w.writeheader()
w.writerows(pooled)
# ---------- adjacent beta-step deltas (pooled over seeds) ----------
steps = []
for dataset, alpha in product(DATASETS, ALPHAS):
seq = [0.0, 0.25, 0.5, 0.75, 1.0]
for lo, hi in zip(seq, seq[1:]):
dv, dt = [], []
for seed in SEEDS:
a = by_key.get((dataset, alpha, seed, "support_weighted", lo))
b = by_key.get((dataset, alpha, seed, "support_weighted", hi))
if a and b:
dv.append((b["val_sel_per_client_rare_f1"] or 0) - (a["val_sel_per_client_rare_f1"] or 0))
dt.append((b["test_rare_f1_global"] or 0) - (a["test_rare_f1_global"] or 0))
steps.append({
"dataset": dataset, "alpha": alpha, "step": f"{lo}->{hi}",
"mean_d_val_sel": st.fmean(dv) if dv else None,
"mean_d_test_rare_f1": st.fmean(dt) if dt else None,
"per_seed_d_val_sel": "|".join(f"{x:.6f}" for x in dv),
"per_seed_d_test_rare_f1": "|".join(f"{x:.6f}" for x in dt),
"n": len(dv),
})
with (OUT / "phase2_beta_steps.csv").open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(steps[0].keys()))
w.writeheader()
w.writerows(steps)
# ---------- per-class test/val F1 deltas at beta=1 and selected beta ----------
# Selected beta per condition = argmax pooled val selection metric.
selected = {}
for dataset, alpha in product(DATASETS, ALPHAS):
cand = [p for p in pooled if p["dataset"] == dataset and p["alpha"] == alpha
and p["beta"] != ""]
if cand:
best = max(cand, key=lambda p: p["val_sel_mean"])
selected[(dataset, alpha)] = float(best["beta"])
class_rows = []
for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS):
base = runs.get((dataset, alpha, seed, None))
if not base:
continue
base_class = {int(r["class_id"]): r for r in (base.get("test_per_class") or [])}
base_vclass = {int(r["class_id"]): r for r in (base.get("val_per_class") or [])}
for beta in [1.0, selected.get((dataset, alpha))]:
cur = runs.get((dataset, alpha, seed, beta))
if not cur:
continue
cur_class = {int(r["class_id"]): r for r in (cur.get("test_per_class") or [])}
cur_vclass = {int(r["class_id"]): r for r in (cur.get("val_per_class") or [])}
for cid in sorted(base_class):
b = base_class[cid]
c = cur_class.get(cid)
if not c:
continue
bv = base_vclass.get(cid, {})
cv = cur_vclass.get(cid, {})
class_rows.append({
"dataset": dataset, "alpha": alpha, "seed": seed, "beta": beta,
"class_id": cid, "class_name": b.get("class_name"),
"support": b.get("support"),
"base_f1": fnum(b.get("f1")), "cur_f1": fnum(c.get("f1")),
"delta_f1": (fnum(c.get("f1")) or 0) - (fnum(b.get("f1")) or 0),
"base_recall": fnum(b.get("recall")), "cur_recall": fnum(c.get("recall")),
"delta_recall": (fnum(c.get("recall")) or 0) - (fnum(b.get("recall")) or 0),
"base_val_f1": fnum(bv.get("f1")), "cur_val_f1": fnum(cv.get("f1")),
"delta_val_f1": ((fnum(cv.get("f1")) or 0) - (fnum(bv.get("f1")) or 0))
if bv and cv else None,
})
with (OUT / "phase2_per_class_deltas.csv").open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(class_rows[0].keys()))
w.writeheader()
w.writerows(class_rows)
# pooled per-class delta
class_pooled = []
keys = sorted({(r["dataset"], r["alpha"], r["beta"], r["class_id"]) for r in class_rows})
for dataset, alpha, beta, cid in keys:
sub = [r for r in class_rows if r["dataset"] == dataset and r["alpha"] == alpha
and r["beta"] == beta and r["class_id"] == cid]
class_pooled.append({
"dataset": dataset, "alpha": alpha, "beta": beta, "class_id": cid,
"class_name": sub[0]["class_name"], "n_seeds": len(sub),
"mean_delta_test_f1": st.fmean([r["delta_f1"] for r in sub]),
"mean_delta_test_recall": st.fmean([r["delta_recall"] for r in sub]),
"mean_delta_val_f1": st.fmean([r["delta_val_f1"] for r in sub
if r["delta_val_f1"] is not None])
if any(r["delta_val_f1"] is not None for r in sub) else None,
"mean_test_support": st.fmean([float(r["support"]) for r in sub]),
})
with (OUT / "phase2_per_class_pooled.csv").open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(class_pooled[0].keys()))
w.writeheader()
w.writerows(class_pooled)
# ---------- per-client deltas (validation) at beta=1 ----------
client_rows = []
for dataset, alpha, seed in product(DATASETS, ALPHAS, SEEDS):
base = runs.get((dataset, alpha, seed, None))
cur = runs.get((dataset, alpha, seed, 1.0))
if not base or not cur:
continue
bmap = {int(r["client_id"]): r for r in (base.get("val_per_client") or [])}
for r in cur.get("val_per_client") or []:
cid = int(r["client_id"])
b = bmap.get(cid)
if not b:
continue
bv = fnum(b.get("rare_class_macro_f1"))
cv = fnum(r.get("rare_class_macro_f1"))
client_rows.append({
"dataset": dataset, "alpha": alpha, "seed": seed, "client_id": cid,
"n_samples": r.get("n_samples"),
"base_val_rare_f1": bv, "cur_val_rare_f1": cv,
"delta_val_rare_f1": (cv - bv) if bv is not None and cv is not None else None,
"base_val_rare_recall": fnum(b.get("rare_class_recall")),
"cur_val_rare_recall": fnum(r.get("rare_class_recall")),
})
with (OUT / "phase2_per_client_delta_beta1.csv").open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(client_rows[0].keys()))
w.writeheader()
w.writerows(client_rows)
# ---------- displacement table ----------
disp = {}
disp_path = REPO / "artifacts" / "analysis" / "final__prototype_displacement_by_beta.csv"
if disp_path.exists():
for r in read_csv(disp_path):
disp[(r["dataset"], float(r["alpha"]), float(r["beta"]))] = float(r["mean_displacement"])
# ---------- summary JSON ----------
def cond_mean(sel, key):
xs = [r[key] for r in rows if sel(r) and r[key] is not None]
return st.fmean(xs) if xs else None
worst = sorted(
[d for d in deltas if d["reference"] == "vs_original"],
key=lambda d: d["d_test_rare_f1"])[:10]
summary = {
"n_runs_loaded": len(runs),
"selected_beta_by_condition": {f"{k[0]}__a{k[1]}": v for k, v in selected.items()},
"pooled_pooled": pooled,
"largest_test_losses_vs_original": worst,
"displacement_beta1": {f"{k[0]}__a{k[1]}": v for k, v in disp.items() if k[2] == 1.0},
}
(OUT / "phase2_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(json.dumps({
"runs": len(runs),
"rows": len(rows),
"deltas": len(deltas),
"selected": summary["selected_beta_by_condition"],
}, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/pool_driver.py
+446
View File
@@ -0,0 +1,446 @@
#!/usr/bin/env python
"""Unattended driver for exploration_v2.
Phases: smoke -> screening -> rank (freeze) -> confirm -> analysis -> DONE.
Safety properties
-----------------
* one driver at a time (flock held by driver.sh)
* resumable from filesystem markers (COMPLETE / .failed / .running / scopes)
* retries a failed run once, from a clean directory
* concurrency <= 3 processes and <= 36 GiB of MemoryMax caps; <= 2 5G-NIDD
* heartbeat written every loop (target: < 1 minute old)
* STOP, failure-rate, budget and repeated-memory-kill stop rules from PLAN.md
* never kills a process it did not start; on resume it only observes scopes
"""
from __future__ import annotations
import csv
import fcntl
import hashlib
import json
import os
import shutil
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
EXPL = Path(__file__).resolve().parents[1]
RUNS = EXPL / "results" / "runs"
ATTEMPTS = EXPL / "results" / "attempts"
LOGS = EXPL / "logs"
STATE = EXPL / "state.json"
HEARTBEAT = EXPL / "HEARTBEAT"
STOP = EXPL / "STOP"
DONE = EXPL / "DONE"
STOP_REASON = EXPL / "STOP_REASON"
ISSUES = EXPL / "ISSUES.md"
BUDGET_START = EXPL / "BUDGET_START"
RUN_ONE = EXPL / "run_one_v2.sh"
PY = REPO / ".venv" / "bin" / "python"
MEM_CAP = {"xiiotid": 13, "5gnidd": 15} # GiB; X-IIoTID raised 11->13 after OOMs
MEM_TOTAL = 36
MAX_PROCS = 2
MAX_5GNIDD = 2
MIN_AVAILABLE_GIB = 12.0 # host-level guard before launching
BUDGET_HOURS = 96
def utc():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def now_epoch():
return time.time()
def unit_for(run_id: str) -> str:
h = hashlib.sha1(run_id.encode()).hexdigest()[:12]
return f"explorev2-{h}"
def scope_active(unit: str) -> bool:
try:
r = subprocess.run(["systemctl", "--user", "is-active", "--quiet", f"{unit}.scope"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20)
return r.returncode == 0
except Exception:
return False
def read_csv(path: Path):
if not path.is_file():
return []
with path.open(newline="", encoding="utf-8") as fh:
return list(csv.DictReader(fh))
def load_state():
if STATE.is_file():
try:
return json.loads(STATE.read_text(encoding="utf-8"))
except Exception:
pass
return {"phase": "smoke", "attempts": {}, "terminal_failed": [],
"max_procs": MAX_PROCS, "notes": [], "budget_start_utc": None,
"screening_started": False, "ranking_frozen": False}
def save_state(st):
tmp = STATE.with_suffix(".tmp")
tmp.write_text(json.dumps(st, indent=2, sort_keys=True) + "\n", encoding="utf-8")
os.replace(tmp, STATE)
def append_issue(msg: str):
with ISSUES.open("a", encoding="utf-8") as fh:
fh.write(f"\n## {utc()} — driver\n\n{msg}\n")
def run_dir(run_id: str) -> Path:
return RUNS / run_id
def primary_ok(rid: str) -> bool:
d = run_dir(rid)
res = d / "results" / rid
ana = d / "analysis" / f"{rid}__prototype_snapshot.json"
need = [
res / "metadata.json", res / "history.json",
res / "test_global_metrics.json", res / "test_confusion_matrix.json",
res / "test_per_class.csv", res / "val_global_metrics.json",
res / "val_selection.json", res / "per_client.csv",
res / "val_per_client.csv", res / "train_support_Nij.csv",
res / "characterization.csv", res / "rare_unseen.csv", ana,
]
return all(p.is_file() and p.stat().st_size > 0 for p in need)
def status_of(rid: str):
d = run_dir(rid)
if (d / "COMPLETE").is_file() and primary_ok(rid):
return "done"
if (d / ".failed").is_file():
return "failed"
if (d / ".running").is_file() or scope_active(unit_for(rid)):
return "running"
if d.is_dir():
return "incomplete"
return "queued"
def archive_attempt(rid: str, n: int):
src = run_dir(rid)
if not src.exists():
return
ATTEMPTS.mkdir(parents=True, exist_ok=True)
k = n
dst = ATTEMPTS / f"{rid}__attempt{k}"
while dst.exists():
k += 1
dst = ATTEMPTS / f"{rid}__attempt{k}"
shutil.move(str(src), str(dst))
def terminal(st: dict, rid: str) -> bool:
return status_of(rid) == "done" or rid in st["terminal_failed"]
def any_memory_kill_recent(st: dict, window: int = 2) -> bool:
recent = st.get("recent_exits", [])[-window:]
return len(recent) == window and all(x == 137 for x in recent)
def write_heartbeat(st: dict, running: dict, phase: str, note: str = ""):
HEARTBEAT.write_text(json.dumps({
"utc": utc(),
"epoch": round(now_epoch(), 1),
"pid": os.getpid(),
"phase": phase,
"running": sorted(running),
"running_count": len(running),
"max_procs": st.get("max_procs", MAX_PROCS),
"note": note,
}, indent=2) + "\n", encoding="utf-8")
def launch(row: dict) -> subprocess.Popen:
cmd = [
"bash", str(RUN_ONE),
row["run_id"], row["dataset"], str(row["alpha"]), str(row["seed"]),
row["mode"], str(row["beta"] or "0"),
str(row["shrink"] or "0"), str(row["cap"] or "0"),
str(row["minsup"] or "0"), row["part_dir"], row["label"],
]
LOGS.mkdir(parents=True, exist_ok=True)
logf = LOGS / f"{row['run_id']}.driver.log"
with logf.open("a") as fh:
fh.write(f"[{utc()}] launch: {' '.join(cmd)}\n")
fh.flush()
return subprocess.Popen(cmd, stdout=fh, stderr=subprocess.STDOUT,
cwd=str(REPO), start_new_session=True)
def mem_available_gib() -> float:
try:
with open("/proc/meminfo", encoding="ascii") as fh:
for line in fh:
if line.startswith("MemAvailable:"):
return int(line.split()[1]) / (1024 * 1024)
except Exception:
return 0.0
return 0.0
def can_launch(st: dict, running: dict, row: dict) -> bool:
if mem_available_gib() < MIN_AVAILABLE_GIB:
return False
if len(running) >= int(st.get("max_procs", MAX_PROCS)):
return False
ds = row["dataset"]
used = sum(MEM_CAP.get(r["dataset"], 11) for r in running.values())
if used + MEM_CAP.get(ds, 11) > MEM_TOTAL:
return False
if ds == "5gnidd":
if sum(1 for r in running.values() if r["dataset"] == "5gnidd") >= MAX_5GNIDD:
return False
return True
def prepare_retry(st: dict, rid: str) -> bool:
"""Archive a failed/incomplete attempt. Return True if a retry is allowed."""
n = int(st["attempts"].get(rid, 0))
if n >= 1:
if rid not in st["terminal_failed"]:
st["terminal_failed"].append(rid)
append_issue(f"run `{rid}` failed after {n + 1} attempt(s); marked terminal failed.")
return False
st["attempts"][rid] = n + 1
archive_attempt(rid, n + 1)
save_state(st)
return True
def run_script(name: str, *args) -> int:
cmd = [str(PY), str(EXPL / "scripts" / name), *args]
logf = LOGS / f"{name}.log"
with logf.open("a") as fh:
fh.write(f"\n[{utc()}] {' '.join(cmd)}\n")
fh.flush()
r = subprocess.run(cmd, stdout=fh, stderr=subprocess.STDOUT, cwd=str(REPO))
return r.returncode
def main() -> int:
EXPL.mkdir(parents=True, exist_ok=True)
st = load_state()
if STOP.is_file():
write_heartbeat(st, {}, "stopped", "STOP exists at startup")
append_issue("STOP exists at driver startup; no run launched.")
return 0
# smoke (re-run after the plan hash; non-scientific)
if not (EXPL / "smoke" / "SMOKE.json").is_file() or st.get("phase") == "smoke":
rc = run_script("smoke_test.py")
if rc != 0:
append_issue(f"smoke test failed (rc={rc}); see exploration_v2/smoke/SMOKE.json. Aborting.")
STOP_REASON.write_text(f"smoke failure at {utc()}\n", encoding="utf-8")
return 1
st["phase"] = "screening"
save_state(st)
running: dict[str, dict] = {}
procs: dict[str, subprocess.Popen] = {}
last_hb = 0.0
stall_loops = 0
while True:
if time.time() - last_hb >= 15:
write_heartbeat(st, running, st["phase"])
last_hb = time.time()
# observe finished children
for rid in list(procs):
p = procs[rid]
rc = p.poll()
if rc is not None:
del procs[rid]
running.pop(rid, None)
st.setdefault("recent_exits", []).append(rc)
st["recent_exits"] = st["recent_exits"][-4:]
save_state(st)
if rc != 0:
append_issue(f"run `{rid}` wrapper exit {rc}; see its logs.")
if rc == 137:
if any_memory_kill_recent(st):
old = int(st.get("max_procs", MAX_PROCS))
st["max_procs"] = max(1, old - 1)
append_issue(f"two consecutive exit-137 kills; lowering max_procs "
f"from {old} to {st['max_procs']} and continuing.")
save_state(st)
# observe resumable scopes (driver may have been restarted)
if st["phase"] in ("screening", "confirm"):
queue = (read_csv(EXPL / "queue_screen.csv") if st["phase"] == "screening"
else read_csv(EXPL / "queue_confirm.csv"))
for row in queue:
rid = row["run_id"]
if rid in procs:
continue
if scope_active(unit_for(rid)) and rid not in running:
running[rid] = row
# drop external runs whose scope has ended
for rid in list(running):
if rid in procs:
continue
if not scope_active(unit_for(rid)):
running.pop(rid, None)
# stop rules
stop_now = False
reason = ""
if STOP.is_file():
stop_now, reason = True, "STOP file exists"
elif st["phase"] in ("screening", "confirm"):
queue = (read_csv(EXPL / "queue_screen.csv") if st["phase"] == "screening"
else read_csv(EXPL / "queue_confirm.csv"))
finished = [r["run_id"] for r in queue if terminal(st, r["run_id"])]
failed = [r for r in finished if r in st["terminal_failed"]]
if len(finished) >= 5 and len(failed) / len(finished) > 0.20:
if any_memory_kill_recent(st):
append_issue("failure rate would stop the driver, but the last two "
"exits were memory kills; applying the memory throttle "
"instead of a full stop.")
else:
stop_now, reason = True, (f"failure rate {len(failed)}/{len(finished)} "
f"exceeds 20%")
if st.get("budget_start_utc"):
started = datetime.strptime(st["budget_start_utc"], "%Y-%m-%dT%H:%M:%SZ")
elapsed = (datetime.now(timezone.utc) - started.replace(tzinfo=timezone.utc)).total_seconds()
if elapsed > BUDGET_HOURS * 3600:
stop_now, reason = True, f"96 h budget exhausted ({elapsed/3600:.1f} h)"
if stop_now:
append_issue(f"stopping launches: {reason}")
STOP_REASON.write_text(f"{reason} at {utc()}\n", encoding="utf-8")
st["phase"] = "stopped"
save_state(st)
# phase transitions only when nothing is running
if not running and not procs:
if st["phase"] == "screening":
queue = read_csv(EXPL / "queue_screen.csv")
if all(terminal(st, r["run_id"]) for r in queue):
ready = [r for r in queue if status_of(r["run_id"]) == "done"]
labels = sorted({r["label"] for r in ready
if r["mode"] != "original_protean"})
if len(labels) < 3:
append_issue(f"only {len(labels)} screening variants completed; "
f"PLAN requires at least 3. Ranking not run.")
st["phase"] = "stopped"
STOP_REASON.write_text(f"fewer than 3 variants at {utc()}\n",
encoding="utf-8")
save_state(st)
continue
rc = run_script("rank_screening.py")
if rc != 0:
append_issue(f"rank_screening.py failed rc={rc}; aborting.")
st["phase"] = "stopped"
save_state(st)
continue
ranking = EXPL / "ranking.json"
h = hashlib.sha256(ranking.read_bytes()).hexdigest()
(EXPL / "ranking.sha256").write_text(
f"{h} ranking.json\n{utc()}\n", encoding="utf-8")
st["ranking_frozen"] = True
rc = run_script("make_confirm_queue.py")
if rc != 0:
append_issue(f"make_confirm_queue.py failed rc={rc}; aborting.")
st["phase"] = "stopped"
save_state(st)
continue
# new manifests for seeds 3,4
for ds, alpha in (("xiiotid", 0.25), ("5gnidd", 0.5)):
for seed in (3, 4):
rc = run_script("make_manifest_v2.py", "--dataset", ds,
"--alpha", str(alpha), "--seed", str(seed))
if rc != 0:
append_issue(f"manifest generation failed for {ds} "
f"alpha={alpha} seed={seed} rc={rc}")
st["phase"] = "stopped"
save_state(st)
break
st["phase"] = "confirm"
save_state(st)
continue
elif st["phase"] == "confirm":
queue = read_csv(EXPL / "queue_confirm.csv")
if all(terminal(st, r["run_id"]) for r in queue):
rc = run_script("analyze_results.py")
if rc != 0:
append_issue(f"analyze_results.py failed rc={rc}")
DONE.write_text(json.dumps({
"status": "complete" if rc == 0 else "analysis_failed",
"utc": utc(), "runs_done": sum(1 for r in queue if status_of(r["run_id"]) == "done"),
"runs_failed": len([r for r in queue if r["run_id"] in st["terminal_failed"]]),
}, indent=2) + "\n", encoding="utf-8")
st["phase"] = "done"
save_state(st)
write_heartbeat(st, running, "done")
return 0 if rc == 0 else 1
elif st["phase"] in ("stopped", "done"):
if st["phase"] == "stopped":
rc = run_script("analyze_results.py")
DONE.write_text(json.dumps({
"status": "stopped", "utc": utc(), "analysis_rc": rc,
"reason": (STOP_REASON.read_text(encoding="utf-8").strip()
if STOP_REASON.is_file() else "unknown"),
}, indent=2) + "\n", encoding="utf-8")
st["phase"] = "done"
save_state(st)
write_heartbeat(st, running, "done")
return 0
return 0
# launch queued runs
if st["phase"] in ("screening", "confirm") and not STOP.is_file():
queue = (read_csv(EXPL / "queue_screen.csv") if st["phase"] == "screening"
else read_csv(EXPL / "queue_confirm.csv"))
launched_any = False
for row in queue:
rid = row["run_id"]
if terminal(st, rid) or rid in running or rid in procs:
continue
s = status_of(rid)
if s == "failed" or s == "incomplete":
if not prepare_retry(st, rid):
continue
if not can_launch(st, running, row):
continue
if not st.get("budget_start_utc"):
st["budget_start_utc"] = utc()
BUDGET_START.write_text(st["budget_start_utc"] + "\n", encoding="utf-8")
save_state(st)
procs[rid] = launch(row)
running[rid] = row
launched_any = True
if len(running) >= int(st.get("max_procs", MAX_PROCS)):
break
if launched_any:
stall_loops = 0
elif not running:
stall_loops += 1
if stall_loops >= 6:
append_issue("driver found no launchable and no running runs for 60 s; "
"recording stall and exiting for resume.sh.")
write_heartbeat(st, running, st["phase"], "stalled")
return 2
time.sleep(10)
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/rank_screening.py
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python
"""Validation-only screening ranking for exploration_v2.
Reads ONLY ``val_*`` files. Every opened file is appended to
``exploration_v2/ACCESS_LOG.txt`` with a UTC timestamp.
Writes ``exploration_v2/ranking.json``. This file is frozen (sha256 + time in
``ranking.sha256``) before any confirmatory run starts.
Ranking metric (pre-registered in PLAN.md §5):
score(V) = mean over the two screening conditions of
[val_global_rare_f1(V) - val_global_rare_f1(original)]
where val_global_rare_f1 = rare_class_macro_f1 in val_global_metrics.json.
Secondary (sensitivity): the same difference using
val_rare_class_macro_f1_per_client_mean from val_selection.json.
"""
from __future__ import annotations
import csv
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
EXPL = REPO / "exploration_v2"
RUNS = EXPL / "results" / "runs"
ACCESS = EXPL / "ACCESS_LOG.txt"
SCREEN = EXPL / "queue_screen.csv"
OUT = EXPL / "ranking.json"
CONDITIONS = {
"S1": ("xiiotid", 0.25),
"S2": ("5gnidd", 0.5),
}
BETAS = ["0", "0.25", "0.5", "0.75", "1"]
def utc():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def log_access(path: Path):
with ACCESS.open("a", encoding="utf-8") as fh:
fh.write(f"{utc()}\t{path}\n")
def read_json_val(path: Path):
if not path.is_file():
return None
log_access(path)
with path.open(encoding="utf-8") as fh:
return json.load(fh)
def val_files(base: Path):
"""Return the two validation file paths for a run base directory."""
return (base / "val_global_metrics.json", base / "val_selection.json")
def metrics_for(val_global_path: Path, val_selection_path: Path):
vg = read_json_val(val_global_path)
vs = read_json_val(val_selection_path)
if vg is None and vs is None:
return None
return {
"val_global_rare_f1": (vg or {}).get("rare_class_macro_f1"),
"val_global_rare_recall": (vg or {}).get("rare_class_recall"),
"val_global_macro_f1": (vg or {}).get("macro_f1"),
"val_per_client_rare_f1": (vs or {}).get("val_rare_class_macro_f1_per_client_mean"),
"val_sel_file_present": vs is not None,
}
def main() -> int:
ACCESS.parent.mkdir(parents=True, exist_ok=True)
with ACCESS.open("a", encoding="utf-8") as fh:
fh.write(f"# ranking.py run at {utc()}\n")
# ---- original references ---------------------------------------------
# Primary: an original_protean run in the current environment/partition
# (added as amendment A2). Fallback: the restored grid.
original = {}
beta_refs = {}
baseline_source = {}
for cid, (ds, alpha) in CONDITIONS.items():
base_id = f"{ds}__alpha{alpha:g}__seed0__original_protean"
new_paths = val_files(RUNS / base_id / "results" / base_id)
old_paths = val_files(REPO / "results" / base_id)
if new_paths[0].is_file():
vg, vs = new_paths
source = "in_environment_seed0"
else:
vg, vs = old_paths
source = "restored_grid_seed0"
m = metrics_for(vg, vs)
if m is None or m["val_global_rare_f1"] is None:
raise SystemExit(f"missing original validation metrics for {cid} ({base_id})")
original[cid] = {"run_id": base_id, "source": source, **m}
baseline_source[cid] = source
beta_refs[cid] = {}
for b in BETAS:
rid = f"{ds}__alpha{alpha:g}__seed0__support_weighted__beta{b}"
# Prefer the current-environment 1-thread sweep (same host and
# library versions as the new runs) when it exists.
sweep_paths = val_files(REPO / "results" / "sweep" / "runs" / rid / "results" / rid)
grid_paths = val_files(REPO / "results" / rid)
if sweep_paths[0].is_file():
vg, vs = sweep_paths
src = "sweep_seed0_1thread"
else:
vg, vs = grid_paths
src = "restored_grid_seed0"
mm = metrics_for(vg, vs)
if mm is not None:
beta_refs[cid][b] = {"run_id": rid, "source": src, **mm}
# ---- screening candidates -------------------------------------------
log_access(SCREEN)
with SCREEN.open(newline="", encoding="utf-8") as fh:
queue = list(csv.DictReader(fh))
candidates = {}
for row in queue:
if row["mode"] == "original_protean":
continue
rid = row["run_id"]
vg, vs = val_files(RUNS / rid / "results" / rid)
m = metrics_for(vg, vs)
if m is None or m["val_global_rare_f1"] is None:
raise SystemExit(f"missing validation metrics for screening run {rid}")
candidates.setdefault(row["label"], {
"label": row["label"], "mode": row["mode"],
"params": {"beta": float(row["beta"]),
"shrink_lambda": float(row["shrink"]) if row["shrink"] else None,
"cap_factor": float(row["cap"]) if row["cap"] else None,
"min_support": float(row["minsup"]) if row["minsup"] else None},
"conditions": {},
})
candidates[row["label"]]["conditions"][
"S1" if row["dataset"] == "xiiotid" else "S2"] = {"run_id": rid, **m}
ranked = []
for label, item in candidates.items():
per = {}
deltas_global, deltas_client = [], []
for cid in CONDITIONS:
cond = item["conditions"].get(cid)
if cond is None:
raise SystemExit(f"{label}: missing condition {cid}")
base = original[cid]
dg = cond["val_global_rare_f1"] - base["val_global_rare_f1"]
dc = cond["val_per_client_rare_f1"] - base["val_per_client_rare_f1"]
per[cid] = {
"run_id": cond["run_id"],
"val_global_rare_f1": cond["val_global_rare_f1"],
"baseline_val_global_rare_f1": base["val_global_rare_f1"],
"delta_vs_original": dg,
"val_per_client_rare_f1": cond["val_per_client_rare_f1"],
"baseline_val_per_client_rare_f1": base["val_per_client_rare_f1"],
"delta_per_client_vs_original": dc,
}
deltas_global.append(dg)
deltas_client.append(dc)
item["per_condition"] = per
item["score"] = sum(deltas_global) / len(deltas_global)
item["min_condition_delta"] = min(deltas_global)
item["secondary_score_per_client"] = sum(deltas_client) / len(deltas_client)
ranked.append(item)
ranked.sort(key=lambda x: (-x["score"], -x["min_condition_delta"], x["label"]))
secondary = sorted(ranked, key=lambda x: (-x["secondary_score_per_client"],
x["label"]))
for i, item in enumerate(ranked, 1):
item["rank"] = i
for i, item in enumerate(secondary, 1):
item["rank_secondary"] = i
# ---- current-method reference per condition ------------------------
current_ref = {}
for cid in CONDITIONS:
by_global = max(beta_refs[cid].items(),
key=lambda kv: kv[1]["val_global_rare_f1"])
by_client = max(beta_refs[cid].items(),
key=lambda kv: kv[1]["val_per_client_rare_f1"])
current_ref[cid] = {
"original": original[cid],
"best_beta_by_val_global": {
"beta": float(by_global[0]),
"val_global_rare_f1": by_global[1]["val_global_rare_f1"],
"delta_vs_original": (by_global[1]["val_global_rare_f1"]
- original[cid]["val_global_rare_f1"]),
},
"best_beta_by_val_per_client": {
"beta": float(by_client[0]),
"val_per_client_rare_f1": by_client[1]["val_per_client_rare_f1"],
"delta_vs_original": (by_client[1]["val_per_client_rare_f1"]
- original[cid]["val_per_client_rare_f1"]),
},
"all_betas": beta_refs[cid],
}
top2 = [ranked[0]["label"], ranked[1]["label"]]
payload = {
"generated_utc": utc(),
"stage": "screening",
"ranking_metric": "mean over S1,S2 of val_global_rare_f1(candidate) - "
"val_global_rare_f1(original)",
"secondary_metric": "same with val_rare_class_macro_f1_per_client_mean",
"conditions": {k: {"dataset": v[0], "alpha": v[1]} for k, v in CONDITIONS.items()},
"screening_seed": 0,
"baseline_source": baseline_source,
"original": original,
"current_method_reference": current_ref,
"candidates": ranked,
"secondary_ranking_order": [x["label"] for x in secondary],
"top2": top2,
"selection_margin_top1_minus_top2": ranked[0]["score"] - ranked[1]["score"],
"note": "Validation only. No test metric of any new run was read before "
"this file was frozen.",
}
OUT.write_text(json.dumps(payload, indent=2, sort_keys=False) + "\n",
encoding="utf-8")
print(json.dumps({"top2": top2,
"ranking": [(x["label"], round(x["score"], 6)) for x in ranked],
"margin": payload["selection_margin_top1_minus_top2"]},
indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/run_record_v2.py
Executable
+303
View File
@@ -0,0 +1,303 @@
#!/usr/bin/env python3
"""Record one sweep run in summary.csv and write/validate COMPLETE markers.
Two subcommands:
record -- upsert one row in results/sweep/summary.csv (flock-protected)
marker -- validate the primary artifacts of a finished run and write COMPLETE
The summary is keyed by run config, so re-running a failed config replaces its
row instead of appending a duplicate. A completed row is never downgraded to
"skipped_complete" or "stopped".
"""
from __future__ import annotations
import argparse
import csv
import fcntl
import hashlib
import json
import os
import re
import sys
import tempfile
from datetime import datetime, timezone
from pathlib import Path
HEADER = [
"config",
"dataset",
"alpha",
"beta",
"seed",
"aggregation_mode",
"status",
"wall_s",
"cpu_user_s",
"cpu_sys_s",
"percent_cpu",
"peak_rss_kb",
"exit_code",
"started_utc",
"ended_utc",
"run_dir",
"log_dir",
"notes",
]
def utcnow() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _elapsed_to_seconds(value: str) -> float | None:
value = value.strip()
parts = value.split(":")
try:
if len(parts) == 3:
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
if len(parts) == 2:
return int(parts[0]) * 60 + float(parts[1])
if len(parts) == 1:
return float(parts[0])
except ValueError:
return None
return None
def parse_time_file(path: str | None) -> dict:
out = {
"wall_s": "",
"cpu_user_s": "",
"cpu_sys_s": "",
"percent_cpu": "",
"peak_rss_kb": "",
}
if not path:
return out
p = Path(path)
if not p.is_file() or p.stat().st_size == 0:
return out
text = p.read_text(encoding="utf-8", errors="replace")
patterns = {
"cpu_user_s": r"User time \(seconds\):\s*([0-9.]+)",
"cpu_sys_s": r"System time \(seconds\):\s*([0-9.]+)",
"percent_cpu": r"Percent of CPU this job got:\s*([0-9.]+)%",
"peak_rss_kb": r"Maximum resident set size \(kbytes\):\s*([0-9]+)",
}
for key, pat in patterns.items():
m = re.search(pat, text)
if m:
out[key] = m.group(1)
m = re.search(r"Elapsed \(wall clock\) time \(h:mm:ss or m:ss\):\s*(\S+)", text)
if m:
wall = _elapsed_to_seconds(m.group(1))
if wall is not None:
out["wall_s"] = f"{wall:.2f}"
return out
def _read_rows(path: Path) -> list[dict]:
if not path.is_file() or path.stat().st_size == 0:
return []
with path.open(newline="", encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
for row in rows:
for key in HEADER:
row.setdefault(key, "")
return rows
def _write_rows(path: Path, rows: list[dict]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(
dir=str(path.parent), prefix=".summary.", suffix=".csv", text=True
)
try:
with os.fdopen(fd, "w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=HEADER)
writer.writeheader()
for row in rows:
writer.writerow({key: row.get(key, "") for key in HEADER})
os.replace(tmp, path)
finally:
if os.path.exists(tmp):
os.unlink(tmp)
def upsert(summary: Path, lock: Path, row: dict) -> None:
summary.parent.mkdir(parents=True, exist_ok=True)
with lock.open("a+", encoding="utf-8") as handle:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
rows = _read_rows(summary)
index = next(
(i for i, item in enumerate(rows) if item.get("config") == row["config"]),
None,
)
if index is None:
rows.append({key: row.get(key, "") for key in HEADER})
else:
old = rows[index]
if old.get("status") == "completed" and row.get("status") in (
"skipped_complete",
"stopped",
):
pass # never downgrade a completed run
elif row.get("status") == "completed":
# A fresh completion replaces the whole row, clearing stale
# failure notes from an earlier attempt.
rows[index] = {key: row.get(key, "") for key in HEADER}
else:
for key in HEADER:
value = row.get(key, "")
if value not in ("", None) or key in ("status", "exit_code"):
old[key] = value
rows[index] = old
_write_rows(summary, rows)
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def required_files(run_dir: Path, run_name: str) -> list[Path]:
results = run_dir / "results" / run_name
analysis = run_dir / "analysis"
return [
results / "metadata.json",
results / "history.json",
results / "test_global_metrics.json",
results / "test_confusion_matrix.json",
results / "test_per_class.csv",
results / "val_global_metrics.json",
results / "val_selection.json",
results / "val_per_class.csv",
results / "per_client.csv",
results / "val_per_client.csv",
results / "train_support_Nij.csv",
results / "characterization.csv",
results / "rare_unseen.csv",
analysis / f"{run_name}__prototype_snapshot.json",
]
def cmd_record(args: argparse.Namespace) -> int:
timing = parse_time_file(args.time_file)
row = {
"config": args.run_name,
"dataset": args.dataset,
"alpha": args.alpha,
"beta": args.beta,
"seed": str(args.seed),
"aggregation_mode": args.aggregation_mode,
"status": args.status,
"wall_s": timing["wall_s"],
"cpu_user_s": timing["cpu_user_s"],
"cpu_sys_s": timing["cpu_sys_s"],
"percent_cpu": timing["percent_cpu"],
"peak_rss_kb": timing["peak_rss_kb"],
"exit_code": str(args.exit_code),
"started_utc": args.started or "",
"ended_utc": args.ended or utcnow(),
"run_dir": str(Path(args.run_dir)),
"log_dir": str(Path(args.log_dir)),
"notes": args.notes or "",
}
upsert(Path(args.summary), Path(args.lock), row)
return 0
def cmd_marker(args: argparse.Namespace) -> int:
run_dir = Path(args.run_dir)
missing = [
str(path)
for path in required_files(run_dir, args.run_name)
if not path.is_file() or path.stat().st_size == 0
]
if missing:
print("refusing to write COMPLETE; missing/empty artifacts:", file=sys.stderr)
for path in missing:
print(f" {path}", file=sys.stderr)
return 2
timing = parse_time_file(args.time_file)
primary = {}
for path in required_files(run_dir, args.run_name):
primary[str(path.relative_to(run_dir))] = sha256_file(path)
payload = {
"run_name": args.run_name,
"dataset": args.dataset,
"alpha": args.alpha,
"beta": args.beta,
"seed": args.seed,
"aggregation_mode": args.aggregation_mode,
"exit_code": args.exit_code,
"finished_utc": args.finished or utcnow(),
"wall_s": timing["wall_s"],
"cpu_user_s": timing["cpu_user_s"],
"cpu_sys_s": timing["cpu_sys_s"],
"percent_cpu": timing["percent_cpu"],
"peak_rss_kb": timing["peak_rss_kb"],
"required_files_sha256": primary,
}
marker = run_dir / "COMPLETE"
marker.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"wrote {marker}")
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="command", required=True)
rec = sub.add_parser("record")
rec.add_argument("--summary", required=True)
rec.add_argument("--lock", required=True)
rec.add_argument("--run-name", required=True)
rec.add_argument("--dataset", required=True)
rec.add_argument("--alpha", required=True)
rec.add_argument("--beta", required=True)
rec.add_argument("--seed", type=int, default=0)
rec.add_argument("--aggregation-mode", default="unknown")
rec.add_argument("--status", required=True)
rec.add_argument("--exit-code", type=int, required=True)
rec.add_argument("--time-file", default=None)
rec.add_argument("--started", default=None)
rec.add_argument("--ended", default=None)
rec.add_argument("--run-dir", required=True)
rec.add_argument("--log-dir", required=True)
rec.add_argument("--notes", default=None)
rec.set_defaults(func=cmd_record)
mk = sub.add_parser("marker")
mk.add_argument("--run-dir", required=True)
mk.add_argument("--run-name", required=True)
mk.add_argument("--dataset", required=True)
mk.add_argument("--alpha", required=True)
mk.add_argument("--beta", required=True)
mk.add_argument("--seed", type=int, default=0)
mk.add_argument("--aggregation-mode", default="unknown")
mk.add_argument("--exit-code", type=int, required=True)
mk.add_argument("--time-file", default=None)
mk.add_argument("--finished", default=None)
mk.set_defaults(func=cmd_marker)
return parser
def main(argv=None) -> int:
args = build_parser().parse_args(argv)
return args.func(args)
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/run_variant.py
+493
View File
@@ -0,0 +1,493 @@
#!/usr/bin/env python
"""exploration_v2 copy of scripts/run_baseline.py with aggregation variants.
This file is a *copy*; the original ``scripts/run_baseline.py`` is untouched.
Only three things differ:
1. the repository root is resolved two levels up (this copy lives in
``exploration_v2/scripts/``);
2. ``--aggregation-mode`` also accepts the exploration variants defined in
``exploration_v2/scripts/variants.py``, and their hyperparameters are
explicit CLI flags;
3. the server prototype aggregator is monkey-patched to the variant dispatcher
for the duration of this process. The baseline and ``support_weighted``
modes are delegated to the unmodified project implementation.
Every default output directory points inside ``exploration_v2/`` so that a
missing flag can never write into the tracked ``results/`` or
``artifacts/analysis/`` trees.
Usage (smoke)::
python exploration_v2/scripts/run_variant.py --dataset xiiotid --alpha 0.5 \
--seed 0 --max-rows 4000 --clients 4 --rounds 2 --local-epochs 1
"""
from __future__ import annotations
import argparse
import csv
import json
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO_ROOT / "src"))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from protean_research.data import manifest as manifest_mod # noqa: E402
from protean_research.data import partition as partition_mod # noqa: E402
from protean_research.data import rare_unseen # noqa: E402
from protean_research.aggregation import validate_beta # noqa: E402
from protean_research.data.characterization import characterize_support # noqa: E402
from protean_research.data.pipeline import ( # noqa: E402
create_partition,
prepare_dataset,
preprocessing_content_hash,
)
from protean_research.evaluation.metrics import ( # noqa: E402
classification_metrics,
rare_class_metrics,
)
from protean_research.federated import TrainingConfig, run_federated # noqa: E402
from protean_research.models.protean_cnn import extract_embeddings # noqa: E402
from protean_research.models.prototypes import nearest_prototype_predict # noqa: E402
from protean_research.utils.config import ( # noqa: E402
expand_path,
load_datasets_config,
load_yaml,
)
import variants as variants_mod # noqa: E402
MODE_CHOICES = list(variants_mod.ALL_MODES)
def parse_args(argv=None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dataset", required=True)
parser.add_argument("--alpha", type=float, required=True)
parser.add_argument("--seed", type=int, default=0)
parser.add_argument("--clients", type=int, default=10)
parser.add_argument("--rounds", type=int, default=10)
parser.add_argument("--local-epochs", type=int, default=3)
parser.add_argument("--batch-size", type=int, default=256)
parser.add_argument("--lr", type=float, default=1e-3)
parser.add_argument("--mu", type=float, default=0.1)
parser.add_argument("--lam", type=float, default=1.0)
parser.add_argument(
"--aggregation-mode",
choices=MODE_CHOICES,
default="original_protean",
help="server-side prototype aggregation mode (default: frozen baseline).",
)
parser.add_argument(
"--beta",
type=float,
default=None,
help="support-weighting exponent; required for every weighted mode.",
)
parser.add_argument("--shrink-lambda", type=float, default=0.5,
help="v_shrink: weight on the support-weighted vector (rest goes to equal).")
parser.add_argument("--cap-factor", type=float, default=4.0,
help="v_cap: cap each raw weight at cap_factor * median raw weight.")
parser.add_argument("--min-support", type=float, default=10.0,
help="v_minsup: drop contributors with N_ij below this threshold.")
parser.add_argument("--variant-label", default=None,
help="short label used in the run directory name.")
parser.add_argument(
"--max-rows",
type=int,
default=None,
help="SMOKE ONLY: limit how many raw rows are read; never use for science.",
)
parser.add_argument("--sample-from", type=int, default=200000,
help="SMOKE ONLY: rows to read before random subsampling.")
parser.add_argument("--partitions-dir", default=None)
parser.add_argument("--processed-dir", default=None)
parser.add_argument("--results-dir", default=None)
parser.add_argument("--analysis-dir", default=None)
return parser.parse_args(argv)
def write_json(path: Path, payload) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2, sort_keys=True)
handle.write("\n")
def write_csv(path: Path, rows: list[dict]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if not rows:
path.write_text("", encoding="utf-8")
return
fieldnames = list(rows[0].keys())
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
def main(argv=None) -> int:
args = parse_args(argv)
if args.aggregation_mode == "original_protean":
if args.beta is not None:
raise SystemExit(
"--beta is only valid with a weighted aggregation mode"
)
beta = None
else:
if args.beta is None:
raise SystemExit(
"--beta is required with a weighted aggregation mode"
)
raw_beta = float(args.beta)
if not (raw_beta == raw_beta) or raw_beta in (float("inf"), float("-inf")):
raise SystemExit(f"--beta must be finite, got {args.beta!r}")
if args.aggregation_mode == "v_beta":
if raw_beta < 0.0 or raw_beta > 2.0:
raise SystemExit("v_beta requires 0 <= beta <= 2")
beta = raw_beta
else:
beta = validate_beta(raw_beta)
if args.shrink_lambda < 0.0 or args.shrink_lambda > 1.0:
raise SystemExit("--shrink-lambda must lie in [0, 1]")
if args.cap_factor < 1.0:
raise SystemExit("--cap-factor must be >= 1")
# Patch the server aggregator for this process only. The project modules on
# disk are not modified. Baseline and support_weighted delegate inside
# variants.aggregate_variant to the unmodified project implementation.
from protean_research.federated import server as server_mod
def _variant_aggregator(client_prototypes, client_supports, num_classes,
mode=None, beta=None):
return variants_mod.aggregate_from_client_dicts(
client_prototypes,
client_supports,
num_classes,
mode=mode,
beta=beta,
shrink_lambda=args.shrink_lambda,
cap_factor=args.cap_factor,
min_support=args.min_support,
)
server_mod.aggregate_prototypes = _variant_aggregator
datasets_cfg = load_datasets_config()
protocol = load_yaml(REPO_ROOT / "configs" / "experiments" / "experiment_protocol.yaml")
artifacts_cfg = protocol.get("artifacts", {})
# Safe defaults: never write into the tracked trees. Callers pass explicit
# directories for scientific runs.
partitions_dir = expand_path(
args.partitions_dir or REPO_ROOT / "exploration_v2" / "partitions")
processed_dir = expand_path(
args.processed_dir or REPO_ROOT / "exploration_v2" / "processed")
analysis_dir = expand_path(
args.analysis_dir or REPO_ROOT / "exploration_v2" / "analysis")
results_root = expand_path(
args.results_dir or REPO_ROOT / "exploration_v2" / "results")
# Smoke runs must never pollute the tracked partition/analysis directories.
if args.max_rows is not None:
if args.partitions_dir is None:
partitions_dir = processed_dir / "smoke" / "partitions"
if args.analysis_dir is None:
analysis_dir = processed_dir / "smoke" / "analysis"
if args.results_dir is None:
results_root = results_root / "smoke"
if args.max_rows is not None:
from protean_research.data import loading as loading_mod
import numpy as np
spec = loading_mod.load_spec(args.dataset, datasets_cfg)
pool = loading_mod.read_raw(spec, nrows=args.sample_from)
rng = np.random.default_rng(args.seed)
take = min(args.max_rows, len(pool))
chosen = np.sort(rng.choice(len(pool), size=take, replace=False))
pool = pool.iloc[chosen].reset_index(drop=True)
bundle = prepare_dataset(
args.dataset, seed=args.seed, datasets_cfg=datasets_cfg, raw_frame=pool
)
else:
bundle = prepare_dataset(args.dataset, seed=args.seed, datasets_cfg=datasets_cfg)
# ---- partition (create or reload) -------------------------------------
pre_hash = preprocessing_content_hash(bundle.preprocessor)
manifest_file = manifest_mod.manifest_path(partitions_dir, args.dataset, args.alpha, args.seed)
if manifest_file.exists():
manifest = manifest_mod.load_manifest(manifest_file)
split, assignments, support = manifest_mod.reproduce_train_assignments(manifest, bundle.labels)
# Reuse the manifest's split for pairing; verify it matches the bundle.
bundle.split = split
else:
manifest, assignments, support = create_partition(
bundle, alpha=args.alpha, seed=args.seed, num_clients=args.clients,
partition_dir=partitions_dir, preprocessing_hash=pre_hash,
)
manifest_reload_ok = True
try:
manifest_mod.reproduce_train_assignments(manifest, bundle.labels)
except Exception: # pragma: no cover - surfaced in metadata
manifest_reload_ok = False
labels = bundle.labels
train_features = bundle.features[bundle.split.train]
train_labels = labels[bundle.split.train]
val_features = bundle.features[bundle.split.validation]
val_labels = labels[bundle.split.validation]
test_features = bundle.features[bundle.split.test]
test_labels = labels[bundle.split.test]
benign_index = bundle.cleaned.benign_index
per_client_rare = rare_unseen.rare_class_ids(support, benign_index)
union_rare = sorted({c for row in per_client_rare for c in row})
cfg = TrainingConfig(
rounds=args.rounds,
local_epochs=args.local_epochs,
batch_size=args.batch_size,
learning_rate=args.lr,
mu=args.mu,
lam=args.lam,
num_clients=args.clients,
aggregation_mode=args.aggregation_mode,
beta=beta,
)
result = run_federated(
train_features=train_features,
train_labels=train_labels,
assignments=assignments,
val_features=val_features,
val_labels=val_labels,
num_classes=bundle.num_classes,
input_dim=bundle.features.shape[1],
config=cfg,
init_seed_parts=(args.dataset, f"seed={args.seed}"),
rare_class_ids=union_rare,
)
model = result["model"]
global_prototypes = result["global_prototypes"]
# ---- global test evaluation ------------------------------------------
test_embeddings = extract_embeddings(model, test_features)
test_pred = nearest_prototype_predict(test_embeddings, global_prototypes)
test_metrics = classification_metrics(test_labels, test_pred, bundle.num_classes)
test_rare = rare_class_metrics(test_labels, test_pred, bundle.num_classes, union_rare)
# ---- per-client evaluation -------------------------------------------
eval_assignments, eval_rng = partition_mod.partition_eval_split(
test_labels, args.clients, args.alpha, args.dataset, args.seed, "test"
)
per_client_rows = []
for client_id in range(args.clients):
idx = eval_assignments[client_id]
if len(idx) == 0:
per_client_rows.append({
"client_id": client_id, "n_samples": 0, "accuracy": None,
"macro_f1": None, "rare_class_macro_f1": None, "rare_class_recall": None,
})
continue
emb = extract_embeddings(model, test_features[idx])
pred = nearest_prototype_predict(emb, global_prototypes)
metrics = classification_metrics(test_labels[idx], pred, bundle.num_classes)
rare_ids = per_client_rare[client_id]
if rare_ids:
rare = rare_class_metrics(test_labels[idx], pred, bundle.num_classes, rare_ids)
else:
rare = {"rare_class_macro_f1": None, "rare_class_recall": None}
per_client_rows.append({
"client_id": client_id,
"n_samples": int(len(idx)),
"accuracy": metrics["accuracy"],
"macro_f1": metrics["macro_f1"],
"rare_class_macro_f1": rare["rare_class_macro_f1"],
"rare_class_recall": rare["rare_class_recall"],
})
# ---- global validation evaluation (SELECTION ONLY) -------------------
# This block never influences training and is never used to tune any
# hyperparameter except the prototype-aggregation beta, which is selected
# on validation metrics only. The test split is untouched here.
val_embeddings = extract_embeddings(model, val_features)
val_pred = nearest_prototype_predict(val_embeddings, global_prototypes)
val_metrics = classification_metrics(val_labels, val_pred, bundle.num_classes)
val_rare = rare_class_metrics(val_labels, val_pred, bundle.num_classes, union_rare)
val_eval_assignments, _ = partition_mod.partition_eval_split(
val_labels, args.clients, args.alpha, args.dataset, args.seed, "validation"
)
val_per_client_rows = []
for client_id in range(args.clients):
idx = val_eval_assignments[client_id]
if len(idx) == 0:
val_per_client_rows.append({
"client_id": client_id, "n_samples": 0, "accuracy": None,
"macro_f1": None, "rare_class_macro_f1": None, "rare_class_recall": None,
})
continue
emb = extract_embeddings(model, val_features[idx])
pred = nearest_prototype_predict(emb, global_prototypes)
metrics = classification_metrics(val_labels[idx], pred, bundle.num_classes)
rare_ids = per_client_rare[client_id]
if rare_ids:
rare = rare_class_metrics(val_labels[idx], pred, bundle.num_classes, rare_ids)
else:
rare = {"rare_class_macro_f1": None, "rare_class_recall": None}
val_per_client_rows.append({
"client_id": client_id,
"n_samples": int(len(idx)),
"accuracy": metrics["accuracy"],
"macro_f1": metrics["macro_f1"],
"rare_class_macro_f1": rare["rare_class_macro_f1"],
"rare_class_recall": rare["rare_class_recall"],
})
def _mean_over_present(rows, key):
values = [float(row[key]) for row in rows if row.get(key) is not None]
return float(sum(values) / len(values)) if values else None
val_selection = {
"val_rare_class_macro_f1_global": val_rare["rare_class_macro_f1"],
"val_rare_class_recall_global": val_rare["rare_class_recall"],
"val_macro_f1_global": val_metrics["macro_f1"],
"val_weighted_f1_global": val_metrics["weighted_f1"],
"val_accuracy_global": val_metrics["accuracy"],
"val_rare_class_macro_f1_per_client_mean": _mean_over_present(
val_per_client_rows, "rare_class_macro_f1"
),
"val_rare_class_recall_per_client_mean": _mean_over_present(
val_per_client_rows, "rare_class_recall"
),
"val_n_eval_clients": sum(1 for row in val_per_client_rows if row["n_samples"] > 0),
"selection_metric": "rare_class_macro_f1_per_client_mean",
"selection_split": "validation",
"test_used_for_selection": False,
}
# ---- structured artifacts --------------------------------------------
mode_tag = args.aggregation_mode
if args.variant_label:
mode_tag = args.variant_label
elif args.aggregation_mode == "support_weighted":
mode_tag = f"support_weighted__beta{beta:g}"
elif args.aggregation_mode != "original_protean":
if args.variant_label:
mode_tag = args.variant_label
elif args.aggregation_mode == "v_beta":
mode_tag = f"v_beta__b{beta:g}"
elif args.aggregation_mode == "v_shrink":
mode_tag = f"v_shrink__b{beta:g}__lam{args.shrink_lambda:g}"
elif args.aggregation_mode == "v_cap":
mode_tag = f"v_cap__b{beta:g}__c{args.cap_factor:g}"
elif args.aggregation_mode == "v_minsup":
mode_tag = f"v_minsup__t{args.min_support:g}"
elif args.aggregation_mode == "v_logweight":
mode_tag = "v_logweight"
else: # pragma: no cover
mode_tag = args.aggregation_mode
run_name = f"{args.dataset}__alpha{args.alpha:g}__seed{args.seed}__{mode_tag}"
run_dir = results_root / run_name
run_dir.mkdir(parents=True, exist_ok=True)
metadata = {
"run_name": run_name,
"dataset": args.dataset,
"alpha": args.alpha,
"seed": args.seed,
"num_clients": args.clients,
"num_classes": bundle.num_classes,
"class_names": bundle.class_names,
"benign_class": bundle.spec.benign_label,
"aggregation_mode": args.aggregation_mode,
"beta": beta,
"variant": {
"label": mode_tag,
"shrink_lambda": args.shrink_lambda
if args.aggregation_mode == "v_shrink" else None,
"cap_factor": args.cap_factor
if args.aggregation_mode == "v_cap" else None,
"min_support": args.min_support
if args.aggregation_mode == "v_minsup" else None,
},
"training": {
"rounds": args.rounds,
"local_epochs": args.local_epochs,
"batch_size": args.batch_size,
"learning_rate": args.lr,
"mu": args.mu,
"lambda": args.lam,
},
"smoke": args.max_rows is not None,
"max_rows": args.max_rows,
"manifest_path": str(manifest_file),
"manifest_content_hash": manifest_mod.manifest_content_hash(manifest),
"preprocessing_hash": pre_hash,
"manifest_reload_ok": manifest_reload_ok,
"repo_root": str(REPO_ROOT),
}
write_json(run_dir / "metadata.json", metadata)
write_json(run_dir / "history.json", result["history"])
write_json(run_dir / "test_global_metrics.json", {
"accuracy": test_metrics["accuracy"],
"macro_f1": test_metrics["macro_f1"],
"weighted_f1": test_metrics["weighted_f1"],
**test_rare,
})
write_json(run_dir / "test_confusion_matrix.json", test_metrics["confusion_matrix"])
write_csv(run_dir / "test_per_class.csv", [
{**row, "class_name": bundle.class_names[row["class_id"]]}
for row in test_metrics["per_class"]
])
write_csv(run_dir / "per_client.csv", per_client_rows)
write_json(run_dir / "val_global_metrics.json", {
"accuracy": val_metrics["accuracy"],
"macro_f1": val_metrics["macro_f1"],
"weighted_f1": val_metrics["weighted_f1"],
**val_rare,
})
write_json(run_dir / "val_confusion_matrix.json", val_metrics["confusion_matrix"])
write_csv(run_dir / "val_per_class.csv", [
{**row, "class_name": bundle.class_names[row["class_id"]]}
for row in val_metrics["per_class"]
])
write_csv(run_dir / "val_per_client.csv", val_per_client_rows)
write_json(run_dir / "val_selection.json", val_selection)
write_csv(run_dir / "train_support_Nij.csv", [
{"client_id": i, **{bundle.class_names[c]: int(support[i, c]) for c in range(bundle.num_classes)}}
for i in range(args.clients)
])
write_csv(run_dir / "characterization.csv", characterize_support(support, bundle.class_names))
write_csv(run_dir / "rare_unseen.csv", rare_unseen.rare_unseen_rows(
support, bundle.class_names, benign_index, args.dataset, args.alpha, args.seed
))
processed_dir.mkdir(parents=True, exist_ok=True)
bundle.preprocessor.save(processed_dir / f"{args.dataset}__seed{args.seed}__preprocessor.json")
snapshot = result["snapshot"]
if snapshot is not None:
write_json(analysis_dir / f"{run_name}__prototype_snapshot.json", snapshot)
summary = {
"run": run_name,
"test_accuracy": test_metrics["accuracy"],
"test_macro_f1": test_metrics["macro_f1"],
"test_rare_class_macro_f1": test_rare["rare_class_macro_f1"],
"n_global_prototypes": len(global_prototypes),
"history_last": result["history"][-1] if result["history"] else None,
"results_dir": str(run_dir),
}
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/smoke_test.py
+140
View File
@@ -0,0 +1,140 @@
#!/usr/bin/env python
"""Very short smoke slice of every aggregation variant.
Checks, in order:
1. the exploration unit tests pass (``exploration_v2/tests``);
2. the baseline run completes and writes validation artifacts;
3. ``support_weighted`` beta=0 and ``v_beta`` beta=0 reproduce the baseline
validation metric within a documented smoke tolerance;
4. every candidate variant completes.
This is a runtime sanity check, not a scientific comparison. It writes only
inside ``exploration_v2/smoke``.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
HERE = REPO / "exploration_v2"
SMOKE = HERE / "smoke"
PY = REPO / ".venv" / "bin" / "python"
RUNNER = HERE / "scripts" / "run_variant.py"
TOL = 1e-3
VARIANTS = [
# id, mode, beta, extra args, label
("base", "original_protean", None, [], "smoke_base"),
("beta0", "support_weighted", 0.0, [], "smoke_beta0"),
("beta1", "support_weighted", 1.0, [], "smoke_beta1"),
("v_beta1.5", "v_beta", 1.5, [], "smoke_v_beta1.5"),
("v_shrink", "v_shrink", 1.0, ["--shrink-lambda", "0.5"], "smoke_v_shrink"),
("v_cap4", "v_cap", 1.0, ["--cap-factor", "4"], "smoke_v_cap4"),
("v_cap2", "v_cap", 1.0, ["--cap-factor", "2"], "smoke_v_cap2"),
("v_minsup", "v_minsup", 0.0, ["--min-support", "10"], "smoke_v_minsup"),
("v_logweight", "v_logweight", 0.0, [], "smoke_v_logweight"),
]
def utc():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def run_one(name, mode, beta, extra, label):
cmd = [
str(PY), str(RUNNER),
"--dataset", "xiiotid", "--alpha", "0.25", "--seed", "0",
"--max-rows", "3000", "--sample-from", "30000",
"--clients", "4", "--rounds", "2", "--local-epochs", "1",
"--aggregation-mode", mode, "--variant-label", label,
"--partitions-dir", str(SMOKE / "partitions"),
"--processed-dir", str(SMOKE / "processed"),
"--analysis-dir", str(SMOKE / "analysis"),
"--results-dir", str(SMOKE / "results"),
]
if beta is not None:
cmd[cmd.index("--aggregation-mode"):cmd.index("--aggregation-mode")] = ["--beta", str(beta)]
cmd += extra
env = dict(os.environ)
for k in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
"NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"):
env[k] = "1"
env["PYTHONDONTWRITEBYTECODE"] = "1"
log = SMOKE / "logs"
log.mkdir(parents=True, exist_ok=True)
out = log / f"{name}.stdout.log"
err = log / f"{name}.stderr.log"
with out.open("w") as fo, err.open("w") as fe:
proc = subprocess.run(cmd, stdout=fo, stderr=fe, env=env, timeout=1800)
return proc.returncode, cmd
def val_metric(result_dir: Path):
p = result_dir / "val_global_metrics.json"
if not p.exists():
return None
return float(json.loads(p.read_text())["rare_class_macro_f1"])
def main() -> int:
SMOKE.mkdir(parents=True, exist_ok=True)
report = {"started_utc": utc(), "tolerance": TOL, "runs": {}}
# 1. unit tests
unittest = subprocess.run(
[str(PY), "-m", "pytest", str(HERE / "tests"), "-q", "-p", "no:cacheprovider"],
cwd=str(REPO), capture_output=True, text=True,
env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, timeout=600)
report["pytest_returncode"] = unittest.returncode
report["pytest_tail"] = (unittest.stdout + unittest.stderr).strip().splitlines()[-3:]
if unittest.returncode != 0:
report["verdict"] = "FAIL_UNIT_TESTS"
(SMOKE / "SMOKE.json").write_text(json.dumps(report, indent=2))
return 1
metrics = {}
for name, mode, beta, extra, label in VARIANTS:
rc, cmd = run_one(name, mode, beta, extra, label)
rdir = SMOKE / "results" / f"xiiotid__alpha0.25__seed0__{label}"
metric = val_metric(rdir)
metrics[name] = metric
report["runs"][name] = {
"mode": mode, "beta": beta, "extra": extra,
"returncode": rc, "val_rare_macro_f1": metric,
"results_dir": str(rdir.relative_to(REPO)),
}
if rc != 0 or metric is None:
report["verdict"] = f"FAIL_RUN_{name}"
report["finished_utc"] = utc()
(SMOKE / "SMOKE.json").write_text(json.dumps(report, indent=2))
return 1
base = metrics["base"]
checks = {}
for name in ("beta0", "v_beta1.5"):
# v_beta1.5 is not expected to equal baseline; only beta0 equivalence is checked.
if name == "beta0":
checks[name] = abs(metrics[name] - base)
# v_beta at beta=0 is exercised by the unit tests; here we use beta1.5.
checks["beta0_within_tolerance"] = checks["beta0"] <= TOL
report["equivalence_checks"] = checks
if not checks["beta0_within_tolerance"]:
report["verdict"] = "FAIL_BETA0_EQUIVALENCE"
else:
report["verdict"] = "PASS"
report["finished_utc"] = utc()
(SMOKE / "SMOKE.json").write_text(json.dumps(report, indent=2))
print(json.dumps({"verdict": report["verdict"],
"beta0_abs_diff": checks["beta0"]}, indent=2))
return 0 if report["verdict"] == "PASS" else 1
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/status_table.py
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python
"""Fast status table for exploration_v2 (returns immediately)."""
from __future__ import annotations
import csv
import json
from datetime import datetime, timezone
from pathlib import Path
EXPL = Path(__file__).resolve().parents[1]
RUNS = EXPL / "results" / "runs"
def read_csv(p):
if not p.is_file():
return []
with p.open(newline="", encoding="utf-8") as fh:
return list(csv.DictReader(fh))
def status(rid):
d = RUNS / rid
if (d / "COMPLETE").is_file():
return "done"
if (d / ".failed").is_file():
return "failed"
if (d / ".running").is_file():
return "running"
if d.is_dir():
return "incomplete"
return "queued"
def main():
st = {}
if (EXPL / "state.json").is_file():
st = json.loads((EXPL / "state.json").read_text())
hb = ""
if (EXPL / "HEARTBEAT").is_file():
try:
hb = json.loads((EXPL / "HEARTBEAT").read_text())
except Exception:
hb = {}
queue = read_csv(EXPL / "queue_screen.csv")
phase = st.get("phase", "?")
if phase in ("confirm", "done", "stopped"):
queue = read_csv(EXPL / "queue_confirm.csv") or queue
rows = []
for r in queue:
rows.append((r["run_id"], r["dataset"], r.get("label", ""), status(r["run_id"])))
counts = {}
for _, _, _, s in rows:
counts[s] = counts.get(s, 0) + 1
print(f"phase={phase} budget_start={st.get('budget_start_utc')} "
f"max_procs={st.get('max_procs')}")
print(f"heartbeat={(hb or {}).get('utc', 'MISSING')} "
f"running={(hb or {}).get('running_count', 0)} "
f"note={(hb or {}).get('note', '')}")
print(f"terminal_failed={len(st.get('terminal_failed', []))} "
f"DONE={str((EXPL / 'DONE').is_file()).lower()} "
f"STOP={str((EXPL / 'STOP').is_file()).lower()}")
print("counts: " + " ".join(f"{k}={v}" for k, v in sorted(counts.items())))
print()
print(f"{'run_id':<62} {'dataset':<8} {'label':<28} status")
for rid, ds, label, s in rows:
print(f"{rid:<62} {ds:<8} {label:<28} {s}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/scripts/thread1_launcher.py
Executable
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""Force a single compute thread, then execute scripts/run_baseline.py.
The sweep design is one process = one thread. Environment variables
(OMP_NUM_THREADS etc.) are set by the caller, but PyTorch's intra-op and
inter-op pools are also pinned explicitly here so that no hidden pool can
create additional worker threads.
Usage:
python thread1_launcher.py /path/to/scripts/run_baseline.py [args...]
"""
from __future__ import annotations
import runpy
import sys
import torch
torch.set_num_threads(1)
try:
torch.set_num_interop_threads(1)
except RuntimeError:
# Already initialised (should not happen before any parallel work).
pass
if len(sys.argv) < 2:
raise SystemExit("usage: thread1_launcher.py SCRIPT [args...]")
script = sys.argv[1]
sys.argv = [script] + sys.argv[2:]
runpy.run_path(script, run_name="__main__")
exploration_v2/scripts/variants.py
+161
View File
@@ -0,0 +1,161 @@
"""Candidate server-side prototype aggregation variants (exploration_v2 only).
Every variant operates on the same contributor set semantics as the frozen
project code:
S_j = { i | N_ij > 0 and a valid local prototype C_ij exists }
except ``v_minsup`` which deliberately changes the participant rule and is
labelled as such. Missing classes are never fabricated.
The baseline (``original_protean``) and the current proposed method
(``support_weighted``) are *delegated* to the unmodified project implementation
so that their behaviour cannot drift. Only genuinely new modes are implemented
here.
Mode catalogue
--------------
original_protean equal mean over S_j (delegated, unchanged)
support_weighted w_i proportional to N_ij**beta in [0, 1] (delegated)
v_beta w_i proportional to N_ij**beta, beta allowed in [0, 2]
(extended-grid candidate; beta > 1 is outside the frozen grid)
v_shrink p_i = N_ij**beta / sum(N_ij**beta);
w_i = (1 - lam) / |S_j| + lam * p_i
(shrinkage of the weight vector toward equal weighting)
v_cap w_i proportional to min(N_ij**beta, c * median(N_ij**beta))
(clipping/capping of the weight vector)
v_minsup drop contributors with N_ij < t, then equal mean over the
remaining contributors (participant-rule change)
v_logweight w_i proportional to log1p(N_ij), normalised
(a different, concave weight function)
"""
from __future__ import annotations
import math
from typing import Mapping, Sequence
import numpy as np
from protean_research.aggregation import aggregate_class_prototypes
BASELINE_MODES = ("original_protean", "support_weighted")
VARIANT_MODES = ("v_beta", "v_shrink", "v_cap", "v_minsup", "v_logweight")
ALL_MODES = BASELINE_MODES + VARIANT_MODES
def _collect(contributions):
"""Return class_id -> (list[vector ndarray], list[support float])."""
out = {}
for class_id in sorted(contributions, key=lambda c: int(c)):
protos, supports = [], []
for proto, support in contributions[class_id]:
if proto is None:
continue
if support is None or float(support) <= 0.0:
continue
protos.append(np.asarray(proto, dtype=np.float64))
supports.append(float(support))
if protos:
out[int(class_id)] = (protos, supports)
return out
def _weighted_mean(protos, weights):
w = np.asarray(weights, dtype=np.float64)
total = float(w.sum())
if not math.isfinite(total) or total <= 0.0:
raise ValueError("weights must sum to a positive finite value")
w = w / total
mat = np.stack(protos)
return (w[:, None] * mat).sum(axis=0)
def _normalised_power(supports, beta):
n = np.asarray(supports, dtype=np.float64)
if np.any(~np.isfinite(n)) or np.any(n <= 0):
raise ValueError("supports must be strictly positive and finite")
w = n ** float(beta)
return w / w.sum()
def aggregate_variant(
contributions: Mapping[int, Sequence],
mode,
*,
beta=None,
shrink_lambda: float = 0.5,
cap_factor: float = 4.0,
min_support: float = 10.0,
):
"""Aggregate one round's per-class contributions under *mode*.
``contributions`` maps ``class_id -> [(C_ij, N_ij), ...]`` exactly like
:func:`protean_research.aggregation.aggregate_class_prototypes`.
"""
if mode in BASELINE_MODES:
# Unchanged project behaviour; beta semantics are also the project's.
return aggregate_class_prototypes(contributions, mode, beta=beta)
if mode not in VARIANT_MODES:
raise ValueError(f"unknown aggregation mode {mode!r}")
if beta is None:
raise ValueError(f"{mode} requires beta")
beta = float(beta)
if not math.isfinite(beta) or beta < 0.0:
raise ValueError(f"beta must be finite and >= 0, got {beta}")
collected = _collect(contributions)
result = {}
for class_id, (protos, supports) in collected.items():
n = np.asarray(supports, dtype=np.float64)
if mode == "v_beta":
if beta > 2.0:
raise ValueError("v_beta beta must be <= 2.0")
w = n ** beta
elif mode == "v_shrink":
if not 0.0 <= shrink_lambda <= 1.0:
raise ValueError("shrink_lambda must lie in [0, 1]")
p = _normalised_power(n, beta)
w = (1.0 - shrink_lambda) / len(protos) + shrink_lambda * p
elif mode == "v_cap":
if cap_factor < 1.0:
raise ValueError("cap_factor must be >= 1")
raw = n ** beta
med = float(np.median(raw))
w = np.minimum(raw, cap_factor * med)
elif mode == "v_minsup":
keep = n >= float(min_support)
if not np.any(keep):
# Documented fallback: never emit an empty class. If every
# contributor is below the threshold, keep all of them and
# fall back to equal weighting.
keep = np.ones_like(n, dtype=bool)
sub = [p for p, k in zip(protos, keep) if k]
w = np.ones(len(sub), dtype=np.float64)
protos = sub
elif mode == "v_logweight":
w = np.log1p(n)
else: # pragma: no cover
raise AssertionError(mode)
if not np.any(w > 0):
w = np.ones(len(protos), dtype=np.float64)
result[int(class_id)] = _weighted_mean(protos, w).astype(np.float64).tolist()
return result
def aggregate_from_client_dicts(
client_prototypes,
client_supports,
num_classes: int,
mode,
**kwargs,
):
"""Adapter with the same signature as the project server aggregator."""
contributions = {c: [] for c in range(num_classes)}
for prototypes, supports in zip(client_prototypes, client_supports):
for class_id, proto in prototypes.items():
contributions[int(class_id)].append(
(proto, supports.get(int(class_id), 0))
)
return aggregate_variant(contributions, mode, **kwargs)
exploration_v2/scripts/verify_setup.py
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env python
"""Read-only pre-flight/verification checks for exploration_v2."""
from __future__ import annotations
import hashlib
import json
import subprocess
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
EXPL = REPO / "exploration_v2"
def sh(cmd):
r = subprocess.run(cmd, cwd=str(REPO), capture_output=True, text=True)
return r.returncode, r.stdout.strip(), r.stderr.strip()
def main() -> int:
checks = []
def add(name, ok, detail=""):
checks.append({"check": name, "ok": bool(ok), "detail": detail})
plan = (EXPL / "PLAN.md").read_bytes()
h = hashlib.sha256(plan).hexdigest()
rec = (EXPL / "PLAN.sha256").read_text().splitlines()
add("plan_hash_matches", rec and rec[0].startswith(h), f"{h}")
rc, out, err = sh(["git", "status", "--porcelain"])
tracked = [line for line in out.splitlines()
if not line.startswith("?? exploration_v2/")
and "RESTORE_VERIFICATION_REPORT.md" not in line]
add("no_tracked_modifications", rc == 0 and not tracked, "|".join(tracked[:5]))
rc, out, err = sh(["git", "status", "--porcelain", "planningawal/"])
add("planningawal_unchanged", out == "", out)
raw = list(REPO.glob("**/5G-NIDD.csv")) + list(REPO.glob("**/X-IIoTID-dataset.csv"))
raw = [p for p in raw if ".venv" not in str(p)]
add("no_raw_dataset_in_repo", not raw, "|".join(str(p) for p in raw[:3]))
rc, out, err = sh([str(REPO / ".venv/bin/python"), "-m", "pytest", "-q",
"-p", "no:cacheprovider"])
add("original_tests_pass", rc == 0, out.splitlines()[-1] if out else err[-200:])
rc, out, err = sh([str(REPO / ".venv/bin/python"), "-m", "pytest",
str(EXPL / "tests"), "-q", "-p", "no:cacheprovider"])
add("variant_tests_pass", rc == 0, out.splitlines()[-1] if out else err[-200:])
seeds = set()
if (REPO / "results").is_dir():
for p in (REPO / "results").glob("*seed*"):
name = p.name
for part in name.split("__"):
if part.startswith("seed"):
try:
seeds.add(int(part[4:]))
except ValueError:
pass
add("confirm_seeds_new", not ({3, 4} & seeds), f"restored seeds={sorted(seeds)}")
for p in (EXPL / "partitions").glob("*.json"):
m = json.loads(p.read_text())
if m.get("seed") in (3, 4):
add(f"manifest_ok_{p.name}", m.get("num_clients") == 10 and
len(m.get("support", [])) == 10, f"seed={m.get('seed')}")
ok = all(c["ok"] for c in checks)
lines = ["# exploration_v2 setup check\n\n",
f"Generated: {__import__('datetime').datetime.utcnow().isoformat()}Z\n\n",
"| check | result | detail |\n|---|---|---|\n"]
for c in checks:
lines.append(f"| {c['check']} | {'PASS' if c['ok'] else 'FAIL'} | {c['detail']} |\n")
lines.append(f"\nOverall: {'PASS' if ok else 'FAIL'}\n")
(EXPL / "SETUP_CHECK.md").write_text("".join(lines), encoding="utf-8")
print(json.dumps({"overall": "PASS" if ok else "FAIL", "checks": checks}, indent=2))
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(main())
exploration_v2/smoke/SMOKE.json
+100
View File
@@ -0,0 +1,100 @@
{
"started_utc": "2026-10-30T04:10:46Z",
"tolerance": 0.001,
"runs": {
"base": {
"mode": "original_protean",
"beta": null,
"extra": [],
"returncode": 0,
"val_rare_macro_f1": 0.3506859881859882,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_base"
},
"beta0": {
"mode": "support_weighted",
"beta": 0.0,
"extra": [],
"returncode": 0,
"val_rare_macro_f1": 0.3506859881859882,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_beta0"
},
"beta1": {
"mode": "support_weighted",
"beta": 1.0,
"extra": [],
"returncode": 0,
"val_rare_macro_f1": 0.10617431010522135,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_beta1"
},
"v_beta1.5": {
"mode": "v_beta",
"beta": 1.5,
"extra": [],
"returncode": 0,
"val_rare_macro_f1": 0.0014492753623188406,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_beta1.5"
},
"v_shrink": {
"mode": "v_shrink",
"beta": 1.0,
"extra": [
"--shrink-lambda",
"0.5"
],
"returncode": 0,
"val_rare_macro_f1": 0.07774294670846396,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_shrink"
},
"v_cap4": {
"mode": "v_cap",
"beta": 1.0,
"extra": [
"--cap-factor",
"4"
],
"returncode": 0,
"val_rare_macro_f1": 0.09021243192856791,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_cap4"
},
"v_cap2": {
"mode": "v_cap",
"beta": 1.0,
"extra": [
"--cap-factor",
"2"
],
"returncode": 0,
"val_rare_macro_f1": 0.11728240313145973,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_cap2"
},
"v_minsup": {
"mode": "v_minsup",
"beta": 0.0,
"extra": [
"--min-support",
"10"
],
"returncode": 0,
"val_rare_macro_f1": 0.09491525423728812,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_minsup"
},
"v_logweight": {
"mode": "v_logweight",
"beta": 0.0,
"extra": [],
"returncode": 0,
"val_rare_macro_f1": 0.18652998776009794,
"results_dir": "exploration_v2/smoke/results/xiiotid__alpha0.25__seed0__smoke_v_logweight"
}
},
"pytest_returncode": 0,
"pytest_tail": [
"........... [100%]"
],
"equivalence_checks": {
"beta0": 0.0,
"beta0_within_tolerance": true
},
"verdict": "PASS",
"finished_utc": "2026-10-30T04:11:54Z"
}
exploration_v2/smoke/partitions/xiiotid__alpha0.25__seed0.json
+88
View File
@@ -0,0 +1,88 @@
{
"algorithm": "per_class_dirichlet_multinomial",
"alpha": 0.25,
"benign_class": "Normal",
"class_names": [
"C&C",
"Exfiltration",
"Exploitation",
"Lateral_movement",
"Normal",
"RDOS",
"Reconnaissance",
"Tampering",
"Weaponization"
],
"config_hash": null,
"created_utc": "2026-10-30T04:01:46Z",
"dataset": "xiiotid",
"dataset_name": "X-IIoTID",
"manifest_version": "1",
"num_classes": 9,
"num_clients": 4,
"num_train": 2098,
"partition_hash": "9ac08797480badf66493b2ea557a1f8fd3adfcd190217e1d8fe6ce4accedb3e6",
"preprocessing_hash": "9adc542272436f89b3dfd3285bd6b73427c3f2c1a66caf7da95cd1962cc6ec7b",
"python": "3.12.3",
"rng_seed": 1617371214,
"seed": 0,
"split": {
"hash": "79e21dfb1bb8fa9076e8c24ea14cef409ee702be85c3ab86fc4ccb24fe0e6cb7",
"seed": 0,
"sizes": {
"test": 599,
"train": 2098,
"validation": 300
},
"strategy": "stratified_by_class",
"test_fraction": 0.2,
"train_fraction": 0.7,
"validation_fraction": 0.1
},
"support": [
[
4,
34,
3,
1,
381,
0,
0,
12,
3
],
[
0,
0,
0,
0,
0,
121,
0,
0,
38
],
[
0,
4,
0,
89,
450,
7,
327,
0,
151
],
[
3,
11,
0,
4,
236,
216,
0,
0,
3
]
]
}
exploration_v2/smoke/partitions/xiiotid__alpha0.5__seed0.json
+88
View File
@@ -0,0 +1,88 @@
{
"algorithm": "per_class_dirichlet_multinomial",
"alpha": 0.5,
"benign_class": "Normal",
"class_names": [
"C&C",
"Exfiltration",
"Exploitation",
"Lateral_movement",
"Normal",
"RDOS",
"Reconnaissance",
"Tampering",
"Weaponization"
],
"config_hash": null,
"created_utc": "2026-10-30T03:57:20Z",
"dataset": "xiiotid",
"dataset_name": "X-IIoTID",
"manifest_version": "1",
"num_classes": 9,
"num_clients": 4,
"num_train": 2798,
"partition_hash": "81600a44982bd77e32c0897aa9c367835ef5c8a3c651eb30f2b3e2862cc06a47",
"preprocessing_hash": "83d50d3adb93dd6b6f82e9c9395d9d4b28c30c7a783034508496988f6eddf077",
"python": "3.12.3",
"rng_seed": 2009062324,
"seed": 0,
"split": {
"hash": "f5370ac857a1b4463eb26b0f06a446a6235c008d0fa772019cf11a8d32f2eaf0",
"seed": 0,
"sizes": {
"test": 800,
"train": 2798,
"validation": 399
},
"strategy": "stratified_by_class",
"test_fraction": 0.2,
"train_fraction": 0.7,
"validation_fraction": 0.1
},
"support": [
[
7,
0,
1,
2,
65,
15,
0,
0,
138
],
[
1,
47,
1,
1,
132,
330,
292,
10,
68
],
[
2,
14,
1,
63,
13,
56,
55,
6,
22
],
[
0,
18,
0,
33,
1246,
101,
47,
5,
6
]
]
}
exploration_v2/state.json
+22
View File
@@ -0,0 +1,22 @@
{
"attempts": {
"xiiotid__alpha0.25__seed3__v_cap__b1__c4": 1,
"xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5": 1
},
"budget_start_utc": "2026-10-30T04:11:55Z",
"max_procs": 2,
"notes": [
"2026-10-31T08:35Z recovery: 2 OOM-killed seed4 reference runs requeued at max_procs=2",
"2026-10-31T14:12Z recovery A5: unit-name collision on retries fixed; seed4 refs requeued"
],
"phase": "done",
"ranking_frozen": true,
"recent_exits": [
0,
0,
0,
0
],
"screening_started": false,
"terminal_failed": []
}
exploration_v2/status.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
# Quick exploration_v2 status (returns in well under 60 s).
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/.." && pwd)"
"$REPO_ROOT/.venv/bin/python" "$HERE/scripts/status_table.py"
echo
if [ -f "$HERE/HEARTBEAT" ]; then
echo "heartbeat file age: $(( $(date +%s) - $(stat -c %Y "$HERE/HEARTBEAT") )) s"
fi
echo "driver scope:"
systemctl --user list-units --type=scope --no-legend 2>/dev/null | grep -i explorev2 || true
echo "queue sizes: screen=$(($(wc -l < "$HERE/queue_screen.csv" 2>/dev/null || echo 1) - 1)) confirm=$( [ -f "$HERE/queue_confirm.csv" ] && echo $(( $(wc -l < "$HERE/queue_confirm.csv") - 1 )) || echo 0 )"
exploration_v2/summary.csv
+35
View File
@@ -0,0 +1,35 @@
config,dataset,alpha,beta,seed,aggregation_mode,status,wall_s,cpu_user_s,cpu_sys_s,percent_cpu,peak_rss_kb,exit_code,started_utc,ended_utc,run_dir,log_dir,notes
xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,xiiotid,0.25,1.0,0,v_shrink,completed,10965.00,10331.62,629.68,99,8788104,0,2026-10-30T04:11:55Z,2026-10-30T07:14:40Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/logs,
xiiotid__alpha0.25__seed0__v_cap__b1__c4,xiiotid,0.25,1.0,0,v_cap,completed,11022.00,10392.67,626.13,99,8789080,0,2026-10-30T04:11:55Z,2026-10-30T07:15:37Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4/logs,
xiiotid__alpha0.25__seed0__v_beta__b1.5,xiiotid,0.25,1.5,0,v_beta,completed,11048.00,10424.25,620.24,99,8788696,0,2026-10-30T04:11:55Z,2026-10-30T07:16:03Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5/logs,
xiiotid__alpha0.25__seed0__original_protean,xiiotid,0.25,0,0,original_protean,completed,10879.00,10141.92,628.62,98,8788432,0,2026-10-30T07:14:43Z,2026-10-30T10:16:03Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean/logs,
xiiotid__alpha0.25__seed0__v_minsup__t10,xiiotid,0.25,0.0,0,v_minsup,completed,10904.00,10156.27,630.54,98,8788056,0,2026-10-30T07:16:06Z,2026-10-30T10:17:50Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10/logs,
xiiotid__alpha0.25__seed0__v_cap__b1__c2,xiiotid,0.25,1.0,0,v_cap,completed,10960.00,10217.02,625.13,98,8787984,0,2026-10-30T07:15:45Z,2026-10-30T10:18:26Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2/logs,
xiiotid__alpha0.25__seed0__v_logweight,xiiotid,0.25,0.0,0,v_logweight,completed,10645.00,10013.81,627.86,99,8787588,0,2026-10-30T10:16:12Z,2026-10-30T13:13:38Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight/logs,
5gnidd__alpha0.5__seed0__original_protean,5gnidd,0.5,0,0,original_protean,completed,16807.00,15709.29,1094.58,99,10800928,0,2026-10-30T10:18:28Z,2026-10-30T14:58:36Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean/logs,
5gnidd__alpha0.5__seed0__v_beta__b1.5,5gnidd,0.5,1.5,0,v_beta,completed,17571.00,16433.69,1133.44,99,10787996,0,2026-10-30T13:13:46Z,2026-10-30T18:06:38Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5/logs,
5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,5gnidd,0.5,1.0,0,v_shrink,completed,17467.00,16353.62,1108.69,99,10791372,0,2026-10-30T14:58:45Z,2026-10-30T19:49:52Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/logs,
5gnidd__alpha0.5__seed0__v_cap__b1__c4,5gnidd,0.5,1.0,0,v_cap,completed,16374.00,15305.03,1064.96,99,10795492,0,2026-10-30T18:06:47Z,2026-10-30T22:39:42Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4/logs,
5gnidd__alpha0.5__seed0__v_cap__b1__c2,5gnidd,0.5,1.0,0,v_cap,completed,15875.00,14852.73,1018.36,99,10794444,0,2026-10-30T19:50:02Z,2026-10-31T00:14:37Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2/logs,
5gnidd__alpha0.5__seed0__v_minsup__t10,5gnidd,0.5,0.0,0,v_minsup,completed,16232.00,15203.43,1021.85,99,10787860,0,2026-10-30T22:39:52Z,2026-10-31T03:10:24Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10/logs,
5gnidd__alpha0.5__seed0__v_logweight,5gnidd,0.5,0.0,0,v_logweight,completed,17342.00,16248.83,1085.02,99,10806700,0,2026-10-31T00:14:45Z,2026-10-31T05:03:48Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight/logs,
xiiotid__alpha0.25__seed3__original_protean,xiiotid,0.25,0,3,original_protean,completed,11102.00,10505.16,593.52,99,10986852,0,2026-10-31T05:07:30Z,2026-10-31T08:12:32Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__original_protean/logs,
xiiotid__alpha0.25__seed3__support_weighted__beta0.5,xiiotid,0.25,0.5,3,support_weighted,completed,11105.00,10503.16,599.06,99,10986972,0,2026-10-31T05:07:30Z,2026-10-31T08:12:36Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta0.5/logs,
xiiotid__alpha0.25__seed3__support_weighted__beta1,xiiotid,0.25,1,3,support_weighted,completed,11247.00,10628.09,615.51,99,10986812,0,2026-10-31T05:07:30Z,2026-10-31T08:14:57Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta1/logs,
xiiotid__alpha0.25__seed4__original_protean,xiiotid,0.25,0,4,original_protean,completed,11200.00,10544.02,652.44,99,13373900,0,2026-10-31T14:09:18Z,2026-10-31T17:15:59Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__original_protean/logs,
xiiotid__alpha0.25__seed4__support_weighted__beta0.5,xiiotid,0.25,0.5,4,support_weighted,completed,11147.00,10490.48,653.41,99,13376792,0,2026-10-31T14:09:18Z,2026-10-31T17:15:06Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta0.5/logs,
xiiotid__alpha0.25__seed4__support_weighted__beta1,xiiotid,0.25,1,4,support_weighted,completed,11529.00,10850.72,675.30,99,13377304,0,2026-10-31T17:15:10Z,2026-10-31T20:27:20Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta1/logs,
xiiotid__alpha0.25__seed3__v_cap__b1__c4,xiiotid,0.25,1.0,3,v_cap,completed,10621.00,10020.19,598.36,99,10986660,0,2026-10-31T11:09:05Z,2026-10-31T14:06:08Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_cap__b1__c4/logs,
xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,xiiotid,0.25,1.0,3,v_shrink,completed,10678.00,10077.25,599.26,99,10986572,0,2026-10-31T11:09:48Z,2026-10-31T14:07:47Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5/logs,
xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,xiiotid,0.25,1.0,4,v_shrink,completed,11448.00,10782.09,663.27,99,13377596,0,2026-10-31T17:16:03Z,2026-10-31T20:26:52Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5/logs,
xiiotid__alpha0.25__seed4__v_cap__b1__c4,xiiotid,0.25,1.0,4,v_cap,completed,10168.00,9580.67,584.50,99,13377208,0,2026-10-31T20:27:02Z,2026-10-31T23:16:30Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_cap__b1__c4/logs,
5gnidd__alpha0.5__seed3__original_protean,5gnidd,0.5,0,3,original_protean,completed,15875.00,14911.46,960.26,99,9482756,0,2026-10-31T20:27:23Z,2026-11-01T00:51:58Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__original_protean/logs,
5gnidd__alpha0.5__seed3__support_weighted__beta0.5,5gnidd,0.5,0.5,3,support_weighted,completed,16116.00,15128.78,983.96,99,9507836,0,2026-10-31T23:16:40Z,2026-11-01T03:45:16Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta0.5/logs,
5gnidd__alpha0.5__seed3__support_weighted__beta1,5gnidd,0.5,1,3,support_weighted,completed,16782.00,15774.09,1004.24,99,9480524,0,2026-11-01T00:52:06Z,2026-11-01T05:31:49Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta1/logs,
5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,5gnidd,0.5,1.0,3,v_shrink,completed,16459.00,15453.75,1000.89,99,9501568,0,2026-11-01T03:45:24Z,2026-11-01T08:19:43Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5/logs,
5gnidd__alpha0.5__seed3__v_cap__b1__c4,5gnidd,0.5,1.0,3,v_cap,completed,17023.00,15980.81,1039.03,99,9487356,0,2026-11-01T05:31:54Z,2026-11-01T10:15:38Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_cap__b1__c4/logs,
5gnidd__alpha0.5__seed4__original_protean,5gnidd,0.5,0,4,original_protean,completed,17324.00,16219.34,1100.77,99,11657012,0,2026-11-01T08:19:51Z,2026-11-01T13:08:35Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__original_protean,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__original_protean/logs,
5gnidd__alpha0.5__seed4__support_weighted__beta0.5,5gnidd,0.5,0.5,4,support_weighted,completed,17181.00,16103.55,1073.60,99,11697428,0,2026-11-01T10:15:40Z,2026-11-01T15:02:03Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta0.5/logs,
5gnidd__alpha0.5__seed4__support_weighted__beta1,5gnidd,0.5,1,4,support_weighted,completed,17582.00,16447.80,1131.04,99,11669892,0,2026-11-01T13:08:37Z,2026-11-01T18:01:41Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta1,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta1/logs,
5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,5gnidd,0.5,1.0,4,v_shrink,completed,17234.00,16097.43,1133.25,99,11688404,0,2026-11-01T15:02:11Z,2026-11-01T19:49:27Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5/logs,
5gnidd__alpha0.5__seed4__v_cap__b1__c4,5gnidd,0.5,1.0,4,v_cap,completed,16415.00,15378.21,1031.38,99,11679488,0,2026-11-01T18:01:52Z,2026-11-01T22:35:27Z,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_cap__b1__c4,/home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_cap__b1__c4/logs,
1 config dataset alpha beta seed aggregation_mode status wall_s cpu_user_s cpu_sys_s percent_cpu peak_rss_kb exit_code started_utc ended_utc run_dir log_dir notes
2 xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5 xiiotid 0.25 1.0 0 v_shrink completed 10965.00 10331.62 629.68 99 8788104 0 2026-10-30T04:11:55Z 2026-10-30T07:14:40Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_shrink__b1__lam0.5/logs
3 xiiotid__alpha0.25__seed0__v_cap__b1__c4 xiiotid 0.25 1.0 0 v_cap completed 11022.00 10392.67 626.13 99 8789080 0 2026-10-30T04:11:55Z 2026-10-30T07:15:37Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c4/logs
4 xiiotid__alpha0.25__seed0__v_beta__b1.5 xiiotid 0.25 1.5 0 v_beta completed 11048.00 10424.25 620.24 99 8788696 0 2026-10-30T04:11:55Z 2026-10-30T07:16:03Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_beta__b1.5/logs
5 xiiotid__alpha0.25__seed0__original_protean xiiotid 0.25 0 0 original_protean completed 10879.00 10141.92 628.62 98 8788432 0 2026-10-30T07:14:43Z 2026-10-30T10:16:03Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__original_protean/logs
6 xiiotid__alpha0.25__seed0__v_minsup__t10 xiiotid 0.25 0.0 0 v_minsup completed 10904.00 10156.27 630.54 98 8788056 0 2026-10-30T07:16:06Z 2026-10-30T10:17:50Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_minsup__t10/logs
7 xiiotid__alpha0.25__seed0__v_cap__b1__c2 xiiotid 0.25 1.0 0 v_cap completed 10960.00 10217.02 625.13 98 8787984 0 2026-10-30T07:15:45Z 2026-10-30T10:18:26Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_cap__b1__c2/logs
8 xiiotid__alpha0.25__seed0__v_logweight xiiotid 0.25 0.0 0 v_logweight completed 10645.00 10013.81 627.86 99 8787588 0 2026-10-30T10:16:12Z 2026-10-30T13:13:38Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed0__v_logweight/logs
9 5gnidd__alpha0.5__seed0__original_protean 5gnidd 0.5 0 0 original_protean completed 16807.00 15709.29 1094.58 99 10800928 0 2026-10-30T10:18:28Z 2026-10-30T14:58:36Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__original_protean/logs
10 5gnidd__alpha0.5__seed0__v_beta__b1.5 5gnidd 0.5 1.5 0 v_beta completed 17571.00 16433.69 1133.44 99 10787996 0 2026-10-30T13:13:46Z 2026-10-30T18:06:38Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_beta__b1.5/logs
11 5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5 5gnidd 0.5 1.0 0 v_shrink completed 17467.00 16353.62 1108.69 99 10791372 0 2026-10-30T14:58:45Z 2026-10-30T19:49:52Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_shrink__b1__lam0.5/logs
12 5gnidd__alpha0.5__seed0__v_cap__b1__c4 5gnidd 0.5 1.0 0 v_cap completed 16374.00 15305.03 1064.96 99 10795492 0 2026-10-30T18:06:47Z 2026-10-30T22:39:42Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c4/logs
13 5gnidd__alpha0.5__seed0__v_cap__b1__c2 5gnidd 0.5 1.0 0 v_cap completed 15875.00 14852.73 1018.36 99 10794444 0 2026-10-30T19:50:02Z 2026-10-31T00:14:37Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_cap__b1__c2/logs
14 5gnidd__alpha0.5__seed0__v_minsup__t10 5gnidd 0.5 0.0 0 v_minsup completed 16232.00 15203.43 1021.85 99 10787860 0 2026-10-30T22:39:52Z 2026-10-31T03:10:24Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_minsup__t10/logs
15 5gnidd__alpha0.5__seed0__v_logweight 5gnidd 0.5 0.0 0 v_logweight completed 17342.00 16248.83 1085.02 99 10806700 0 2026-10-31T00:14:45Z 2026-10-31T05:03:48Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed0__v_logweight/logs
16 xiiotid__alpha0.25__seed3__original_protean xiiotid 0.25 0 3 original_protean completed 11102.00 10505.16 593.52 99 10986852 0 2026-10-31T05:07:30Z 2026-10-31T08:12:32Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__original_protean /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__original_protean/logs
17 xiiotid__alpha0.25__seed3__support_weighted__beta0.5 xiiotid 0.25 0.5 3 support_weighted completed 11105.00 10503.16 599.06 99 10986972 0 2026-10-31T05:07:30Z 2026-10-31T08:12:36Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta0.5/logs
18 xiiotid__alpha0.25__seed3__support_weighted__beta1 xiiotid 0.25 1 3 support_weighted completed 11247.00 10628.09 615.51 99 10986812 0 2026-10-31T05:07:30Z 2026-10-31T08:14:57Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta1 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__support_weighted__beta1/logs
19 xiiotid__alpha0.25__seed4__original_protean xiiotid 0.25 0 4 original_protean completed 11200.00 10544.02 652.44 99 13373900 0 2026-10-31T14:09:18Z 2026-10-31T17:15:59Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__original_protean /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__original_protean/logs
20 xiiotid__alpha0.25__seed4__support_weighted__beta0.5 xiiotid 0.25 0.5 4 support_weighted completed 11147.00 10490.48 653.41 99 13376792 0 2026-10-31T14:09:18Z 2026-10-31T17:15:06Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta0.5/logs
21 xiiotid__alpha0.25__seed4__support_weighted__beta1 xiiotid 0.25 1 4 support_weighted completed 11529.00 10850.72 675.30 99 13377304 0 2026-10-31T17:15:10Z 2026-10-31T20:27:20Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta1 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__support_weighted__beta1/logs
22 xiiotid__alpha0.25__seed3__v_cap__b1__c4 xiiotid 0.25 1.0 3 v_cap completed 10621.00 10020.19 598.36 99 10986660 0 2026-10-31T11:09:05Z 2026-10-31T14:06:08Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_cap__b1__c4 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_cap__b1__c4/logs
23 xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5 xiiotid 0.25 1.0 3 v_shrink completed 10678.00 10077.25 599.26 99 10986572 0 2026-10-31T11:09:48Z 2026-10-31T14:07:47Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed3__v_shrink__b1__lam0.5/logs
24 xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5 xiiotid 0.25 1.0 4 v_shrink completed 11448.00 10782.09 663.27 99 13377596 0 2026-10-31T17:16:03Z 2026-10-31T20:26:52Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_shrink__b1__lam0.5/logs
25 xiiotid__alpha0.25__seed4__v_cap__b1__c4 xiiotid 0.25 1.0 4 v_cap completed 10168.00 9580.67 584.50 99 13377208 0 2026-10-31T20:27:02Z 2026-10-31T23:16:30Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_cap__b1__c4 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/xiiotid__alpha0.25__seed4__v_cap__b1__c4/logs
26 5gnidd__alpha0.5__seed3__original_protean 5gnidd 0.5 0 3 original_protean completed 15875.00 14911.46 960.26 99 9482756 0 2026-10-31T20:27:23Z 2026-11-01T00:51:58Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__original_protean /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__original_protean/logs
27 5gnidd__alpha0.5__seed3__support_weighted__beta0.5 5gnidd 0.5 0.5 3 support_weighted completed 16116.00 15128.78 983.96 99 9507836 0 2026-10-31T23:16:40Z 2026-11-01T03:45:16Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta0.5/logs
28 5gnidd__alpha0.5__seed3__support_weighted__beta1 5gnidd 0.5 1 3 support_weighted completed 16782.00 15774.09 1004.24 99 9480524 0 2026-11-01T00:52:06Z 2026-11-01T05:31:49Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta1 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__support_weighted__beta1/logs
29 5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5 5gnidd 0.5 1.0 3 v_shrink completed 16459.00 15453.75 1000.89 99 9501568 0 2026-11-01T03:45:24Z 2026-11-01T08:19:43Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_shrink__b1__lam0.5/logs
30 5gnidd__alpha0.5__seed3__v_cap__b1__c4 5gnidd 0.5 1.0 3 v_cap completed 17023.00 15980.81 1039.03 99 9487356 0 2026-11-01T05:31:54Z 2026-11-01T10:15:38Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_cap__b1__c4 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed3__v_cap__b1__c4/logs
31 5gnidd__alpha0.5__seed4__original_protean 5gnidd 0.5 0 4 original_protean completed 17324.00 16219.34 1100.77 99 11657012 0 2026-11-01T08:19:51Z 2026-11-01T13:08:35Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__original_protean /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__original_protean/logs
32 5gnidd__alpha0.5__seed4__support_weighted__beta0.5 5gnidd 0.5 0.5 4 support_weighted completed 17181.00 16103.55 1073.60 99 11697428 0 2026-11-01T10:15:40Z 2026-11-01T15:02:03Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta0.5/logs
33 5gnidd__alpha0.5__seed4__support_weighted__beta1 5gnidd 0.5 1 4 support_weighted completed 17582.00 16447.80 1131.04 99 11669892 0 2026-11-01T13:08:37Z 2026-11-01T18:01:41Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta1 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__support_weighted__beta1/logs
34 5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5 5gnidd 0.5 1.0 4 v_shrink completed 17234.00 16097.43 1133.25 99 11688404 0 2026-11-01T15:02:11Z 2026-11-01T19:49:27Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_shrink__b1__lam0.5/logs
35 5gnidd__alpha0.5__seed4__v_cap__b1__c4 5gnidd 0.5 1.0 4 v_cap completed 16415.00 15378.21 1031.38 99 11679488 0 2026-11-01T18:01:52Z 2026-11-01T22:35:27Z /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_cap__b1__c4 /home/willtanoe/project/prototype-agregation/exploration_v2/results/runs/5gnidd__alpha0.5__seed4__v_cap__b1__c4/logs
exploration_v2/tests/test_pipeline_scripts.py
+124
View File
@@ -0,0 +1,124 @@
"""Fixture tests for the screening ranking and post-confirmation analysis.
These use a temporary directory tree and never read or write the live
``exploration_v2/results`` or any restored result. The references for the
fixture come from the real restored grid, read-only.
"""
from __future__ import annotations
import csv
import hashlib
import json
import sys
from pathlib import Path
import pytest
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO / "exploration_v2" / "scripts"))
import rank_screening # noqa: E402
import analyze_results # noqa: E402
def _write(path: Path, payload):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload), encoding="utf-8")
def _screen_fixture(tmp: Path):
base = {"S1": 0.60, "S2": 0.85}
labels = [("v_beta__b1.5", 0.01), ("v_shrink__b1__lam0.5", 0.02),
("v_cap__b1__c4", 0.015), ("v_cap__b1__c2", -0.01),
("v_minsup__t10", -0.02), ("v_logweight", 0.005)]
rows = []
for ds, alpha, cid in (("xiiotid", 0.25, "S1"), ("5gnidd", 0.5, "S2")):
for label, off in labels:
rid = f"{ds}__alpha{alpha:g}__seed0__{label}"
d = tmp / "runs" / rid / "results" / rid
_write(d / "val_global_metrics.json",
{"rare_class_macro_f1": base[cid] + off})
_write(d / "val_selection.json",
{"val_rare_class_macro_f1_per_client_mean": 0.5 + off})
# A decoy test file: the ranking must never open it.
_write(d / "test_global_metrics.json", {"rare_class_macro_f1": 999.0})
rows.append(dict(run_id=rid, dataset=ds, alpha=str(alpha), seed="0",
mode="v_beta", beta="1", shrink="", cap="",
minsup="", part_dir="x", label=label))
q = tmp / "queue_screen.csv"
with q.open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
return q
def test_rank_uses_validation_only(tmp_path, monkeypatch):
q = _screen_fixture(tmp_path)
monkeypatch.setattr(rank_screening, "RUNS", tmp_path / "runs")
monkeypatch.setattr(rank_screening, "SCREEN", q)
monkeypatch.setattr(rank_screening, "ACCESS", tmp_path / "ACCESS_LOG.txt")
monkeypatch.setattr(rank_screening, "OUT", tmp_path / "ranking.json")
assert rank_screening.main() == 0
payload = json.loads((tmp_path / "ranking.json").read_text())
assert payload["top2"] == ["v_shrink__b1__lam0.5", "v_cap__b1__c4"]
log = (tmp_path / "ACCESS_LOG.txt").read_text()
assert "test_global_metrics" not in log
assert "per_client.csv" not in log
def test_analysis_refuses_without_freeze(tmp_path, monkeypatch):
monkeypatch.setattr(analyze_results, "EXPL", tmp_path)
monkeypatch.setattr(analyze_results, "RUNS", tmp_path / "runs")
monkeypatch.setattr(analyze_results, "OUT", tmp_path / "analysis")
with pytest.raises(SystemExit):
analyze_results.main()
def _confirm_fixture(tmp: Path, top2):
rows = []
for ds, alpha in (("xiiotid", 0.25), ("5gnidd", 0.5)):
for seed in (3, 4):
for label, val in [("original_protean", 0.60),
("support_weighted__beta0.5", 0.61),
("support_weighted__beta1", 0.605),
(top2[0], 0.62), (top2[1], 0.615)]:
rid = f"{ds}__alpha{alpha:g}__seed{seed}__{label}"
d = tmp / "runs" / rid / "results" / rid
_write(d / "val_global_metrics.json",
{"rare_class_macro_f1": val, "rare_class_recall": 0.8,
"macro_f1": 0.7})
_write(d / "val_selection.json",
{"val_rare_class_macro_f1_per_client_mean": val - 0.05})
_write(d / "test_global_metrics.json",
{"rare_class_macro_f1": val + 0.01,
"rare_class_recall": 0.8, "macro_f1": 0.7})
rows.append(dict(run_id=rid, dataset=ds, alpha=str(alpha),
seed=str(seed), mode="x", beta="", shrink="",
cap="", minsup="", part_dir="x", label=label))
q = tmp / "queue_confirm.csv"
with q.open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
(tmp / "queue_screen.csv").write_text(
"run_id,dataset,alpha,seed,mode,beta,shrink,cap,minsup,part_dir,label\n")
return top2
def test_analysis_writes_tables(tmp_path, monkeypatch):
top2 = ["v_shrink__b1__lam0.5", "v_cap__b1__c4"]
_confirm_fixture(tmp_path, top2)
ranking = {"top2": top2, "candidates": [], "current_method_reference": {}}
(tmp_path / "ranking.json").write_text(json.dumps(ranking))
h = hashlib.sha256((tmp_path / "ranking.json").read_bytes()).hexdigest()
(tmp_path / "ranking.sha256").write_text(f"{h} ranking.json\n2026-01-01T00:00:00Z\n")
monkeypatch.setattr(analyze_results, "EXPL", tmp_path)
monkeypatch.setattr(analyze_results, "RUNS", tmp_path / "runs")
monkeypatch.setattr(analyze_results, "OUT", tmp_path / "analysis")
assert analyze_results.main() == 0
summary = list(csv.DictReader(open(tmp_path / "analysis" / "confirm_summary.csv")))
assert len(summary) == 2
assert summary[0]["positive_pairs_val_vs_original"] == "4"
assert (tmp_path / "REPORT_DATA.md").is_file()
exploration_v2/tests/test_variants.py
+160
View File
@@ -0,0 +1,160 @@
"""Unit tests for the exploration_v2 aggregation variants.
The critical invariants required before any expensive run:
* ``original_protean`` is bit-for-bit the project implementation;
* ``support_weighted`` (including beta = 0) delegates to the project code;
* ``v_beta`` at beta = 0 reproduces the equal mean, and at beta = 1 reproduces
``support_weighted`` beta = 1;
* a single contributor, equal supports and missing classes behave as documented;
* every variant returns finite, deterministic vectors.
Run with:
.venv/bin/python -m pytest exploration_v2/tests -q
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pytest
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO / "src"))
sys.path.insert(0, str(REPO / "exploration_v2" / "scripts"))
from protean_research.aggregation import aggregate_class_prototypes # noqa: E402
import variants as V # noqa: E402
def _fixture():
rng = np.random.default_rng(7)
# class 0: 4 contributors; class 1: 2 contributors; class 2 absent
return {
0: [(rng.normal(size=8), n) for n in (5, 50, 500, 5000)],
1: [(rng.normal(size=8), 3), (rng.normal(size=8), 300)],
}, rng
def _mean(protos):
return np.mean(np.stack([np.asarray(p, float) for p in protos]), axis=0)
def test_original_is_project_implementation():
contributions, _ = _fixture()
got = V.aggregate_variant(contributions, "original_protean")
ref = aggregate_class_prototypes(contributions, "original_protean")
assert set(got) == set(ref) == {0, 1}
for c in ref:
assert np.array_equal(np.asarray(got[c]), np.asarray(ref[c]))
def test_support_weighted_delegates_exactly():
contributions, _ = _fixture()
for beta in (0.0, 0.25, 0.5, 0.75, 1.0):
got = V.aggregate_variant(contributions, "support_weighted", beta=beta)
ref = aggregate_class_prototypes(contributions, "support_weighted", beta=beta)
for c in ref:
assert np.array_equal(np.asarray(got[c]), np.asarray(ref[c]))
def test_beta0_is_equal_mean():
contributions, _ = _fixture()
base = V.aggregate_variant(contributions, "original_protean")
for mode in ("support_weighted", "v_beta"):
got = V.aggregate_variant(contributions, mode, beta=0.0)
for c in base:
np.testing.assert_allclose(got[c], base[c], rtol=0, atol=1e-12)
# and v_beta beta1 == support_weighted beta1
a = V.aggregate_variant(contributions, "v_beta", beta=1.0)
b = V.aggregate_variant(contributions, "support_weighted", beta=1.0)
for c in a:
np.testing.assert_allclose(a[c], b[c], rtol=0, atol=1e-12)
def test_single_contributor_invariance():
proto = np.arange(6, dtype=float)
contributions = {0: [(proto, 17)]}
for mode in V.VARIANT_MODES:
for beta in (0.0, 0.5, 1.0):
got = V.aggregate_variant(contributions, mode, beta=beta)
np.testing.assert_allclose(got[0], proto, rtol=0, atol=0)
def test_equal_support_invariance():
rng = np.random.default_rng(11)
contributions = {0: [(rng.normal(size=5), 100) for _ in range(4)]}
means = [np.asarray(V.aggregate_variant(contributions, "support_weighted", beta=b)[0])
for b in (0.0, 0.25, 0.5, 0.75, 1.0)]
for m in means[1:]:
np.testing.assert_allclose(m, means[0], rtol=0, atol=1e-12)
def test_missing_class_never_fabricated():
contributions, _ = _fixture()
for mode in V.ALL_MODES:
kwargs = {"beta": 0.0} if mode != "original_protean" else {}
got = V.aggregate_variant(contributions, mode, **kwargs)
assert 2 not in got
for vec in got.values():
assert np.all(np.isfinite(vec))
assert np.any(np.asarray(vec) != 0.0)
def test_shrink_endpoints():
contributions, _ = _fixture()
equal = V.aggregate_variant(contributions, "original_protean")
sw = V.aggregate_variant(contributions, "support_weighted", beta=1.0)
s0 = V.aggregate_variant(contributions, "v_shrink", beta=1.0, shrink_lambda=0.0)
s1 = V.aggregate_variant(contributions, "v_shrink", beta=1.0, shrink_lambda=1.0)
for c in equal:
np.testing.assert_allclose(s0[c], equal[c], atol=1e-12)
np.testing.assert_allclose(s1[c], sw[c], atol=1e-12)
def test_cap_extremes():
contributions, _ = _fixture()
sw = V.aggregate_variant(contributions, "support_weighted", beta=1.0)
big = V.aggregate_variant(contributions, "v_cap", beta=1.0, cap_factor=1e9)
for c in sw:
np.testing.assert_allclose(big[c], sw[c], atol=1e-9)
small = V.aggregate_variant(contributions, "v_cap", beta=1.0, cap_factor=1.0)
for c in sw:
assert np.all(np.isfinite(small[c]))
def test_minsup_participant_change():
rng = np.random.default_rng(3)
p_small, p_big = rng.normal(size=4), rng.normal(size=4)
contributions = {0: [(p_small, 1), (p_big, 100)]}
got = V.aggregate_variant(contributions, "v_minsup", beta=0.0, min_support=10)
np.testing.assert_allclose(got[0], p_big, atol=0)
# fallback when every contributor is below the threshold
contributions = {0: [(p_small, 1), (p_big, 2)]}
got = V.aggregate_variant(contributions, "v_minsup", beta=0.0, min_support=10)
np.testing.assert_allclose(got[0], _mean([p_small, p_big]), atol=1e-12)
def test_logweight_positive_and_deterministic():
contributions, _ = _fixture()
a = V.aggregate_variant(contributions, "v_logweight", beta=0.0)
b = V.aggregate_variant(contributions, "v_logweight", beta=0.0)
for c in a:
assert np.array_equal(np.asarray(a[c]), np.asarray(b[c]))
assert np.all(np.isfinite(a[c]))
def test_invalid_parameters():
contributions, _ = _fixture()
with pytest.raises(ValueError):
V.aggregate_variant(contributions, "v_beta", beta=2.5)
with pytest.raises(ValueError):
V.aggregate_variant(contributions, "v_shrink", beta=0.5, shrink_lambda=1.5)
with pytest.raises(ValueError):
V.aggregate_variant(contributions, "v_cap", beta=0.5, cap_factor=0.5)
with pytest.raises(ValueError):
V.aggregate_variant(contributions, "not_a_mode", beta=0.5)
with pytest.raises(ValueError):
V.aggregate_variant(contributions, "v_beta", beta=None)