docs: validate baseline and add deterministic regression coverage
Session 3 baseline validation (no beta/support-weighted code added). - Two measured full-data original_protean conditions (X-IIoTID and 5G-NIDD, alpha=0.5, seed=0); frozen manifests reused and reload-verified. - Partition consistency, per-class/per-client inspection, round-0, loss audit, inference validation, reproducibility and a deterministic end-to-end test. - S_j participant rule remains an explicitly labelled REPRODUCTION ASSUMPTION; |S_j| < M quantified in 53/162 class-condition cells (32.7%), 15/18 conditions. - Adds docs/baseline/BASELINE_VALIDATION.md, tests/test_baseline_validation.py, scripts/validate_sj_participant_rule.py and small analysis artifacts. - Test suite: 46 passed. planningawal/ and raw datasets unchanged.
This commit is contained in:
willtanoe committed
1 parent
374651a0fe
commit
16b3add2cf
9 files changed
+24265
-200
No files matched your search
+11575
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,163 @@
|
||||
dataset,alpha,seed,M,class_id,class_name,S_j,missing_clients,N_contrib_min,N_contrib_max,N_total
|
||||
5gnidd,0.25,0,10,0,Benign,10,,1,185504,334415
|
||||
5gnidd,0.25,0,10,1,HTTPFlood,9,4,1,48469,98569
|
||||
5gnidd,0.25,0,10,2,ICMPFlood,9,5,2,319,808
|
||||
5gnidd,0.25,0,10,3,SYNFlood,7,1|5|7,144,4762,6805
|
||||
5gnidd,0.25,0,10,4,SYNScan,7,2|5|7,237,6524,14030
|
||||
5gnidd,0.25,0,10,5,SlowrateDoS,9,7,1,16517,51187
|
||||
5gnidd,0.25,0,10,6,TCPConnectScan,9,6,2,10870,14037
|
||||
5gnidd,0.25,0,10,7,UDPFlood,10,,1,127301,320138
|
||||
5gnidd,0.25,0,10,8,UDPScan,10,,2,4601,11134
|
||||
5gnidd,0.25,1,10,0,Benign,10,,30,177603,334415
|
||||
5gnidd,0.25,1,10,1,HTTPFlood,9,0,5,45645,98569
|
||||
5gnidd,0.25,1,10,2,ICMPFlood,10,,1,257,808
|
||||
5gnidd,0.25,1,10,3,SYNFlood,9,1,10,2997,6805
|
||||
5gnidd,0.25,1,10,4,SYNScan,10,,7,3477,14030
|
||||
5gnidd,0.25,1,10,5,SlowrateDoS,8,0|7,211,23478,51187
|
||||
5gnidd,0.25,1,10,6,TCPConnectScan,8,3|7,36,3627,14037
|
||||
5gnidd,0.25,1,10,7,UDPFlood,10,,1,128338,320138
|
||||
5gnidd,0.25,1,10,8,UDPScan,6,0|3|4|8,67,5172,11134
|
||||
5gnidd,0.25,2,10,0,Benign,10,,10,227468,334415
|
||||
5gnidd,0.25,2,10,1,HTTPFlood,7,0|2|8,43,36762,98569
|
||||
5gnidd,0.25,2,10,2,ICMPFlood,8,2|5,6,355,808
|
||||
5gnidd,0.25,2,10,3,SYNFlood,9,7,3,3136,6805
|
||||
5gnidd,0.25,2,10,4,SYNScan,10,,2,10072,14030
|
||||
5gnidd,0.25,2,10,5,SlowrateDoS,10,,4,31740,51187
|
||||
5gnidd,0.25,2,10,6,TCPConnectScan,9,7,3,3619,14037
|
||||
5gnidd,0.25,2,10,7,UDPFlood,10,,113,82051,320138
|
||||
5gnidd,0.25,2,10,8,UDPScan,7,1|3|8,243,3899,11134
|
||||
5gnidd,0.5,0,10,0,Benign,10,,226,108257,334415
|
||||
5gnidd,0.5,0,10,1,HTTPFlood,10,,185,61393,98569
|
||||
5gnidd,0.5,0,10,2,ICMPFlood,9,3,8,243,808
|
||||
5gnidd,0.5,0,10,3,SYNFlood,9,4,2,3250,6805
|
||||
5gnidd,0.5,0,10,4,SYNScan,10,,11,6713,14030
|
||||
5gnidd,0.5,0,10,5,SlowrateDoS,10,,80,28363,51187
|
||||
5gnidd,0.5,0,10,6,TCPConnectScan,10,,1,4175,14037
|
||||
5gnidd,0.5,0,10,7,UDPFlood,10,,1,88943,320138
|
||||
5gnidd,0.5,0,10,8,UDPScan,10,,14,3420,11134
|
||||
5gnidd,0.5,1,10,0,Benign,10,,803,104938,334415
|
||||
5gnidd,0.5,1,10,1,HTTPFlood,10,,51,36928,98569
|
||||
5gnidd,0.5,1,10,2,ICMPFlood,10,,8,284,808
|
||||
5gnidd,0.5,1,10,3,SYNFlood,10,,26,2093,6805
|
||||
5gnidd,0.5,1,10,4,SYNScan,10,,259,3621,14030
|
||||
5gnidd,0.5,1,10,5,SlowrateDoS,10,,26,16089,51187
|
||||
5gnidd,0.5,1,10,6,TCPConnectScan,10,,3,7188,14037
|
||||
5gnidd,0.5,1,10,7,UDPFlood,10,,1907,95125,320138
|
||||
5gnidd,0.5,1,10,8,UDPScan,10,,1,5065,11134
|
||||
5gnidd,0.5,2,10,0,Benign,10,,255,225038,334415
|
||||
5gnidd,0.5,2,10,1,HTTPFlood,10,,24,48351,98569
|
||||
5gnidd,0.5,2,10,2,ICMPFlood,10,,1,358,808
|
||||
5gnidd,0.5,2,10,3,SYNFlood,10,,8,3530,6805
|
||||
5gnidd,0.5,2,10,4,SYNScan,10,,5,4830,14030
|
||||
5gnidd,0.5,2,10,5,SlowrateDoS,10,,104,17642,51187
|
||||
5gnidd,0.5,2,10,6,TCPConnectScan,9,6,125,6952,14037
|
||||
5gnidd,0.5,2,10,7,UDPFlood,10,,105,138381,320138
|
||||
5gnidd,0.5,2,10,8,UDPScan,9,0,2,6727,11134
|
||||
5gnidd,0.75,0,10,0,Benign,10,,1410,69384,334415
|
||||
5gnidd,0.75,0,10,1,HTTPFlood,10,,557,27843,98569
|
||||
5gnidd,0.75,0,10,2,ICMPFlood,8,0|3,25,239,808
|
||||
5gnidd,0.75,0,10,3,SYNFlood,9,4,26,3213,6805
|
||||
5gnidd,0.75,0,10,4,SYNScan,10,,82,4335,14030
|
||||
5gnidd,0.75,0,10,5,SlowrateDoS,10,,2,15285,51187
|
||||
5gnidd,0.75,0,10,6,TCPConnectScan,10,,52,3531,14037
|
||||
5gnidd,0.75,0,10,7,UDPFlood,10,,2320,124158,320138
|
||||
5gnidd,0.75,0,10,8,UDPScan,10,,74,3370,11134
|
||||
5gnidd,0.75,1,10,0,Benign,10,,3896,83388,334415
|
||||
5gnidd,0.75,1,10,1,HTTPFlood,10,,621,25248,98569
|
||||
5gnidd,0.75,1,10,2,ICMPFlood,9,3,15,285,808
|
||||
5gnidd,0.75,1,10,3,SYNFlood,10,,57,2358,6805
|
||||
5gnidd,0.75,1,10,4,SYNScan,9,6,133,3883,14030
|
||||
5gnidd,0.75,1,10,5,SlowrateDoS,10,,514,12114,51187
|
||||
5gnidd,0.75,1,10,6,TCPConnectScan,10,,326,2861,14037
|
||||
5gnidd,0.75,1,10,7,UDPFlood,10,,1031,114825,320138
|
||||
5gnidd,0.75,1,10,8,UDPScan,10,,5,2727,11134
|
||||
5gnidd,0.75,2,10,0,Benign,10,,536,118638,334415
|
||||
5gnidd,0.75,2,10,1,HTTPFlood,10,,1484,25410,98569
|
||||
5gnidd,0.75,2,10,2,ICMPFlood,10,,1,270,808
|
||||
5gnidd,0.75,2,10,3,SYNFlood,10,,6,2670,6805
|
||||
5gnidd,0.75,2,10,4,SYNScan,10,,91,4022,14030
|
||||
5gnidd,0.75,2,10,5,SlowrateDoS,10,,99,18020,51187
|
||||
5gnidd,0.75,2,10,6,TCPConnectScan,10,,84,3335,14037
|
||||
5gnidd,0.75,2,10,7,UDPFlood,10,,2637,63416,320138
|
||||
5gnidd,0.75,2,10,8,UDPScan,10,,37,3282,11134
|
||||
xiiotid,0.25,0,10,0,C&C,10,,21,1065,2004
|
||||
xiiotid,0.25,0,10,1,Exfiltration,8,1|4,4,7950,15494
|
||||
xiiotid,0.25,0,10,2,Exploitation,6,1|2|5|9,2,595,793
|
||||
xiiotid,0.25,0,10,3,Lateral_movement,9,6,2,13857,22117
|
||||
xiiotid,0.25,0,10,4,Normal,10,,1,122800,292777
|
||||
xiiotid,0.25,0,10,5,RDOS,10,,166,31089,98883
|
||||
xiiotid,0.25,0,10,6,Reconnaissance,8,3|6,46,65307,89161
|
||||
xiiotid,0.25,0,10,7,Tampering,8,3|9,9,2272,3586
|
||||
xiiotid,0.25,0,10,8,Weaponization,8,1|4,43,23381,45094
|
||||
xiiotid,0.25,1,10,0,C&C,9,2,1,986,2004
|
||||
xiiotid,0.25,1,10,1,Exfiltration,10,,1,6204,15494
|
||||
xiiotid,0.25,1,10,2,Exploitation,8,2|6,1,321,793
|
||||
xiiotid,0.25,1,10,3,Lateral_movement,10,,1,10094,22117
|
||||
xiiotid,0.25,1,10,4,Normal,8,2|6,198,129678,292777
|
||||
xiiotid,0.25,1,10,5,RDOS,9,2,168,53482,98883
|
||||
xiiotid,0.25,1,10,6,Reconnaissance,8,2|8,138,65101,89161
|
||||
xiiotid,0.25,1,10,7,Tampering,9,4,2,1804,3586
|
||||
xiiotid,0.25,1,10,8,Weaponization,9,3,1,12024,45094
|
||||
xiiotid,0.25,2,10,0,C&C,6,0|3|5|6,2,1265,2004
|
||||
xiiotid,0.25,2,10,1,Exfiltration,8,4|9,4,8269,15494
|
||||
xiiotid,0.25,2,10,2,Exploitation,6,0|3|5|8,1,485,793
|
||||
xiiotid,0.25,2,10,3,Lateral_movement,7,0|6|9,1,12496,22117
|
||||
xiiotid,0.25,2,10,4,Normal,9,2,452,118937,292777
|
||||
xiiotid,0.25,2,10,5,RDOS,10,,2,48839,98883
|
||||
xiiotid,0.25,2,10,6,Reconnaissance,10,,1,46541,89161
|
||||
xiiotid,0.25,2,10,7,Tampering,10,,1,2584,3586
|
||||
xiiotid,0.25,2,10,8,Weaponization,9,4,4,22853,45094
|
||||
xiiotid,0.5,0,10,0,C&C,10,,1,966,2004
|
||||
xiiotid,0.5,0,10,1,Exfiltration,9,6,45,6305,15494
|
||||
xiiotid,0.5,0,10,2,Exploitation,8,0|6,10,353,793
|
||||
xiiotid,0.5,0,10,3,Lateral_movement,10,,49,8385,22117
|
||||
xiiotid,0.5,0,10,4,Normal,10,,604,68657,292777
|
||||
xiiotid,0.5,0,10,5,RDOS,10,,168,49283,98883
|
||||
xiiotid,0.5,0,10,6,Reconnaissance,10,,1137,29234,89161
|
||||
xiiotid,0.5,0,10,7,Tampering,9,0,2,1266,3586
|
||||
xiiotid,0.5,0,10,8,Weaponization,10,,18,32409,45094
|
||||
xiiotid,0.5,1,10,0,C&C,8,2|6,2,687,2004
|
||||
xiiotid,0.5,1,10,1,Exfiltration,10,,8,6273,15494
|
||||
xiiotid,0.5,1,10,2,Exploitation,9,1,1,491,793
|
||||
xiiotid,0.5,1,10,3,Lateral_movement,10,,49,8361,22117
|
||||
xiiotid,0.5,1,10,4,Normal,10,,260,89253,292777
|
||||
xiiotid,0.5,1,10,5,RDOS,10,,6,47276,98883
|
||||
xiiotid,0.5,1,10,6,Reconnaissance,10,,1842,23967,89161
|
||||
xiiotid,0.5,1,10,7,Tampering,9,6,8,1228,3586
|
||||
xiiotid,0.5,1,10,8,Weaponization,10,,917,11526,45094
|
||||
xiiotid,0.5,2,10,0,C&C,9,2,13,576,2004
|
||||
xiiotid,0.5,2,10,1,Exfiltration,10,,20,5300,15494
|
||||
xiiotid,0.5,2,10,2,Exploitation,10,,1,298,793
|
||||
xiiotid,0.5,2,10,3,Lateral_movement,10,,11,6341,22117
|
||||
xiiotid,0.5,2,10,4,Normal,10,,79,92031,292777
|
||||
xiiotid,0.5,2,10,5,RDOS,10,,4,48860,98883
|
||||
xiiotid,0.5,2,10,6,Reconnaissance,10,,208,22975,89161
|
||||
xiiotid,0.5,2,10,7,Tampering,10,,26,2437,3586
|
||||
xiiotid,0.5,2,10,8,Weaponization,10,,17,23228,45094
|
||||
xiiotid,0.75,0,10,0,C&C,8,2|4,5,760,2004
|
||||
xiiotid,0.75,0,10,1,Exfiltration,10,,27,4392,15494
|
||||
xiiotid,0.75,0,10,2,Exploitation,9,9,1,240,793
|
||||
xiiotid,0.75,0,10,3,Lateral_movement,10,,190,5435,22117
|
||||
xiiotid,0.75,0,10,4,Normal,10,,1151,92460,292777
|
||||
xiiotid,0.75,0,10,5,RDOS,10,,347,22821,98883
|
||||
xiiotid,0.75,0,10,6,Reconnaissance,10,,458,23824,89161
|
||||
xiiotid,0.75,0,10,7,Tampering,10,,31,1227,3586
|
||||
xiiotid,0.75,0,10,8,Weaponization,10,,285,12915,45094
|
||||
xiiotid,0.75,1,10,0,C&C,9,8,87,538,2004
|
||||
xiiotid,0.75,1,10,1,Exfiltration,10,,239,3749,15494
|
||||
xiiotid,0.75,1,10,2,Exploitation,10,,4,246,793
|
||||
xiiotid,0.75,1,10,3,Lateral_movement,10,,556,4082,22117
|
||||
xiiotid,0.75,1,10,4,Normal,10,,4311,73956,292777
|
||||
xiiotid,0.75,1,10,5,RDOS,10,,268,34051,98883
|
||||
xiiotid,0.75,1,10,6,Reconnaissance,10,,520,22127,89161
|
||||
xiiotid,0.75,1,10,7,Tampering,10,,25,1004,3586
|
||||
xiiotid,0.75,1,10,8,Weaponization,10,,269,10726,45094
|
||||
xiiotid,0.75,2,10,0,C&C,10,,1,546,2004
|
||||
xiiotid,0.75,2,10,1,Exfiltration,10,,9,3572,15494
|
||||
xiiotid,0.75,2,10,2,Exploitation,10,,11,382,793
|
||||
xiiotid,0.75,2,10,3,Lateral_movement,10,,3,5709,22117
|
||||
xiiotid,0.75,2,10,4,Normal,10,,1930,64353,292777
|
||||
xiiotid,0.75,2,10,5,RDOS,10,,636,18655,98883
|
||||
xiiotid,0.75,2,10,6,Reconnaissance,10,,1528,21418,89161
|
||||
xiiotid,0.75,2,10,7,Tampering,10,,1,1321,3586
|
||||
xiiotid,0.75,2,10,8,Weaponization,10,,169,17328,45094
|
||||
|
+10
@@ -0,0 +1,10 @@
|
||||
class_id,M,S_j,missing_clients,N_ij_contributors,displacement_Sj_vs_literal_1M_zero,mean_norm_Cbar_Sj,mean_contributor_norm,hypothetical
|
||||
0,10,10,,126|559|92|966|1|4|44|20|4|188,0.0,5.81850391528672,6.47624817952452,True
|
||||
1,10,9,6,901|2212|6305|1509|1151|554|2414|45|403,0.7242890510522884,7.242890510522882,8.181070757376908,True
|
||||
2,10,8,0|6,69|353|140|18|53|10|138|12,1.13084867021236,5.654243351061799,6.407363782365948,True
|
||||
3,10,10,,5915|180|949|630|1747|315|588|8385|3359|49,0.0,5.571182640715375,6.320120476548096,True
|
||||
4,10,10,,48503|34165|604|13892|733|11521|34309|57789|68657|22604,0.0,6.0422850094072285,6.51180863394013,True
|
||||
5,10,10,,9867|6640|10123|49283|168|1150|5683|9870|5008|1091,0.0,8.705113176029744,9.077145119705389,True
|
||||
6,10,10,,29234|5054|1636|1137|14033|27596|3655|2431|1662|2723,0.0,6.113286531206159,6.597388552754765,True
|
||||
7,10,9,0,1266|5|832|336|2|93|506|385|161,0.5095948820849879,5.0959488208498795,6.110935859052799,True
|
||||
8,10,10,,18|486|32409|1220|334|691|713|2971|5815|437,0.0,6.165797271067824,7.069469485204846,True
|
||||
|
+11313
File diff suppressed because it is too large.
Load diff
@@ -8,6 +8,7 @@ sweep was launched.
|
||||
|
||||
Companion documents:
|
||||
|
||||
- `docs/baseline/BASELINE_VALIDATION.md` — Session-3 validation and reproduction evidence.
|
||||
- `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md` — evidence for baseline semantics.
|
||||
- `docs/research/EXPERIMENT_PROTOCOL.md` — protocol and provenance tags.
|
||||
- `docs/research/IMPLEMENTATION_PLAN.md` — architecture.
|
||||
@@ -169,13 +170,19 @@ and hash, so manifests stay small.
|
||||
|
||||
## 11. Tests
|
||||
|
||||
`python -m pytest -q` -> **39 tests** (see run result in the handoff). Coverage includes
|
||||
`python -m pytest -q` -> **46 tests** (Session-3 count; the Session-2 handoff reported 34
|
||||
and an earlier draft of this file said 39, which was inaccurate). Coverage includes
|
||||
dataset config resolution, deterministic label mapping, deterministic/train-only
|
||||
preprocessing, deterministic stratified split, deterministic Dirichlet partition,
|
||||
manifest reload identity and corruption detection, rare/unseen semantics, local prototype
|
||||
means, baseline aggregation (including that `support_weighted` is not implemented), metrics
|
||||
on known fixtures, and a small end-to-end federated smoke test.
|
||||
|
||||
`tests/test_baseline_validation.py` (Session 3) adds round-0 handling, an independent
|
||||
replication of the documented loss formula, frozen-coefficient guards, equal-weight model
|
||||
averaging, inference/class-mapping checks, and a deterministic end-to-end regression test.
|
||||
See `docs/baseline/BASELINE_VALIDATION.md`.
|
||||
|
||||
## 12. Smoke test
|
||||
|
||||
```
|
||||
@@ -201,10 +208,18 @@ manifests -> `artifacts/partitions/`.
|
||||
## 14. Remaining assumptions and unresolved issues
|
||||
|
||||
1. **Participant rule** (`S_j`, equal weights) is a reproduction assumption (see §7).
|
||||
Session-3 validation quantified it: `|S_j| < M` in 53/162 class-condition cells
|
||||
(32.7 %) across 15/18 conditions, and the hypothetical zero-filled `1/M` reading is
|
||||
displaced by ~10–20 % of the prototype norm for the affected classes
|
||||
(`docs/baseline/BASELINE_VALIDATION.md` §D). It remains unresolved, not disproven.
|
||||
2. `lambda = 1.0`, optimiser and schedule are unverified against the paper.
|
||||
3. Prototype-alignment form and round-0 handling are inferred.
|
||||
4. Per-client evaluation partitioning (val/test assigned to clients via Dirichlet) is a
|
||||
project decision; the paper does not describe it.
|
||||
5. No full-scale run has been timed; compute feasibility of a real sweep is unmeasured.
|
||||
6. Whether the literal `1/M` reading differs from the `S_j` assumption is open and must
|
||||
be tested with a deliberate control in the verification phase.
|
||||
5. Full-scale compute is now measured: X-IIoTID (`alpha=0.5`, seed 0) 42:07 wall /
|
||||
6.21 GiB peak, 5G-NIDD 1:04:52 wall / 10.37 GiB peak, CPU-only (see
|
||||
`docs/baseline/BASELINE_VALIDATION.md` §B).
|
||||
6. Whether the literal `1/M` reading differs from the `S_j` assumption is open. It cannot
|
||||
be settled without original code (none exists); the only feasible treatments are
|
||||
documented interpretation choices, and the zero-fill variant was analysed as a
|
||||
hypothetical artifact only.
|
||||
@@ -0,0 +1,430 @@
|
||||
# Baseline Validation
|
||||
|
||||
Status: baseline validation and reproduction verification (Session 3, 2026-09-24).
|
||||
Scope: determine whether the frozen `original_protean` baseline behaves coherently,
|
||||
reproducibly and consistently with the best available PROTEAN evidence, and identify
|
||||
what is still a reproduction assumption. **No beta / support-weighted / `equal_available`
|
||||
functionality was added.**
|
||||
|
||||
Companion documents:
|
||||
|
||||
- `docs/baseline/BASELINE_IMPLEMENTATION.md` — what the baseline implements.
|
||||
- `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md` — paper/thesis evidence.
|
||||
- `docs/research/EXPERIMENT_PROTOCOL.md` — frozen protocol.
|
||||
- `docs/handoff/SESSION_HANDOFF.md` — continuation point.
|
||||
|
||||
---
|
||||
|
||||
## A. Validation status
|
||||
|
||||
Three different questions must not be conflated:
|
||||
|
||||
1. **Implementation correctness** (does the code do what the frozen protocol says,
|
||||
deterministically and without leakage?). **Validated** for the checks below.
|
||||
2. **Reproduction evidence** (does the code reproduce original PROTEAN?). **Not
|
||||
established**, because no official PROTEAN implementation exists
|
||||
(`PROTEAN_BASELINE_INVESTIGATION.md` §2) and the paper is internally inconsistent
|
||||
about the prototype participant rule.
|
||||
3. **Unresolved assumptions** (what remains a documented reproduction assumption?).
|
||||
Listed in §G; the central one is the `S_j` participant/denominator rule (§D).
|
||||
|
||||
| Validation item | Result | Evidence |
|
||||
|---|---|---|
|
||||
| Full-data run completes | PASS | §B |
|
||||
| Partition consistency | PASS | §B, §D |
|
||||
| Global / per-client / per-class inspection | PASS (reviewed) | §C |
|
||||
| `S_j` ambiguity quantified | PASS (quantified, not resolved) | §D |
|
||||
| Round-0 + prototype-alignment entry | PASS | §E |
|
||||
| Loss implementation audit | PASS | §E |
|
||||
| Inference validation | PASS | §E |
|
||||
| Deterministic end-to-end regression test | PASS | §F |
|
||||
| Repeat-run reproducibility | PASS | §F |
|
||||
| Total test suite | PASS (46 tests) | §F |
|
||||
|
||||
---
|
||||
|
||||
## B. Full-data run
|
||||
|
||||
### B.1 Measured condition (primary)
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| dataset | `xiiotid` (X-IIoTID) |
|
||||
| alpha | 0.50 |
|
||||
| seed | 0 |
|
||||
| aggregation | `original_protean` |
|
||||
| num_clients | 10 |
|
||||
| rounds / local epochs / batch / lr / mu / lambda | 10 / 3 / 256 / 1e-3 / 0.1 / 1.0 |
|
||||
| device | CPU (`torch 2.14.0+cpu`, CUDA unavailable) |
|
||||
| run dir | `results/xiiotid__alpha0.5__seed0__original_protean/` |
|
||||
| manifest | `artifacts/partitions/xiiotid__alpha0.5__seed0.json` (reused, not regenerated) |
|
||||
| manifest content hash | `e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c` |
|
||||
| preprocessing hash | `8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a` |
|
||||
| snapshot | `artifacts/analysis/xiiotid__alpha0.5__seed0__original_protean__prototype_snapshot.json` |
|
||||
|
||||
### B.2 Time and resource measurements (`/usr/bin/time -v`)
|
||||
|
||||
| Measure | Value |
|
||||
|---|---|
|
||||
| Wall-clock (incl. load + preprocessing + 10 rounds + eval) | **42:06.78** |
|
||||
| User CPU time | 18231.44 s |
|
||||
| System CPU time | 481.31 s |
|
||||
| CPU utilisation | 740 % |
|
||||
| Peak resident set size (host) | **6,509,448 KB ≈ 6.21 GiB** |
|
||||
| GPU memory | n/a (CPU-only) |
|
||||
| Swaps | 0 |
|
||||
| Exit status | 0 |
|
||||
|
||||
Separate component timings were not instrumented inside the runner. A calibration on a
|
||||
50,000-row subsample measured `raw read ≈ 0.9 s`, `prepare_dataset ≈ 1.3 s` and
|
||||
`≈ 0.163 ms` per sample-epoch (input_dim = 78 there), i.e. preprocessing is negligible
|
||||
next to training. No retries or client failures occurred; all 10 clients remained active
|
||||
in all 10 rounds (the smallest client, id 4, still had 18,521 training samples).
|
||||
|
||||
### B.3 Global test metrics
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| accuracy | 0.8904 |
|
||||
| macro-F1 | 0.7077 |
|
||||
| weighted-F1 | 0.9176 |
|
||||
| rare-class macro-F1 | 0.6096 |
|
||||
| rare-class recall | 0.8715 |
|
||||
| rare-class evaluation support | 25,454 |
|
||||
|
||||
Validation trajectory (history): accuracy rises to ~0.88 by round 3 and then plateaus
|
||||
(0.881–0.889); macro-F1 plateaus ~0.70; rare-class macro-F1 plateaus ~0.59–0.62. The
|
||||
curve is stable with mild round-to-round fluctuation, consistent with a converged (not
|
||||
diverging or collapsed) baseline.
|
||||
|
||||
### B.4 Second-dataset sanity run (5G-NIDD)
|
||||
|
||||
The optional second full-data condition was run to confirm the full-data path works on
|
||||
both datasets. Same frozen settings, `original_protean`, 10 clients / 10 rounds / 3 local
|
||||
epochs.
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| dataset | `5gnidd` (5G-NIDD) |
|
||||
| alpha / seed | 0.50 / 0 |
|
||||
| run dir | `results/5gnidd__alpha0.5__seed0__original_protean/` |
|
||||
| manifest content hash | `0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6` (reused, reload-verified) |
|
||||
| preprocessing hash | `ff8bc0e52967a2a081392282ae62ec86bc3f2b88648a9fa20de43fb0e7e2cc3a` |
|
||||
| wall-clock | **1:04:52** |
|
||||
| CPU / RSS | 739 % / **10,875,780 KB ≈ 10.37 GiB** |
|
||||
| accuracy | 0.7344 |
|
||||
| macro-F1 | 0.8369 |
|
||||
| weighted-F1 | 0.7088 |
|
||||
| rare-class macro-F1 | 0.8720 |
|
||||
| rare-class recall | 0.9743 |
|
||||
| exit status | 0 (10/10 rounds, no failures) |
|
||||
|
||||
Notable per-class behaviour: `Benign` (support 95,547) has precision 1.000 but recall
|
||||
only 0.380, while `UDPFlood` (support 91,468) has recall 1.000 and precision 0.616 — i.e.
|
||||
the nearest-prototype classifier over-predicts `UDPFlood` for many benign flows. This is
|
||||
why macro-F1 (0.837) exceeds weighted-F1 (0.709). As with X-IIoTID this is plausible
|
||||
nearest-prototype behaviour under non-IID data, not an obvious anomaly; 9/9 prototypes
|
||||
were present and no class had zero predictions. Per-client accuracy ranges 0.537–0.991.
|
||||
|
||||
Both full-data conditions therefore complete the frozen path end to end, reuse their
|
||||
frozen manifests, and produce finite, structurally explainable metrics.
|
||||
|
||||
---
|
||||
|
||||
## C. Per-client / per-class inspection
|
||||
|
||||
### C.1 Per-class test behaviour (X-IIoTID α=0.5 seed 0)
|
||||
|
||||
| class | support | precision | recall | F1 | note |
|
||||
|---|---|---|---|---|---|
|
||||
| C&C | 573 | 0.092 | 0.990 | 0.169 | very low precision; over-predicted |
|
||||
| Exfiltration | 4,427 | 1.000 | 0.952 | 0.975 | strong |
|
||||
| Exploitation | 227 | 0.019 | 0.581 | 0.037 | very low precision; over-predicted |
|
||||
| Lateral_movement | 6,319 | 0.964 | 0.727 | 0.829 | good precision, moderate recall |
|
||||
| Normal | 83,651 | 0.971 | 0.928 | 0.949 | strong |
|
||||
| RDOS | 28,252 | 0.992 | 0.997 | 0.995 | strong |
|
||||
| Reconnaissance | 25,474 | 0.992 | 0.627 | 0.768 | low recall |
|
||||
| Tampering | 1,024 | 0.563 | 0.992 | 0.718 | recall high, precision moderate |
|
||||
| Weaponization | 12,884 | 0.879 | 0.987 | 0.930 | strong |
|
||||
|
||||
The confusion matrix shows the dominant errors are structural, not random: 5,181 true
|
||||
`Normal` samples are predicted `C&C` (which is why `C&C` precision collapses), and 5,683
|
||||
true `Reconnaissance` samples are predicted `Exploitation`. The lowest-support classes
|
||||
(`C&C` n=573, `Exploitation` n=227) are the ones whose nearest-prototype precision
|
||||
collapses toward the large `Normal`/`Reconnaissance` prototypes. This is **plausible
|
||||
non-IID / nearest-prototype difficulty with scarce support**, not an obvious
|
||||
implementation anomaly: recalls are high, predictions are finite, no class has zero
|
||||
predictions, and no class is missing from the global prototype set (9/9 present).
|
||||
|
||||
Classification of the low rare-class F1: *insufficient support + prototype separation
|
||||
difficulty under non-IID data*, not a bug. It is **not** interpreted as improved or
|
||||
degraded method behaviour (no method comparison has been run).
|
||||
|
||||
### C.2 Per-client behaviour (X-IIoTID α=0.5 seed 0)
|
||||
|
||||
Per-client test metrics vary with each client's local eval partition:
|
||||
|
||||
- accuracy 0.691–0.986; macro-F1 0.552–0.840.
|
||||
- clients 4 (acc 0.691, rare macro-F1 0.135) and 7 (rare macro-F1 0.045) are the weakest.
|
||||
- clients 2 (rare macro-F1 0.876) and 5 (0.698) are the strongest.
|
||||
- 10/10 clients evaluated; no empty client eval set.
|
||||
|
||||
Per-client training structure (from the manifest):
|
||||
|
||||
| client | train samples | observed classes | unseen (attack) classes | rare classes (2 smallest positive attack support) |
|
||||
|---|---|---|---|---|
|
||||
| 0 | 94,564 | 7 | 2 | Weaponization, C&C |
|
||||
| 1 | 50,631 | 9 | 0 | Exploitation, Lateral_movement |
|
||||
| 2 | 52,476 | 9 | 0 | Tampering, C&C |
|
||||
| 3 | 69,609 | 9 | 0 | Exploitation, Lateral_movement |
|
||||
| 4 | 18,521 | 9 | 0 | C&C, Exploitation |
|
||||
| 5 | 41,886 | 9 | 0 | Tampering, C&C |
|
||||
| 6 | 45,085 | 7 | 2 | C&C, Tampering |
|
||||
| 7 | 84,396 | 9 | 0 | Exploitation, C&C |
|
||||
| 8 | 85,073 | 9 | 0 | C&C, Exfiltration |
|
||||
| 9 | 27,668 | 9 | 0 | Exploitation, Lateral_movement |
|
||||
|
||||
Only clients 0 and 6 are missing attack classes (`Exploitation` and `Tampering`/`Exfiltration`),
|
||||
i.e. under this middle-alpha condition most clients see all classes, but two do not.
|
||||
|
||||
---
|
||||
|
||||
## D. `S_j` participant-rule ambiguity analysis
|
||||
|
||||
### D.1 The question
|
||||
|
||||
Paper/thesis Eq. (2)/(5.5) writes `Cbar_j = (1/M) * sum_i C_ij` over all `M` clients,
|
||||
while the thesis also states prototypes are shared only for locally observed classes.
|
||||
The paper never defines how `C_ij` is treated when `N_ij = 0`. The baseline therefore
|
||||
assumes:
|
||||
|
||||
```
|
||||
S_j = { i | N_ij > 0 and a valid C_ij exists }
|
||||
Cbar_j = mean_{i in S_j} C_ij (equal 1/|S_j| weights)
|
||||
```
|
||||
|
||||
This remains a **REPRODUCTION ASSUMPTION**, not verified original behaviour.
|
||||
|
||||
### D.2 How often does `|S_j| < M` actually occur?
|
||||
|
||||
Across all 18 frozen manifests (`artifacts/analysis/sj_participant_rule_analysis.csv`):
|
||||
|
||||
- **15 of 18** `(dataset, alpha, seed)` conditions contain at least one class with `|S_j| < M`.
|
||||
- **53 of 162** class-condition cells (32.7 %) have `|S_j| < M`.
|
||||
- Affected classes are not only exotic ones: `Exploitation` (6 conditions), `C&C` (6),
|
||||
`ICMPFlood` (5), `SYNFlood` (5), `TCPConnectScan` (4), `Tampering` (4), and even the
|
||||
**benign** classes `Normal` (2) and `Benign` (implicit where listed among the 5G classes).
|
||||
- The effect is largest at `alpha = 0.25` (up to 7/9 classes missing a client) and can be
|
||||
zero at `alpha = 0.75` for some seeds.
|
||||
|
||||
Conclusion: the ambiguity is **material, not rare**. A literal `1/M` reading differs from
|
||||
the `S_j` reading for a large fraction of class-condition combinations.
|
||||
|
||||
### D.3 Direct prototype displacement (fixed `C_ij`, no retraining)
|
||||
|
||||
From the round-0 snapshot (fixed `C_ij`, `N_ij`) for the measured condition, comparing the
|
||||
current `S_j` mean against one hypothetical literal reading (**missing prototype replaced
|
||||
by the zero vector**, then divided by `M`):
|
||||
|
||||
| class | |S_j| | missing clients | ||Cbar_Sj − Cbar_literal1M,zero|| | ||Cbar_Sj|| |
|
||||
|---|---|---|---|---|
|
||||
| C&C | 10 | — | 0.000 | 5.819 |
|
||||
| Exfiltration | 9 | 6 | 0.724 | 7.243 |
|
||||
| Exploitation | 8 | 0, 6 | 1.131 | 5.654 |
|
||||
| Lateral_movement | 10 | — | 0.000 | 5.571 |
|
||||
| Normal | 10 | — | 0.000 | 6.042 |
|
||||
| RDOS | 10 | — | 0.000 | 8.705 |
|
||||
| Reconnaissance | 10 | — | 0.000 | 6.113 |
|
||||
| Tampering | 9 | 0 | 0.510 | 5.096 |
|
||||
| Weaponization | 10 | — | 0.000 | 6.166 |
|
||||
|
||||
For the three affected classes the hypothetical zero-filled `1/M` prototype is displaced
|
||||
by roughly 10–20 % of the prototype norm. Because inference is nearest-prototype in raw
|
||||
embedding space (not scale invariant), such displacement can change predictions for
|
||||
samples near class boundaries.
|
||||
|
||||
**Important caveats** (why this does not resolve the ambiguity):
|
||||
|
||||
- The zero-filled `1/M` prototype is only **one** hypothetical placeholder and is labelled
|
||||
hypothetical/analysis-only
|
||||
(`artifacts/analysis/sj_vs_literal_1m_hypothetical_xiiotid__alpha0.5__seed0__original_protean.csv`).
|
||||
Reproduced by the read-only script `scripts/validate_sj_participant_rule.py`.
|
||||
It was **not** used for training and is **not** an experiment mode.
|
||||
- Other equally-unspecified readings exist (retain previous round's prototype, skip the
|
||||
missing client but still divide by `M`, use the global prototype, etc.) and would give
|
||||
different answers.
|
||||
- The thesis statement "prototypes are shared only for locally observed classes" is the
|
||||
strongest available evidence for the `S_j` reading; the literal `1/M` equation is the
|
||||
strongest evidence for a denominator of `M`, but cannot be evaluated without an arbitrary
|
||||
placeholder.
|
||||
|
||||
### D.4 Defensibility of the current choice
|
||||
|
||||
The current `S_j` equal-weight rule remains **defensible as an explicitly documented
|
||||
reproduction assumption**: it is the only reading consistent with the thesis's stated
|
||||
sharing rule, and it avoids fabricating prototypes. It is **not** verified original
|
||||
behaviour, and the displacement analysis shows the choice is not immaterial. This project
|
||||
therefore continues to label `original_protean` a *documented interpretation*, and does
|
||||
**not** claim paper-reproduction success.
|
||||
|
||||
The `equal_available` control remains **unnecessary**: the baseline aggregation already
|
||||
uses equal weights over `S_j`, so it is arithmetically identical to the proposed `beta = 0`
|
||||
rule; introducing a separate mode would add no information. It stays deferred.
|
||||
|
||||
---
|
||||
|
||||
## E. Round-0, loss and inference validation
|
||||
|
||||
### E.1 Round-0 handling
|
||||
|
||||
Verified by `test_round0_sees_no_global_prototypes_and_round1_does`: during round 0 every
|
||||
client receives **no** global prototypes (empty), so the prototype-alignment term is
|
||||
skipped; after round-0 server aggregation the first global prototypes are broadcast back
|
||||
and used from round 1 onward. The round-0 snapshot stores the pre-aggregation client
|
||||
prototypes. Provenance: **INFERRED / PROJECT DECISION** (the paper's Algorithms say
|
||||
"initialize the global prototype set" but never define it).
|
||||
|
||||
### E.2 Loss implementation audit
|
||||
|
||||
The implemented local objective is (client.py):
|
||||
|
||||
```
|
||||
CE(logits, y) # nll_loss, mean over the batch
|
||||
+ lambda * ( sum_{matched samples x} ||phi(x) - Cbar_y||^2 / count_matched )
|
||||
+ (mu / 2) * sum_{params} ||w - w_global||^2
|
||||
```
|
||||
|
||||
Findings:
|
||||
|
||||
- **classification loss**: `F.nll_loss` on log-softmax outputs, default mean reduction.
|
||||
- **prototype alignment**: per **sample**, only for classes that currently have a global
|
||||
prototype; normalised by the number of matched samples in the batch (a mean), *not* a
|
||||
raw sum. Provenance **INFERRED** (FedProto-style reading of Eq. 3; the paper writes a
|
||||
class-mean-to-class-mean term).
|
||||
- **model alignment**: sum over parameter tensors of squared deviation from the round's
|
||||
global parameters, coefficient `mu/2`, `mu = 0.1` (**PAPER**).
|
||||
- **coefficients**: `lambda = 1.0`, `mu = 0.1`; guarded by
|
||||
`test_training_config_coefficients_are_frozen`.
|
||||
- **gradients flow** to the shared model parameters through all three terms (verified
|
||||
implicitly by parameter-change comparison).
|
||||
- **missing prototypes**: a label whose class has no global prototype contributes **no**
|
||||
alignment term and does **not** enter the normaliser.
|
||||
|
||||
`test_local_update_matches_documented_loss_formula` independently re-transcribes the
|
||||
documented formula and reproduces `local_update` parameters to `atol=1e-5`, pinning the
|
||||
reductions and coefficients. `test_prototype_alignment_skips_classes_without_a_global_prototype`
|
||||
confirms classes without a global prototype are skipped.
|
||||
|
||||
### E.3 Inference validation
|
||||
|
||||
- Rule: embedding `z = phi(x)`, prediction `argmin_j ||z - Cbar_j||^2` computed via the
|
||||
expanded-norm identity; class labels returned are the actual class ids
|
||||
(`test_nearest_prototype_predict_maps_noncontiguous_class_ids`).
|
||||
- `test_nearest_prototype_predict_matches_squared_l2_bruteforce` matches an independent
|
||||
brute-force squared-L2 argmin over 20 random points and 3 prototypes.
|
||||
- Empty global prototype sets raise an explicit `ValueError` from both
|
||||
`nearest_prototype_predict` and `server.predict_global`; global prototypes are never
|
||||
fabricated (`test_collect_global_omits_missing_class`).
|
||||
- `test_compute_local_prototypes_class_index_mapping` confirms local prototypes are keyed
|
||||
by true class id and only for observed classes.
|
||||
- Provenance: **PAPER** for the rule (thesis Eqs. 5.11–5.13); "classes considered = those
|
||||
with a global prototype" is **INFERRED**.
|
||||
|
||||
Minor note: the expanded-norm distance can be very slightly negative due to float
|
||||
rounding; `argmin` is unaffected and no sample observed a wrong nearest prototype under
|
||||
the brute-force cross-check.
|
||||
|
||||
---
|
||||
|
||||
## F. Determinism and reproducibility
|
||||
|
||||
### F.1 Repeat-run reproducibility (measured)
|
||||
|
||||
The same smoke condition (`xiiotid`, `alpha=0.5`, `seed=0`, 4,000 rows, 4 clients, 3 rounds,
|
||||
1 local epoch) was run twice into separate output roots. Diff of the two run trees:
|
||||
|
||||
- `history.json`, `test_global_metrics.json`, `test_confusion_matrix.json`,
|
||||
`test_per_class.csv`, `per_client.csv`, `train_support_Nij.csv`,
|
||||
`characterization.csv`, `rare_unseen.csv` and the prototype snapshot: **byte-identical**.
|
||||
- `manifest_content_hash` identical (`b5cf7ff...`); `preprocessing_hash` identical
|
||||
(`70fa831...`); `manifest_reload_ok = true` in both.
|
||||
- Only differences were run-specific metadata: the manifest `created_utc` timestamp and
|
||||
the explicit `manifest_path` (which pointed at the two different output roots). The
|
||||
content hash deliberately excludes `created_utc`/`python`, so partition identity is
|
||||
unchanged.
|
||||
|
||||
### F.2 Determinism source and tolerance
|
||||
|
||||
All randomness is isolated through `utils.seeding.derive_seed` (split, Dirichlet partition,
|
||||
model initialisation, client batch order). PyTorch dropout draws from the global torch RNG;
|
||||
the deterministic regression test reseeds it identically before each run, and two full
|
||||
`run_federated` invocations with the same seed produced **exactly equal** parameters,
|
||||
prototypes and history rows (`test_end_to_end_baseline_is_deterministic_and_learns_separable_fixture`).
|
||||
|
||||
Known nondeterminism source: none observed at the tested scales on this CPU backend. If a
|
||||
future accelerated backend introduces nondeterministic ops, the tolerance clause is the
|
||||
`atol=1e-5 / rtol=1e-4` used in the loss-replication test.
|
||||
|
||||
### F.3 Test suite
|
||||
|
||||
```
|
||||
.venv/bin/python -m pytest -q -> 46 passed
|
||||
```
|
||||
|
||||
34 pre-existing tests + 12 new baseline-validation tests
|
||||
(`tests/test_baseline_validation.py`).
|
||||
|
||||
---
|
||||
|
||||
## G. Remaining unresolved assumptions / blockers
|
||||
|
||||
1. **`S_j` participant/denominator rule** (§D) is a reproduction assumption, quantified but
|
||||
not resolved. No official code exists to settle it. Any between-method conclusion must
|
||||
carry this caveat.
|
||||
2. **`lambda = 1.0`** has no paper value; it is a PROJECT DECISION. Optimiser (Adam),
|
||||
learning rate (1e-3) and batch size (256) are also unspecified by the paper.
|
||||
3. **Prototype-alignment form** (per-sample embedding → class-mean global prototype) is an
|
||||
INFERENCE; the paper writes class-mean → class-mean.
|
||||
4. **Round-0 global-prototype initialisation** is INFERRED/PROJECT DECISION.
|
||||
5. **Model aggregation** is equal `1/M` (PAPER Eq. 2); the thesis intro's FedAvg-style
|
||||
"weighted average" wording conflicts with this and is not implemented (documented
|
||||
conflict, not silently chosen).
|
||||
6. **Per-client evaluation partitioning** (Dirichlet over the test split) is a PROJECT
|
||||
DECISION; the paper does not describe per-client test assignment.
|
||||
7. **Manifest provenance gap**: the Session-2 manifests store `preprocessing_hash = null`
|
||||
and `config_hash = null`, so they do not bind the preprocessing recipe directly. The
|
||||
split hash, partition hash and `N_ij` do reproduce exactly, and the current runner
|
||||
records the live preprocessing hash in run metadata; binding it into future manifests
|
||||
would close this gap.
|
||||
8. **Only the `(alpha=0.50, seed=0)` condition per dataset has been run**; the remaining
|
||||
`(alpha, seed)` grid is unmeasured. No sweep may be launched yet.
|
||||
9. Reproduction success is **not** claimed: two completed runs (one per dataset, one
|
||||
condition each) are consistent with a working baseline, not proof that original PROTEAN
|
||||
is reproduced.
|
||||
|
||||
---
|
||||
|
||||
## Appendix: provenance checklist for the frozen baseline
|
||||
|
||||
Classification: CODE (none exist), PAPER, INFERRED, PROJECT DECISION, UNRESOLVED.
|
||||
|
||||
| Item | Frozen value | Provenance |
|
||||
|---|---|---|
|
||||
| Dataset preprocessing | median-impute + standardise + one-hot, train-fitted, `__MISSING__` category, dup drop, `crypto-ransomware` dropped (X-IIoTID) | PROJECT DECISION (label drop: PAPER) |
|
||||
| Split | stratified 70/10/20, seed-derived | PAPER (80/20 test) + PROJECT DECISION (10 % validation, stratification) |
|
||||
| Label mapping | canonical labels, `Lateral_movement` fix, sorted class ids | PAPER (fix required) + PROJECT DECISION |
|
||||
| Model architecture | 2×Conv1D(64,128)+ReLU+pool+dropout(0.2,0.5), dense 128, log-softmax | PAPER (kernel 3/padding 1: PROJECT DECISION) |
|
||||
| Optimizer / lr / batch | Adam / 1e-3 / 256 | PROJECT DECISION |
|
||||
| Local epochs / rounds | 3 / 10 | PAPER |
|
||||
| mu | 0.1 | PAPER |
|
||||
| lambda | 1.0 | PROJECT DECISION |
|
||||
| Local prototype | end-of-round mean embedding per observed class | PAPER Eq. 1 |
|
||||
| Model aggregation | equal 1/M | PAPER Eq. 2/5.5 |
|
||||
| Prototype aggregation | equal mean over `S_j` | **REPRODUCTION ASSUMPTION** (PROJECT DECISION; paper unresolved) |
|
||||
| Round-0 handling | no prototypes, prototype term skipped | INFERRED / PROJECT DECISION |
|
||||
| Prototype alignment | per-sample, matched classes only, mean over matched | INFERRED |
|
||||
| Inference | nearest global prototype, squared L2, available classes | PAPER (available-class restriction INFERRED) |
|
||||
| Per-client eval partition | Dirichlet over test split | PROJECT DECISION |
|
||||
| Seed policy | `{0,1,2}`, derive_seed per purpose | PROJECT DECISION |
|
||||
|
||||
`planningawal/` was not modified during validation.
|
||||
+182
-196
@@ -3,9 +3,11 @@
|
||||
This is the canonical continuation point for the next agent session. It is written to be
|
||||
readable without prior chat history, terminal scrollback, or git history.
|
||||
|
||||
Last updated: 2026-09-24 (Session 2, baseline-only checkpoint).
|
||||
Session 2 type: minimum reproducible baseline implementation + smoke test, then cleanup so
|
||||
the checkpoint contains **baseline only**. **No full-scale training and no beta sweep.**
|
||||
Last updated: 2026-09-24 (Session 3, baseline validation and reproduction verification).
|
||||
Session 3 type: validation-only. No beta / support-weighted / `equal_available` code was
|
||||
added. Two measured full-data baseline conditions were run; partition consistency, per-
|
||||
class/per-client behaviour, the `S_j` ambiguity, round-0, the loss, inference,
|
||||
reproducibility and a deterministic end-to-end regression test were all checked.
|
||||
|
||||
---
|
||||
|
||||
@@ -22,260 +24,244 @@ S_j = { i | N_ij > 0 and a valid local prototype C_ij exists }
|
||||
beta in {0.0, 0.25, 0.5, 0.75, 1.0}
|
||||
```
|
||||
|
||||
**This checkpoint contains the baseline only.** The proposed beta-weighted aggregation and
|
||||
the `equal_available` control are intentionally NOT implemented yet; they are deferred to
|
||||
the later beta/controls phase, which must not start until the baseline is verified.
|
||||
**The repo still contains the baseline only.** The beta-weighted aggregation and the
|
||||
`equal_available` control are still NOT implemented. Session 3 did not add them.
|
||||
|
||||
## 2. Authoritative source
|
||||
## 2. Authoritative source and immutability
|
||||
|
||||
- `planningawal/PROTEAN_Revisi_Lurus.md` — authoritative research specification.
|
||||
- `planningawal/` is **immutable / read-only**. Verified unchanged:
|
||||
SHA-256 `9ba1d7beb94354db0c4b588199fb1e2823aec8a23be1625ec09a756507fbcb89`
|
||||
(only file in `planningawal/`).
|
||||
- `planningawal/` is immutable. Re-verified unchanged this session:
|
||||
SHA-256 `9ba1d7beb94354db0c4b588199fb1e2823aec8a23be1625ec09a756507fbcb89`.
|
||||
|
||||
## 3. Dataset locations (read-only source data)
|
||||
## 3. Dataset locations (read-only external source data)
|
||||
|
||||
- `~/datasets/X-IIoTID/X-IIoTID-dataset.csv` — SHA-256
|
||||
`7b9290057ee42e784da3c0d84b781815502c9205c74175c96374e71a5ffd98a0` (mtime 2021-10-07)
|
||||
`7b9290057ee42e784da3c0d84b781815502c9205c74175c96374e71a5ffd98a0` (re-verified unchanged).
|
||||
- `~/datasets/5G-NIDD/5G-NIDD.csv` — SHA-256
|
||||
`fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59` (mtime 2023-02-18)
|
||||
`fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59` (re-verified unchanged).
|
||||
|
||||
Re-verified unchanged. Paths resolve from `configs/datasets/datasets.yaml` with env
|
||||
overrides (`PROTEAN_XIIOTID_RAW_PATH`, `PROTEAN_5GNIDD_RAW_PATH`). Raw files are never
|
||||
modified or copied into the repo.
|
||||
## 4. Session 3 objective and outcome
|
||||
|
||||
## 4. Current repository state
|
||||
Objective: determine whether the frozen `original_protean` baseline is coherent,
|
||||
reproducible and consistent with the best available PROTEAN evidence, and explicitly
|
||||
identify unresolved reproduction assumptions. **All acceptance items were met.**
|
||||
|
||||
Baseline code:
|
||||
## 5. Full-data runs completed
|
||||
|
||||
```
|
||||
src/protean_research/
|
||||
├── data/ loading, preprocessing, splits, partition, manifest,
|
||||
│ characterization, rare_unseen, pipeline
|
||||
├── models/ protean_cnn, prototypes
|
||||
├── aggregation/ prototype_aggregation (baseline original_protean ONLY)
|
||||
├── federated/ client, server, round
|
||||
├── evaluation/ metrics
|
||||
└── utils/ config, seeding, integrity
|
||||
scripts/ run_baseline.py, generate_partitions.py
|
||||
tests/ 34 baseline-focused tests
|
||||
artifacts/partitions/ 18 full-data manifests (tracked, small, content-hashed)
|
||||
artifacts/analysis/ 18 characterization + 18 rare_unseen CSVs (tracked, small)
|
||||
artifacts/processed/ ignored: preprocessor state + smoke partitions/snapshots
|
||||
results/ ignored: smoke run outputs
|
||||
```
|
||||
Both with the frozen protocol (`original_protean`, 10 clients, 10 rounds, 3 local epochs,
|
||||
batch 256, lr 1e-3, mu 0.1, lambda 1.0), CPU-only (`torch 2.14.0+cpu`, CUDA unavailable).
|
||||
|
||||
Environment: `.venv/` exists (gitignored) with numpy 2.5.3, pandas 3.0.6,
|
||||
torch 2.14.0+cpu, pyyaml 6.0.3, pytest 9.1.1. Use `.venv/bin/python`.
|
||||
| condition | wall | peak RSS | accuracy | macro-F1 | weighted-F1 | rare macro-F1 | rare recall | exit |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID α=0.50 seed 0 | 42:07 | 6.21 GiB | 0.8904 | 0.7077 | 0.9176 | 0.6096 | 0.8715 | 0 |
|
||||
| 5G-NIDD α=0.50 seed 0 | 1:04:52 | 10.37 GiB | 0.7344 | 0.8369 | 0.7088 | 0.8720 | 0.9743 | 0 |
|
||||
|
||||
## 5. What was completed this session
|
||||
Both re-used their frozen manifest (`manifest_reload_ok = true`), completed all 10 rounds
|
||||
with all 10 clients active and no failures, and produced 9/9 global prototypes. Artifact
|
||||
dirs: `results/xiiotid__alpha0.5__seed0__original_protean/`,
|
||||
`results/5gnidd__alpha0.5__seed0__original_protean/` (ignored by git). Full detail:
|
||||
`docs/baseline/BASELINE_VALIDATION.md`.
|
||||
|
||||
1. Froze every previously-PROVISIONAL protocol value in
|
||||
`configs/experiments/experiment_protocol.yaml` and `docs/research/EXPERIMENT_PROTOCOL.md`.
|
||||
2. Dataset config: explicit `categorical_columns`, `drop_duplicate_rows`, missing tokens,
|
||||
fixed 5G-NIDD `Unnamed: 0` (quoted).
|
||||
3. Data foundation (`data/`): read-only loading, deterministic canonicalisation, leak-free
|
||||
train-fitted preprocessing, deterministic stratified 70/10/20 split, Dirichlet partition
|
||||
with content-hashed manifests and `N_ij`, characterization, rare/unseen metadata.
|
||||
4. Model (`models/`): baseline CNN, parameter averaging, embedding extraction, local
|
||||
prototypes, global prototype collection, nearest-prototype inference.
|
||||
5. Baseline aggregation (`aggregation/`): `original_protean` only. The proposed
|
||||
`support_weighted` and the `equal_available` control were **removed** so the checkpoint
|
||||
is baseline only; an unavailable-mode guard raises `ValueError`.
|
||||
6. Federated loop (`federated/`): local update (CE + prototype alignment + model
|
||||
alignment), server model/prototype aggregation, round orchestration, round-0 snapshot.
|
||||
7. Evaluation (`evaluation/`): accuracy, macro/weighted F1, per-class P/R/F1, rare/unseen.
|
||||
8. Scripts: `generate_partitions.py` (18 manifests, full data, reload-verified);
|
||||
`run_baseline.py` (one baseline condition; no beta/mode options).
|
||||
9. Tests: 34 baseline-only tests, all passing.
|
||||
10. Smoke test: baseline-only on both datasets.
|
||||
11. Docs: `docs/baseline/BASELINE_IMPLEMENTATION.md` rewritten with provenance tags.
|
||||
Resource sizing for the future grid: ~42 min and ~6 GiB for X-IIoTID, ~65 min and ~10 GiB
|
||||
for 5G-NIDD, per `(dataset, alpha, seed)` condition. A full grid
|
||||
`2 datasets × 3 alphas × 3 seeds = 18` conditions is therefore on the order of
|
||||
15–20 CPU-hours plus beta multiplicity; measure before launching anything large.
|
||||
|
||||
## 6. Baseline decisions and provenance
|
||||
## 6. Tests added / test status
|
||||
|
||||
Full detail in `docs/baseline/BASELINE_IMPLEMENTATION.md`. Summary:
|
||||
- New file `tests/test_baseline_validation.py` (12 tests): round-0 handling (spy confirms
|
||||
round 0 receives no global prototypes and round 1 does), independent replication of the
|
||||
documented loss formula (`atol=1e-5`), prototype-alignment skip for classes without a
|
||||
global prototype, frozen-coefficient guard (`lambda=1.0`, `mu=0.1`, rounds=10, epochs=3,
|
||||
batch=256, lr=1e-3), equal-weight model averaging, non-contiguous class-id inference,
|
||||
brute-force squared-L2 cross-check, empty-prototype errors, local prototype class
|
||||
mapping, and a deterministic end-to-end regression test.
|
||||
- Total suite: `.venv/bin/python -m pytest -q` -> **46 passed** (34 pre-existing + 12 new).
|
||||
- Note: `BASELINE_IMPLEMENTATION.md` previously said "39 tests"; the accurate Session-2
|
||||
count was 34. Corrected this session.
|
||||
|
||||
- **No official PROTEAN code exists**, so no decision is source-confirmed by code.
|
||||
- **PAPER**: architecture, `mu = 0.1`, 10 rounds / 3 local epochs, 80/20 test, equal `1/M`
|
||||
model averaging, end-of-round mean-embedding prototypes, nearest-prototype inference.
|
||||
- **REPRODUCTION ASSUMPTION (project decision)**: `original_protean` participant rule is
|
||||
`S_j` with equal `1/|S_j|` weights. The paper writes `1/M` but is ambiguous for classes
|
||||
missing on a client; this assumption follows the thesis sentence that prototypes are
|
||||
shared only for locally observed classes. It is **not** treated as verified.
|
||||
- **PROJECT DECISION**: `lambda = 1.0`; Adam `lr = 1e-3`; batch size 256; kernel size 3;
|
||||
stratified 70/10/20 split with an added validation set; seeds `{0,1,2}`;
|
||||
preprocessing recipe; per-class Dirichlet allocation; per-client eval partitioning.
|
||||
- **INFERRED**: per-sample prototype alignment (FedProto-style reading of Eq. 3); round-0
|
||||
handling (no prototypes, prototype term skipped).
|
||||
- **UNRESOLVED**: whether a literal `1/M` missing-class reading differs from `S_j`.
|
||||
## 7. `S_j` stress-test findings (central methodological result)
|
||||
|
||||
## 7. Seeds (explicit)
|
||||
- `|S_j| < M` occurs in **53/162 class-condition cells (32.7 %)** across **15/18**
|
||||
`(dataset, alpha, seed)` conditions (`artifacts/analysis/sj_participant_rule_analysis.csv`).
|
||||
- Affected classes include high-frequency ones and even the benign class in two conditions;
|
||||
the effect is strongest at `alpha = 0.25`.
|
||||
- On the fixed X-IIoTID round-0 snapshot, the hypothetical literal `1/M` reading with
|
||||
missing prototypes replaced by the zero vector is displaced by **~10–20 % of the
|
||||
prototype norm** for the three affected classes (Exfiltration 0.724, Exploitation 1.131,
|
||||
Tampering 0.510). Hypothetical-only, never used for training:
|
||||
`artifacts/analysis/sj_vs_literal_1m_hypothetical_xiiotid__alpha0.5__seed0__original_protean.csv`,
|
||||
reproducible via `scripts/validate_sj_participant_rule.py`.
|
||||
- Conclusion: the ambiguity is **material, not rare**, but **cannot be resolved** without
|
||||
original code (none exists). The current `S_j` equal-weight rule remains a documented
|
||||
REPRODUCTION ASSUMPTION, defensible from the thesis sentence that prototypes are shared
|
||||
only for locally observed classes. It is **not** verified original behaviour. The
|
||||
`equal_available` control is unnecessary: the baseline already *is* equal-over-`S_j`, so
|
||||
it is arithmetically identical to the proposed `beta = 0` rule.
|
||||
|
||||
- Exact values: **`{0, 1, 2}`**.
|
||||
- Rationale: the paper averages over three random seeds but names none, so the values are
|
||||
**not source-confirmed**. `{0, 1, 2}` is the minimal three-seed set and is an explicit
|
||||
**project decision**.
|
||||
- Each seed deterministically controls the split, Dirichlet partition, model
|
||||
initialisation and batch order, via `protean_research.utils.seeding.derive_seed`.
|
||||
- The set is identical across all baseline conditions.
|
||||
## 8. Round-0, loss and inference validation
|
||||
|
||||
## 8. Manifests (18)
|
||||
- **Round 0**: no global prototypes exist; the prototype-alignment term is skipped; the
|
||||
first global prototypes come from round-0 aggregation and are used from round 1.
|
||||
Provenance INFERRED / PROJECT DECISION. Covered by a focused test.
|
||||
- **Loss**: `nll_loss(mean) + lambda * mean_matched ||phi(x)-Cbar_y||^2 + (mu/2)*sum||w-w_global||^2`.
|
||||
Prototype term is per-sample and only for classes with a global prototype; missing
|
||||
prototypes add no term and do not enter the normaliser. Coefficients `lambda=1.0`
|
||||
(PROJECT DECISION), `mu=0.1` (PAPER). Independently replicated to `atol=1e-5`.
|
||||
- **Inference**: nearest global prototype by squared L2 (expanded norm), returns true
|
||||
class ids, restricts to classes with a global prototype, raises on empty prototype sets.
|
||||
Cross-checked against brute force.
|
||||
|
||||
`artifacts/partitions/` holds **18** manifests from `2 datasets × 3 alphas × 3 seeds`.
|
||||
They are content-hashed, reload-verified, and reused by later variants for pairing. Sample
|
||||
indices are regenerated deterministically from the stored recipe/RNG seed and checked
|
||||
against the stored `N_ij` and hash, keeping manifests small.
|
||||
## 9. Reproducibility findings
|
||||
|
||||
## 9. Verification performed this session
|
||||
The same smoke condition was run twice into separate output roots. All result artifacts
|
||||
(`history.json`, `test_global_metrics.json`, `test_confusion_matrix.json`,
|
||||
`test_per_class.csv`, `per_client.csv`, `train_support_Nij.csv`, `characterization.csv`,
|
||||
`rare_unseen.csv`, prototype snapshot) were **byte-identical**; `manifest_content_hash` and
|
||||
`preprocessing_hash` were identical. Only differences were the run-specific manifest
|
||||
`created_utc` timestamp and the explicit `manifest_path`. The deterministic regression test
|
||||
also produced exactly equal parameters/prototypes/history across two in-process runs.
|
||||
No nondeterminism source was observed on this CPU backend; tolerance clause for future
|
||||
accelerated backends is `atol=1e-5 / rtol=1e-4`.
|
||||
|
||||
- `python -m pytest -q` -> **34 passed**.
|
||||
- Full partition generation on both datasets: 18 manifests, each reload-verified.
|
||||
- Non-IID check (seed 0): X-IIoTID `alpha=0.75` max 1 missing class/client, `alpha=0.25`
|
||||
up to 3; 5G-NIDD similar. Qualitatively consistent with the paper setting.
|
||||
- Baseline-only smoke on X-IIoTID (3000 rows, 4 clients, 2 rounds, 1 epoch): full path
|
||||
succeeds, ~4.5 s.
|
||||
- Baseline-only smoke on 5G-NIDD (4000 rows): succeeds.
|
||||
- No beta/support_weighted code is invoked or present.
|
||||
- Integrity: `planningawal/` and both raw datasets retain their recorded SHA-256 values.
|
||||
## 10. Partition consistency (X-IIoTID α=0.50 seed 0)
|
||||
|
||||
## 10. Files added this session
|
||||
All PASS: manifest reproduces (split hash + partition hash + `N_ij`); splits disjoint and
|
||||
cover the cleaned dataset; each train sample assigned exactly once with no cross-client
|
||||
duplication; `N_ij` matches actual assigned samples; client totals sum to `num_train`
|
||||
(569,909) and class totals match train label counts.
|
||||
|
||||
Source: `src/protean_research/data/{__init__,loading,preprocessing,splits,partition,manifest,
|
||||
characterization,rare_unseen,pipeline}.py`,
|
||||
`src/protean_research/models/{__init__,protean_cnn,prototypes}.py`,
|
||||
`src/protean_research/federated/{__init__,client,server,round}.py`,
|
||||
`src/protean_research/evaluation/{__init__,metrics}.py`,
|
||||
`src/protean_research/utils/{seeding,integrity}.py`.
|
||||
## 11. Files created / modified this session
|
||||
|
||||
Scripts: `scripts/run_baseline.py`, `scripts/generate_partitions.py`.
|
||||
Created:
|
||||
|
||||
Tests: `tests/test_data_pipeline.py`, `tests/test_partition.py`, `tests/test_prototypes.py`,
|
||||
`tests/test_baseline_aggregation.py`, `tests/test_metrics.py`, `tests/test_federated_smoke.py`.
|
||||
- `docs/baseline/BASELINE_VALIDATION.md` (new authoritative validation record).
|
||||
- `tests/test_baseline_validation.py` (12 tests).
|
||||
- `scripts/validate_sj_participant_rule.py` (read-only analysis, no experiment mode).
|
||||
- `artifacts/analysis/sj_participant_rule_analysis.csv` (all 18 conditions, |S_j| analysis).
|
||||
- `artifacts/analysis/sj_vs_literal_1m_hypothetical_xiiotid__alpha0.5__seed0__original_protean.csv`.
|
||||
- `artifacts/analysis/xiiotid__alpha0.5__seed0__original_protean__prototype_snapshot.json`.
|
||||
- `artifacts/analysis/5gnidd__alpha0.5__seed0__original_protean__prototype_snapshot.json`.
|
||||
|
||||
Docs: `docs/baseline/BASELINE_IMPLEMENTATION.md`.
|
||||
Modified:
|
||||
|
||||
Artifacts (tracked, small): `artifacts/partitions/*.json` (18),
|
||||
`artifacts/analysis/*.csv` (36).
|
||||
- `docs/baseline/BASELINE_IMPLEMENTATION.md` (validation link, corrected test count,
|
||||
updated remaining-assumptions §14).
|
||||
- `docs/handoff/SESSION_HANDOFF.md` (this file).
|
||||
|
||||
## 11. Files modified this session
|
||||
Ignored/generated (not tracked): `results/xiiotid__alpha0.5__seed0__original_protean/`,
|
||||
`results/5gnidd__alpha0.5__seed0__original_protean/`,
|
||||
`artifacts/processed/*` (preprocessor state).
|
||||
|
||||
- `configs/datasets/datasets.yaml`
|
||||
- `configs/experiments/experiment_protocol.yaml`
|
||||
- `docs/research/EXPERIMENT_PROTOCOL.md`
|
||||
- `docs/research/IMPLEMENTATION_PLAN.md`
|
||||
- `README.md`
|
||||
- `src/protean_research/aggregation/__init__.py`
|
||||
- `src/protean_research/aggregation/prototype_aggregation.py`
|
||||
- `src/protean_research/models/prototypes.py`
|
||||
- `src/protean_research/federated/server.py`
|
||||
- `src/protean_research/federated/round.py`
|
||||
- `scripts/run_baseline.py`
|
||||
- tests (baseline-only aggregation suite; beta tests removed)
|
||||
`planningawal/` and both raw datasets were not modified.
|
||||
|
||||
`planningawal/` was not touched.
|
||||
## 12. Git status
|
||||
|
||||
## 12. Removed from this checkpoint (deferred to beta/controls phase)
|
||||
No commit was requested or made. Session 2 baseline commit `0aa8e2b` and the docs commit
|
||||
`374651a` are on `main` at `origin/main`. Working tree has one modified tracked file
|
||||
(`docs/baseline/BASELINE_IMPLEMENTATION.md`) and the new untracked files listed above.
|
||||
`results/`, `artifacts/processed/`, `.venv/`, `__pycache__/` remain ignored. No secrets,
|
||||
no raw datasets. `AGENTS.md` is tracked (commit `374651a`).
|
||||
|
||||
- `support_weighted` aggregation code and `beta` parameters.
|
||||
- `equal_available` mode (redundant with the baseline `S_j` rule; only needed if a later
|
||||
investigation shows the baseline participant rule differs).
|
||||
- `--mode` / `--beta` CLI options in `run_baseline.py`.
|
||||
- Beta-specific tests (`beta = 0/1`, single-contributor/equal-support beta invariance,
|
||||
normalised beta weights, the `original_protean == support_weighted(beta=0)` equivalence
|
||||
test). The former `tests/test_prototype_aggregation.py` was removed and folded into a
|
||||
baseline-only `tests/test_baseline_aggregation.py`.
|
||||
- Any claim that `original_protean == support_weighted(beta=0)` demonstrates equivalence
|
||||
with original PROTEAN. That was at most an internal implementation-equivalence check and
|
||||
is no longer present in code or docs.
|
||||
## 13. Unresolved assumptions / blockers
|
||||
|
||||
## 13. Git status
|
||||
1. **`S_j` participant/denominator rule** is a reproduction assumption, quantified but
|
||||
unresolved (see §7). All future between-method claims must carry this caveat.
|
||||
2. `lambda = 1.0`, optimiser (Adam), lr (1e-3), batch size (256) are PROJECT DECISIONS.
|
||||
3. Prototype-alignment form (per-sample) and round-0 initialisation are INFERRED.
|
||||
4. Model aggregation is equal `1/M` (PAPER); a FedAvg-style weighted wording in the thesis
|
||||
intro conflicts and is not implemented (documented conflict).
|
||||
5. Per-client evaluation partitioning is a PROJECT DECISION.
|
||||
6. Session-2 manifests store `preprocessing_hash = null` / `config_hash = null`; they do
|
||||
not bind the preprocessing recipe. Split/partition hashes and `N_ij` reproduce exactly,
|
||||
and run metadata records the live preprocessing hash. Binding the hash into future
|
||||
manifests would close this provenance gap.
|
||||
7. Only one full-data condition per dataset has been validated; no reproduction success is
|
||||
claimed from two runs.
|
||||
|
||||
Not committed (no commit was requested). `git status --short` shows modified tracked files
|
||||
(configs, docs, aggregation package, federated modules, README, run script) plus untracked
|
||||
source/tests/scripts/docs, the 18 partition manifests and 36 analysis CSVs. `AGENTS.md` was
|
||||
already untracked before this session. `.venv/`, `artifacts/processed/`, `results/`,
|
||||
`logs/`, and `*.log` are ignored. No checkpoints or raw datasets are tracked.
|
||||
## 14. Exact next task
|
||||
|
||||
## 14. Unresolved issues / blockers
|
||||
**Recommended: (B) freeze the validated baseline and prepare the paired beta
|
||||
aggregation implementation — not a full grid.**
|
||||
|
||||
1. **Baseline is implemented but not yet VALIDATED.** The `original_protean` participant
|
||||
rule is a reproduction assumption, not verified behavior.
|
||||
2. `lambda`, optimiser/schedule, and round-0 handling are unverified.
|
||||
3. No full-scale run has been timed; compute feasibility of a real sweep is unmeasured
|
||||
(X-IIoTID ~570k train rows, 5G-NIDD ~851k; CPU-only).
|
||||
4. Per-client evaluation partitioning is a project decision.
|
||||
5. Prototype-alignment form (per-sample vs class-mean) is a documented inference.
|
||||
6. Whether the literal `1/M` missing-class reading differs from the `S_j` assumption is
|
||||
open; a deliberate control is needed in the verification phase.
|
||||
Justification from evidence:
|
||||
|
||||
## 15. Exact next task
|
||||
- Implementation correctness, determinism, partition consistency and the full-data paths
|
||||
are validated on both datasets (Sessions 2–3).
|
||||
- Option (A), "resolve remaining baseline reproduction ambiguity", is **not achievable
|
||||
with available evidence**: no official PROTEAN code exists and the paper is internally
|
||||
inconsistent. The ambiguity has been quantified; further "resolution" would only be
|
||||
another documented decision.
|
||||
- Therefore freeze the baseline definition *including* the documented `S_j` equal-weight
|
||||
assumption and its quantified materiality, then implement the proposed beta weighting
|
||||
as a new aggregation mode **without altering the baseline** and run a **single paired
|
||||
`(dataset, alpha, seed)` condition first** (e.g. X-IIoTID α=0.5 seed 0, reusing the
|
||||
frozen manifest) across `beta in {0, 0.25, 0.5, 0.75, 1}` before any grid.
|
||||
|
||||
**Baseline validation and reproduction verification — NOT beta weighting.**
|
||||
Next session must:
|
||||
|
||||
Next session should:
|
||||
1. Add `support_weighted` (`w_ij = N_ij^beta`) as a new mode only; keep
|
||||
`original_protean` byte-identical and add the deferred beta-invariant tests
|
||||
(`beta=0` == equal mean over `S_j`; `beta=1` == support weighting; single-contributor
|
||||
and equal-support invariance; normalised weights sum to 1).
|
||||
2. Do **not** add an executable `equal_available` mode (baseline already is equal-over-`S_j`).
|
||||
3. Keep validation/test separation: beta selection on validation rare-class macro-F1,
|
||||
test only after selection.
|
||||
4. Run one paired condition end to end and inspect; measure before any grid.
|
||||
5. Report the `S_j` reproduction assumption explicitly alongside any beta result.
|
||||
|
||||
1. Run one full-data baseline condition (e.g. X-IIoTID, `alpha=0.5`, `seed=0`,
|
||||
`original_protean`) with measured wall-clock and memory; record it.
|
||||
2. Inspect the run: per-class precision/recall/F1, per-client rare metrics, confusion
|
||||
matrix. Check plausibility and missing-class-client behavior.
|
||||
3. Stress-test the `S_j` reproduction assumption: compare against a literal `1/M`
|
||||
omitted-class reading to quantify the effect of the assumption (introduce the control
|
||||
only for this investigation, clearly marked).
|
||||
4. Add a deterministic end-to-end baseline regression test pinning a small result.
|
||||
5. Only after the baseline is demonstrably verified and recorded, begin the paired beta
|
||||
comparison on a single `(dataset, alpha, seed)` before any grid.
|
||||
Do **not** launch the full grid until the single paired condition is verified.
|
||||
|
||||
Do **not** launch the full grid, and do not change the architecture, loss coefficients, or
|
||||
inference rule.
|
||||
|
||||
## 16. Important commands
|
||||
## 15. Important commands
|
||||
|
||||
```bash
|
||||
# Tests
|
||||
# Tests (46 expected)
|
||||
.venv/bin/python -m pytest -q
|
||||
|
||||
# Generate / verify all partition manifests (full data, ~30 s)
|
||||
.venv/bin/python scripts/generate_partitions.py
|
||||
|
||||
# Baseline smoke on both datasets (writes only to ignored dirs)
|
||||
.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \
|
||||
--max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1
|
||||
.venv/bin/python scripts/run_baseline.py --dataset 5gnidd --alpha 0.25 --seed 1 \
|
||||
--max-rows 4000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1
|
||||
|
||||
# Full-data baseline for one condition (next session; expect long CPU runtime)
|
||||
# Full-data baseline for one condition (measure first; ~40-65 min, CPU)
|
||||
.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \
|
||||
--clients 10 --rounds 10 --local-epochs 3
|
||||
.venv/bin/python scripts/run_baseline.py --dataset 5gnidd --alpha 0.5 --seed 0 \
|
||||
--clients 10 --rounds 10 --local-epochs 3
|
||||
|
||||
# Smoke (writes only to ignored dirs)
|
||||
.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \
|
||||
--max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1
|
||||
|
||||
# Read-only S_j participant-rule validation (writes analysis artifacts only)
|
||||
.venv/bin/python scripts/validate_sj_participant_rule.py \
|
||||
--snapshot artifacts/analysis/xiiotid__alpha0.5__seed0__original_protean__prototype_snapshot.json
|
||||
|
||||
# Regenerate / verify all partition manifests (full data, ~30 s)
|
||||
.venv/bin/python scripts/generate_partitions.py
|
||||
```
|
||||
|
||||
## 17. Recommended next-session prompt
|
||||
## 16. Recommended next-session prompt
|
||||
|
||||
```
|
||||
You are continuing the perdalam-protean research repository (Session 3).
|
||||
You are continuing the perdalam-protean research repository (Session 4).
|
||||
|
||||
FIRST, read completely:
|
||||
1. docs/handoff/SESSION_HANDOFF.md
|
||||
2. docs/baseline/BASELINE_IMPLEMENTATION.md
|
||||
3. docs/research/EXPERIMENT_PROTOCOL.md
|
||||
|
||||
Then read as needed:
|
||||
- planningawal/PROTEAN_Revisi_Lurus.md
|
||||
- docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md
|
||||
- docs/research/IMPLEMENTATION_PLAN.md
|
||||
2. docs/baseline/BASELINE_VALIDATION.md
|
||||
3. docs/baseline/BASELINE_IMPLEMENTATION.md
|
||||
4. docs/research/EXPERIMENT_PROTOCOL.md
|
||||
|
||||
Critical rules:
|
||||
- planningawal/ is immutable. Do not edit/create/move/rename/delete anything inside it.
|
||||
- ~/datasets/X-IIoTID/X-IIoTID-dataset.csv and ~/datasets/5G-NIDD/5G-NIDD.csv are
|
||||
read-only external source data. Do not modify or copy them into the repo.
|
||||
- This checkpoint is BASELINE ONLY. Do NOT add the proposed support_weighted/beta code
|
||||
or the equal_available control until the baseline is verified.
|
||||
- Do NOT launch the full alpha x seed x dataset x beta sweep.
|
||||
- The baseline is validated but the S_j participant rule is a REPRODUCTION ASSUMPTION
|
||||
(material in 32.7% of class-condition cells). Carry this caveat in all reporting.
|
||||
- Do NOT add an executable equal_available mode.
|
||||
- Do NOT launch the full alpha x seed x dataset x beta grid.
|
||||
|
||||
Your task is Section 15 of the handoff: BASELINE VALIDATION AND REPRODUCTION
|
||||
VERIFICATION. Run one full-data baseline condition with measured runtime, inspect its
|
||||
per-class/per-client behavior, stress-test the S_j participant-rule assumption, and add a
|
||||
deterministic end-to-end baseline regression test. Record everything in
|
||||
docs/baseline/BASELINE_IMPLEMENTATION.md and update docs/handoff/SESSION_HANDOFF.md.
|
||||
Task (Section 14 of the handoff): freeze the validated baseline and add the proposed
|
||||
support_weighted aggregation (w_ij = N_ij^beta) as a NEW mode without changing
|
||||
original_protean. Add the deferred beta-invariant tests. Then run ONE paired
|
||||
(dataset, alpha, seed) condition across beta in {0, 0.25, 0.5, 0.75, 1}, select beta on
|
||||
validation rare-class macro-F1, and evaluate the selected beta on test. Measure runtime
|
||||
before scaling. Update docs and docs/handoff/SESSION_HANDOFF.md.
|
||||
|
||||
Use .venv/bin/python for all Python commands.
|
||||
Use .venv/bin/python for all Python commands. Do not commit or push unless asked.
|
||||
```
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python
|
||||
"""Read-only validation of the baseline ``S_j`` prototype participant rule.
|
||||
|
||||
This is a **baseline-validation analysis**, not an experiment mode. It never
|
||||
trains, never changes the baseline, and never fabricates prototypes for training.
|
||||
|
||||
It answers two questions:
|
||||
|
||||
1. Over the frozen partition manifests, how often is ``|S_j| < M`` (i.e. how
|
||||
often does the paper's literal ``1/M`` become ambiguous), for which classes
|
||||
and conditions?
|
||||
2. Given a fixed round snapshot of ``C_ij`` and ``N_ij`` (no retraining), how far
|
||||
is the current ``S_j`` mean from one hypothetical literal-``1/M`` reading that
|
||||
replaces missing prototypes with the zero vector?
|
||||
|
||||
Outputs (analysis artifacts, clearly marked hypothetical):
|
||||
|
||||
- ``artifacts/analysis/sj_participant_rule_analysis.csv``
|
||||
- ``artifacts/analysis/sj_vs_literal_1m_hypothetical_<dataset>_alpha<..>_seed<..>.csv``
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/validate_sj_participant_rule.py
|
||||
python scripts/validate_sj_participant_rule.py \
|
||||
--snapshot artifacts/analysis/xiiotid__alpha0.5__seed0__original_protean__prototype_snapshot.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_PARTITIONS = REPO_ROOT / "artifacts" / "partitions"
|
||||
DEFAULT_ANALYSIS = REPO_ROOT / "artifacts" / "analysis"
|
||||
|
||||
|
||||
def summarize_manifests(partition_dir: Path, analysis_dir: Path) -> None:
|
||||
rows = []
|
||||
n_cells = n_less = n_conditions = n_conditions_less = 0
|
||||
for path in sorted(partition_dir.glob("*.json")):
|
||||
m = json.loads(path.read_text())
|
||||
support = m["support"]
|
||||
M = int(m["num_clients"])
|
||||
names = m["class_names"]
|
||||
condition_less = 0
|
||||
for j, name in enumerate(names):
|
||||
column = [int(support[i][j]) for i in range(M)]
|
||||
contributors = [i for i, v in enumerate(column) if v > 0]
|
||||
missing = [i for i, v in enumerate(column) if v == 0]
|
||||
n_cells += 1
|
||||
if len(contributors) < M:
|
||||
n_less += 1
|
||||
condition_less += 1
|
||||
rows.append(
|
||||
{
|
||||
"dataset": m["dataset"],
|
||||
"alpha": m["alpha"],
|
||||
"seed": m["seed"],
|
||||
"M": M,
|
||||
"class_id": j,
|
||||
"class_name": name,
|
||||
"S_j": len(contributors),
|
||||
"missing_clients": "|".join(str(i) for i in missing),
|
||||
"N_contrib_min": min([column[i] for i in contributors]) if contributors else 0,
|
||||
"N_contrib_max": max([column[i] for i in contributors]) if contributors else 0,
|
||||
"N_total": sum(column),
|
||||
}
|
||||
)
|
||||
n_conditions += 1
|
||||
if condition_less:
|
||||
n_conditions_less += 1
|
||||
|
||||
out = analysis_dir / "sj_participant_rule_analysis.csv"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fields = list(rows[0].keys())
|
||||
with out.open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
print(f"manifests scanned: {n_conditions}")
|
||||
print(f"conditions with any |S_j| < M: {n_conditions_less}/{n_conditions}")
|
||||
print(f"class-condition cells with |S_j| < M: {n_less}/{n_cells} "
|
||||
f"({100.0 * n_less / n_cells:.1f}%)")
|
||||
print(f"wrote {out}")
|
||||
|
||||
|
||||
def snapshot_variants(snapshot_path: Path, analysis_dir: Path) -> None:
|
||||
snap = json.loads(snapshot_path.read_text())
|
||||
prototypes = snap["client_prototypes"]
|
||||
supports = snap["client_supports"]
|
||||
M = len(prototypes)
|
||||
classes = sorted(
|
||||
{int(c) for row in prototypes for c in row}
|
||||
| {int(c) for row in supports for c in row}
|
||||
)
|
||||
dim = len(next(iter(prototypes[0].values())))
|
||||
|
||||
rows = []
|
||||
for j in classes:
|
||||
contributors = [
|
||||
(i, np.asarray(prototypes[i][str(j)], dtype=np.float64), int(supports[i][str(j)]))
|
||||
for i in range(M)
|
||||
if str(j) in prototypes[i]
|
||||
]
|
||||
if not contributors:
|
||||
continue
|
||||
cbar_sj = np.mean([v for _, v, _ in contributors], axis=0)
|
||||
# Hypothetical only: literal 1/M with missing prototypes replaced by zeros.
|
||||
zero_sum = sum((v for _, v, _ in contributors), np.zeros(dim, dtype=np.float64))
|
||||
cbar_zero = zero_sum / M
|
||||
rows.append(
|
||||
{
|
||||
"class_id": j,
|
||||
"M": M,
|
||||
"S_j": len(contributors),
|
||||
"missing_clients": "|".join(
|
||||
str(i) for i in range(M) if i not in {i for i, _, _ in contributors}
|
||||
),
|
||||
"N_ij_contributors": "|".join(
|
||||
str(int(supports[i][str(j)])) for i, _, _ in contributors
|
||||
),
|
||||
"displacement_Sj_vs_literal_1M_zero": float(np.linalg.norm(cbar_sj - cbar_zero)),
|
||||
"mean_norm_Cbar_Sj": float(np.linalg.norm(cbar_sj)),
|
||||
"mean_contributor_norm": float(
|
||||
np.mean([np.linalg.norm(v) for _, v, _ in contributors])
|
||||
),
|
||||
"hypothetical": True,
|
||||
}
|
||||
)
|
||||
|
||||
tag = snapshot_path.name.replace("__prototype_snapshot.json", "")
|
||||
out = analysis_dir / f"sj_vs_literal_1m_hypothetical_{tag}.csv"
|
||||
fields = list(rows[0].keys())
|
||||
with out.open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(f"snapshot round={snap.get('round')} M={M} dim={dim}; wrote {out}")
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--partitions-dir", default=str(DEFAULT_PARTITIONS))
|
||||
parser.add_argument("--analysis-dir", default=str(DEFAULT_ANALYSIS))
|
||||
parser.add_argument("--snapshot", default=None, help="optional round-0 prototype snapshot JSON")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
args = parse_args(argv)
|
||||
analysis_dir = Path(args.analysis_dir)
|
||||
summarize_manifests(Path(args.partitions_dir), analysis_dir)
|
||||
if args.snapshot:
|
||||
snapshot_variants(Path(args.snapshot), analysis_dir)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,408 @@
|
||||
"""Baseline validation tests (round-0 handling, loss audit, inference, regression).
|
||||
|
||||
These tests protect the frozen baseline semantics against accidental change when
|
||||
the proposed support-weighted aggregation is added later. They assert robust
|
||||
invariants and, where needed, replicate the documented loss formula
|
||||
independently rather than calling the production loss inline.
|
||||
|
||||
Nothing here implements or exercises the proposed beta/support-weighted method.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
from protean_research.federated import TrainingConfig, run_federated # noqa: E402
|
||||
from protean_research.federated import client as client_mod # noqa: E402
|
||||
from protean_research.federated import server as server_mod # noqa: E402
|
||||
from protean_research.federated.client import ClientUpdate, local_update # noqa: E402
|
||||
from protean_research.models.protean_cnn import ( # noqa: E402
|
||||
ModelConfig,
|
||||
average_parameters,
|
||||
build_model,
|
||||
get_parameters,
|
||||
set_parameters,
|
||||
)
|
||||
from protean_research.models.prototypes import ( # noqa: E402
|
||||
compute_local_prototypes,
|
||||
nearest_prototype_predict,
|
||||
)
|
||||
|
||||
|
||||
def make_separable(num_classes=3, per_class=12, dim=5, seed=0):
|
||||
rng = np.random.default_rng(seed)
|
||||
features = rng.normal(size=(num_classes * per_class, dim)).astype(np.float32)
|
||||
labels = np.repeat(np.arange(num_classes), per_class).astype(np.int64)
|
||||
features = features + labels[:, None] * 2.0
|
||||
return features, labels
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 7: round-0 handling and prototype alignment entry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_round0_sees_no_global_prototypes_and_round1_does(monkeypatch):
|
||||
"""Round 0 must start with no global prototypes; later rounds receive them."""
|
||||
captured: list = []
|
||||
original = client_mod.local_update
|
||||
|
||||
def spy(*args, **kwargs):
|
||||
# global_prototypes is the 4th positional argument in round.py.
|
||||
captured.append(args[3] if len(args) > 3 else kwargs.get("global_prototypes"))
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(client_mod, "local_update", spy)
|
||||
|
||||
features, labels = make_separable(num_classes=3, per_class=8, dim=8)
|
||||
num_clients = 3
|
||||
assignments = [np.arange(0, 8), np.arange(8, 16), np.arange(16, 24)]
|
||||
config = TrainingConfig(
|
||||
rounds=2,
|
||||
local_epochs=1,
|
||||
batch_size=8,
|
||||
num_clients=num_clients,
|
||||
embedding_dim=8,
|
||||
dense_units=8,
|
||||
)
|
||||
run_federated(
|
||||
train_features=features,
|
||||
train_labels=labels,
|
||||
assignments=assignments,
|
||||
val_features=features,
|
||||
val_labels=labels,
|
||||
num_classes=3,
|
||||
input_dim=features.shape[1],
|
||||
config=config,
|
||||
init_seed_parts=("round0", "seed=0"),
|
||||
)
|
||||
|
||||
assert len(captured) == 2 * num_clients
|
||||
round0 = captured[:num_clients]
|
||||
round1 = captured[num_clients:]
|
||||
# Round 0: the prototype-alignment term is skipped (empty/None prototypes).
|
||||
assert all(g is None or len(g) == 0 for g in round0)
|
||||
# Round 1: the first aggregated global prototypes are broadcast back.
|
||||
assert all(g is not None and len(g) > 0 for g in round1)
|
||||
|
||||
|
||||
def test_local_update_without_global_prototypes_still_trains():
|
||||
features, labels = make_separable(num_classes=2, per_class=6, dim=4)
|
||||
model = build_model(ModelConfig(input_dim=4, num_classes=2, embedding_dim=6, dense_units=6))
|
||||
params = get_parameters(model)
|
||||
update = local_update(
|
||||
0,
|
||||
model,
|
||||
params,
|
||||
None,
|
||||
features,
|
||||
labels,
|
||||
np.arange(len(labels)),
|
||||
num_classes=2,
|
||||
local_epochs=1,
|
||||
batch_size=6,
|
||||
learning_rate=1e-3,
|
||||
mu=0.1,
|
||||
lam=1.0,
|
||||
seed=1,
|
||||
)
|
||||
assert isinstance(update, ClientUpdate)
|
||||
assert set(update.prototypes) == {0, 1}
|
||||
assert update.n_samples == len(labels)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 8: loss implementation audit (independent replication of the formula)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _manual_local_update(
|
||||
model,
|
||||
global_parameters,
|
||||
global_prototypes,
|
||||
features,
|
||||
labels,
|
||||
*,
|
||||
local_epochs,
|
||||
batch_size,
|
||||
learning_rate,
|
||||
mu,
|
||||
lam,
|
||||
seed,
|
||||
):
|
||||
"""Independent transcription of the documented objective.
|
||||
|
||||
CE(mean) + lam * mean_matched ||phi(x) - Cbar_y||^2 + (mu/2) * sum ||w - w_global||^2
|
||||
"""
|
||||
import torch.nn.functional as F
|
||||
|
||||
set_parameters(model, global_parameters)
|
||||
device = next(model.parameters()).device
|
||||
x = torch.as_tensor(features, dtype=torch.float32, device=device)
|
||||
y = torch.as_tensor(np.asarray(labels), dtype=torch.long, device=device)
|
||||
global_tensors = [
|
||||
torch.as_tensor(p, dtype=torch.float32, device=device) for p in global_parameters
|
||||
]
|
||||
proto_tensors = {
|
||||
int(c): torch.as_tensor(v, dtype=torch.float32, device=device)
|
||||
for c, v in (global_prototypes or {}).items()
|
||||
}
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
|
||||
generator = torch.Generator(device="cpu")
|
||||
generator.manual_seed(seed)
|
||||
|
||||
model.train()
|
||||
for _ in range(local_epochs):
|
||||
order = torch.randperm(x.shape[0], generator=generator)
|
||||
for start in range(0, x.shape[0], batch_size):
|
||||
idx = order[start : start + batch_size]
|
||||
xb, yb = x[idx], y[idx]
|
||||
optimizer.zero_grad()
|
||||
log_prob = model(xb)
|
||||
loss = F.nll_loss(log_prob, yb)
|
||||
if proto_tensors:
|
||||
emb = model.embedding(xb)
|
||||
align = torch.zeros((), device=device)
|
||||
count = 0
|
||||
for class_id, proto in proto_tensors.items():
|
||||
mask = yb == class_id
|
||||
if mask.any():
|
||||
diff = emb[mask] - proto.unsqueeze(0)
|
||||
align = align + (diff ** 2).sum(dim=1).sum()
|
||||
count += int(mask.sum().item())
|
||||
if count:
|
||||
loss = loss + lam * align / count
|
||||
if mu:
|
||||
penalty = torch.zeros((), device=device)
|
||||
for param, target in zip(model.parameters(), global_tensors):
|
||||
penalty = penalty + ((param - target) ** 2).sum()
|
||||
loss = loss + (mu / 2.0) * penalty
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
return get_parameters(model)
|
||||
|
||||
|
||||
def test_local_update_matches_documented_loss_formula():
|
||||
"""Same seed/batch order -> identical parameters as the documented formula."""
|
||||
torch.manual_seed(0)
|
||||
dim, num_classes = 8, 3
|
||||
cfg = ModelConfig(input_dim=dim, num_classes=num_classes, embedding_dim=6, dense_units=6)
|
||||
global_model = build_model(cfg)
|
||||
global_parameters = get_parameters(global_model)
|
||||
|
||||
rng = np.random.default_rng(3)
|
||||
features = rng.normal(size=(9, dim)).astype(np.float32)
|
||||
labels = np.array([0, 1, 2, 0, 1, 2, 0, 1, 2], dtype=np.int64)
|
||||
global_prototypes = {
|
||||
0: rng.normal(size=6).astype(np.float32),
|
||||
1: rng.normal(size=6).astype(np.float32),
|
||||
}
|
||||
|
||||
seed = 12345
|
||||
lam, mu = 0.7, 0.1
|
||||
|
||||
impl_model = build_model(cfg)
|
||||
# Dropout draws from the global torch RNG; reseed identically for each path.
|
||||
torch.manual_seed(seed)
|
||||
impl = local_update(
|
||||
0,
|
||||
impl_model,
|
||||
global_parameters,
|
||||
global_prototypes,
|
||||
features,
|
||||
labels,
|
||||
np.arange(len(labels)),
|
||||
num_classes=num_classes,
|
||||
local_epochs=1,
|
||||
batch_size=4,
|
||||
learning_rate=1e-3,
|
||||
mu=mu,
|
||||
lam=lam,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
ref_model = build_model(cfg)
|
||||
torch.manual_seed(seed)
|
||||
expected = _manual_local_update(
|
||||
ref_model,
|
||||
global_parameters,
|
||||
global_prototypes,
|
||||
features,
|
||||
labels,
|
||||
local_epochs=1,
|
||||
batch_size=4,
|
||||
learning_rate=1e-3,
|
||||
mu=mu,
|
||||
lam=lam,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
for got, want in zip(impl.parameters, expected):
|
||||
assert torch.allclose(got, want, atol=1e-5, rtol=1e-4), (
|
||||
"local_update parameters diverge from the documented loss formula"
|
||||
)
|
||||
|
||||
|
||||
def test_prototype_alignment_skips_classes_without_a_global_prototype():
|
||||
"""Labels without a global prototype must not add an alignment term."""
|
||||
torch.manual_seed(1)
|
||||
dim, num_classes = 8, 3
|
||||
cfg = ModelConfig(input_dim=dim, num_classes=num_classes, embedding_dim=5, dense_units=5)
|
||||
global_parameters = get_parameters(build_model(cfg))
|
||||
|
||||
rng = np.random.default_rng(7)
|
||||
features = rng.normal(size=(6, dim)).astype(np.float32)
|
||||
labels = np.array([0, 0, 2, 2, 2, 2], dtype=np.int64)
|
||||
# Only class 0 has a global prototype; class 2 must be skipped entirely.
|
||||
global_prototypes = {0: rng.normal(size=5).astype(np.float32)}
|
||||
seed = 999
|
||||
|
||||
impl_model = build_model(cfg)
|
||||
torch.manual_seed(seed)
|
||||
impl = local_update(
|
||||
0, impl_model, global_parameters, global_prototypes, features, labels,
|
||||
np.arange(len(labels)), num_classes=num_classes, local_epochs=1,
|
||||
batch_size=6, learning_rate=1e-3, mu=0.0, lam=1.0, seed=seed,
|
||||
)
|
||||
ref_model = build_model(cfg)
|
||||
torch.manual_seed(seed)
|
||||
expected = _manual_local_update(
|
||||
ref_model, global_parameters, global_prototypes, features, labels,
|
||||
local_epochs=1, batch_size=6, learning_rate=1e-3, mu=0.0, lam=1.0, seed=seed,
|
||||
)
|
||||
for got, want in zip(impl.parameters, expected):
|
||||
assert torch.allclose(got, want, atol=1e-5, rtol=1e-4)
|
||||
|
||||
|
||||
def test_training_config_coefficients_are_frozen():
|
||||
"""Guard lambda/mu against drifting away from the frozen protocol."""
|
||||
config = TrainingConfig()
|
||||
assert config.lam == pytest.approx(1.0)
|
||||
assert config.mu == pytest.approx(0.1)
|
||||
assert config.rounds == 10
|
||||
assert config.local_epochs == 3
|
||||
assert config.batch_size == 256
|
||||
assert config.learning_rate == pytest.approx(1e-3)
|
||||
|
||||
|
||||
def test_model_aggregation_is_equal_weight_mean():
|
||||
torch.manual_seed(2)
|
||||
cfg = ModelConfig(input_dim=8, num_classes=2, embedding_dim=4, dense_units=4)
|
||||
a = get_parameters(build_model(cfg))
|
||||
b = get_parameters(build_model(cfg))
|
||||
avg = average_parameters([a, b])
|
||||
for pa, pb, pm in zip(a, b, avg):
|
||||
assert torch.allclose(pm, (pa + pb) / 2.0, atol=1e-6)
|
||||
# Equal 1/M averaging: identical lists collapse to themselves.
|
||||
same = average_parameters([a, a, a])
|
||||
for pa, pm in zip(a, same):
|
||||
assert torch.allclose(pm, pa, atol=1e-6)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 9: inference validation (synthetic fixture)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_nearest_prototype_predict_maps_noncontiguous_class_ids():
|
||||
prototypes = {2: np.array([0.0, 0.0]), 5: np.array([10.0, 10.0])}
|
||||
embeddings = np.array([[0.5, 0.5], [9.0, 8.5]])
|
||||
predictions = nearest_prototype_predict(embeddings, prototypes)
|
||||
assert predictions.tolist() == [2, 5]
|
||||
|
||||
|
||||
def test_nearest_prototype_predict_matches_squared_l2_bruteforce():
|
||||
rng = np.random.default_rng(11)
|
||||
prototypes = {c: rng.normal(size=7) for c in (1, 4, 9)}
|
||||
embeddings = rng.normal(size=(20, 7))
|
||||
expected = []
|
||||
for emb in embeddings:
|
||||
dists = {
|
||||
c: float(((emb - p) ** 2).sum()) for c, p in prototypes.items()
|
||||
}
|
||||
expected.append(min(dists, key=dists.get))
|
||||
got = nearest_prototype_predict(embeddings, prototypes)
|
||||
assert got.tolist() == expected
|
||||
|
||||
|
||||
def test_nearest_prototype_predict_raises_on_empty_prototypes():
|
||||
with pytest.raises(ValueError):
|
||||
nearest_prototype_predict(np.zeros((1, 3)), {})
|
||||
|
||||
|
||||
def test_predict_global_raises_without_prototypes():
|
||||
class _M:
|
||||
pass
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
server_mod.predict_global(_M(), np.zeros((1, 3)), {})
|
||||
|
||||
|
||||
def test_compute_local_prototypes_class_index_mapping():
|
||||
embeddings = np.array([[0.0, 0.0], [2.0, 0.0], [4.0, 4.0]], dtype=np.float32)
|
||||
labels = np.array([1, 1, 3], dtype=np.int64)
|
||||
prototypes, supports = compute_local_prototypes(embeddings, labels, num_classes=5)
|
||||
assert set(prototypes) == {1, 3}
|
||||
assert supports[1] == 2
|
||||
assert prototypes[1] == pytest.approx([1.0, 0.0])
|
||||
assert prototypes[3] == pytest.approx([4.0, 4.0])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 10: deterministic end-to-end baseline regression
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_regression(seed_parts):
|
||||
features, labels = make_separable(num_classes=3, per_class=10, dim=6, seed=5)
|
||||
assignments = [
|
||||
np.arange(0, 10),
|
||||
np.arange(10, 20),
|
||||
np.arange(20, 30),
|
||||
]
|
||||
config = TrainingConfig(
|
||||
rounds=2,
|
||||
local_epochs=1,
|
||||
batch_size=10,
|
||||
num_clients=3,
|
||||
embedding_dim=8,
|
||||
dense_units=8,
|
||||
)
|
||||
return run_federated(
|
||||
train_features=features,
|
||||
train_labels=labels,
|
||||
assignments=assignments,
|
||||
val_features=features,
|
||||
val_labels=labels,
|
||||
num_classes=3,
|
||||
input_dim=features.shape[1],
|
||||
config=config,
|
||||
init_seed_parts=seed_parts,
|
||||
)
|
||||
|
||||
|
||||
def test_end_to_end_baseline_is_deterministic_and_learns_separable_fixture():
|
||||
first = _run_regression(("regression", "seed=0"))
|
||||
second = _run_regression(("regression", "seed=0"))
|
||||
|
||||
# Determinism: identical parameters, prototypes and metrics for the same seed.
|
||||
for a, b in zip(first["parameters"], second["parameters"]):
|
||||
assert torch.equal(a, b)
|
||||
assert set(first["global_prototypes"]) == set(second["global_prototypes"]) == {0, 1, 2}
|
||||
for c in first["global_prototypes"]:
|
||||
assert np.array_equal(
|
||||
np.asarray(first["global_prototypes"][c]),
|
||||
np.asarray(second["global_prototypes"][c]),
|
||||
)
|
||||
assert first["history"] == second["history"]
|
||||
|
||||
# Robust invariants (not fragile exact floats): the pipeline runs and the
|
||||
# prototype read-out beats chance on a linearly separable fixture.
|
||||
last = first["history"][-1]
|
||||
assert np.isfinite(last["accuracy"])
|
||||
assert np.isfinite(last["macro_f1"])
|
||||
assert last["accuracy"] > 0.5
|
||||
assert first["snapshot"] is not None
|
||||
Reference in new issue
Block a user