From 0aa8e2ba79e5ee04b2bea11571abc8719996ad1b Mon Sep 17 00:00:00 2001 From: willtanoe Date: Thu, 24 Sep 2026 17:39:00 +0000 Subject: [PATCH] feat: implement reproducible PROTEAN baseline pipeline --- README.md | 40 +- ...dd__alpha0.25__seed0__characterization.csv | 10 + .../5gnidd__alpha0.25__seed0__rare_unseen.csv | 91 ++++ ...dd__alpha0.25__seed1__characterization.csv | 10 + .../5gnidd__alpha0.25__seed1__rare_unseen.csv | 91 ++++ ...dd__alpha0.25__seed2__characterization.csv | 10 + .../5gnidd__alpha0.25__seed2__rare_unseen.csv | 91 ++++ ...idd__alpha0.5__seed0__characterization.csv | 10 + .../5gnidd__alpha0.5__seed0__rare_unseen.csv | 91 ++++ ...idd__alpha0.5__seed1__characterization.csv | 10 + .../5gnidd__alpha0.5__seed1__rare_unseen.csv | 91 ++++ ...idd__alpha0.5__seed2__characterization.csv | 10 + .../5gnidd__alpha0.5__seed2__rare_unseen.csv | 91 ++++ ...dd__alpha0.75__seed0__characterization.csv | 10 + .../5gnidd__alpha0.75__seed0__rare_unseen.csv | 91 ++++ ...dd__alpha0.75__seed1__characterization.csv | 10 + .../5gnidd__alpha0.75__seed1__rare_unseen.csv | 91 ++++ ...dd__alpha0.75__seed2__characterization.csv | 10 + .../5gnidd__alpha0.75__seed2__rare_unseen.csv | 91 ++++ ...id__alpha0.25__seed0__characterization.csv | 10 + ...xiiotid__alpha0.25__seed0__rare_unseen.csv | 91 ++++ ...id__alpha0.25__seed1__characterization.csv | 10 + ...xiiotid__alpha0.25__seed1__rare_unseen.csv | 91 ++++ ...id__alpha0.25__seed2__characterization.csv | 10 + ...xiiotid__alpha0.25__seed2__rare_unseen.csv | 91 ++++ ...tid__alpha0.5__seed0__characterization.csv | 10 + .../xiiotid__alpha0.5__seed0__rare_unseen.csv | 91 ++++ ...tid__alpha0.5__seed1__characterization.csv | 10 + .../xiiotid__alpha0.5__seed1__rare_unseen.csv | 91 ++++ ...tid__alpha0.5__seed2__characterization.csv | 10 + .../xiiotid__alpha0.5__seed2__rare_unseen.csv | 91 ++++ ...id__alpha0.75__seed0__characterization.csv | 10 + ...xiiotid__alpha0.75__seed0__rare_unseen.csv | 91 ++++ ...id__alpha0.75__seed1__characterization.csv | 10 + ...xiiotid__alpha0.75__seed1__rare_unseen.csv | 91 ++++ ...id__alpha0.75__seed2__characterization.csv | 10 + ...xiiotid__alpha0.75__seed2__rare_unseen.csv | 91 ++++ .../partitions/5gnidd__alpha0.25__seed0.json | 154 ++++++ .../partitions/5gnidd__alpha0.25__seed1.json | 154 ++++++ .../partitions/5gnidd__alpha0.25__seed2.json | 154 ++++++ .../partitions/5gnidd__alpha0.5__seed0.json | 154 ++++++ .../partitions/5gnidd__alpha0.5__seed1.json | 154 ++++++ .../partitions/5gnidd__alpha0.5__seed2.json | 154 ++++++ .../partitions/5gnidd__alpha0.75__seed0.json | 154 ++++++ .../partitions/5gnidd__alpha0.75__seed1.json | 154 ++++++ .../partitions/5gnidd__alpha0.75__seed2.json | 154 ++++++ .../partitions/xiiotid__alpha0.25__seed0.json | 154 ++++++ .../partitions/xiiotid__alpha0.25__seed1.json | 154 ++++++ .../partitions/xiiotid__alpha0.25__seed2.json | 154 ++++++ .../partitions/xiiotid__alpha0.5__seed0.json | 154 ++++++ .../partitions/xiiotid__alpha0.5__seed1.json | 154 ++++++ .../partitions/xiiotid__alpha0.5__seed2.json | 154 ++++++ .../partitions/xiiotid__alpha0.75__seed0.json | 154 ++++++ .../partitions/xiiotid__alpha0.75__seed1.json | 154 ++++++ .../partitions/xiiotid__alpha0.75__seed2.json | 154 ++++++ configs/datasets/datasets.yaml | 28 +- configs/experiments/experiment_protocol.yaml | 55 ++- docs/baseline/BASELINE_IMPLEMENTATION.md | 210 ++++++++ docs/handoff/SESSION_HANDOFF.md | 459 ++++++++---------- docs/research/EXPERIMENT_PROTOCOL.md | 217 ++++++--- docs/research/IMPLEMENTATION_PLAN.md | 75 +-- docs/research/RESEARCH_SPEC_INTERPRETATION.md | 7 +- scripts/generate_partitions.py | 124 +++++ scripts/run_baseline.py | 301 ++++++++++++ src/protean_research/aggregation/__init__.py | 12 +- .../aggregation/prototype_aggregation.py | 175 +++---- src/protean_research/data/__init__.py | 1 + src/protean_research/data/characterization.py | 66 +++ src/protean_research/data/loading.py | 225 +++++++++ src/protean_research/data/manifest.py | 159 ++++++ src/protean_research/data/partition.py | 129 +++++ src/protean_research/data/pipeline.py | 131 +++++ src/protean_research/data/preprocessing.py | 187 +++++++ src/protean_research/data/rare_unseen.py | 92 ++++ src/protean_research/data/splits.py | 109 +++++ src/protean_research/evaluation/__init__.py | 17 + src/protean_research/evaluation/metrics.py | 127 +++++ src/protean_research/federated/__init__.py | 15 + src/protean_research/federated/client.py | 130 +++++ src/protean_research/federated/round.py | 158 ++++++ src/protean_research/federated/server.py | 45 ++ src/protean_research/models/__init__.py | 27 ++ src/protean_research/models/protean_cnn.py | 116 +++++ src/protean_research/models/prototypes.py | 78 +++ src/protean_research/utils/integrity.py | 39 ++ src/protean_research/utils/seeding.py | 51 ++ tests/test_baseline_aggregation.py | 76 +++ tests/test_data_pipeline.py | 156 ++++++ tests/test_federated_smoke.py | 50 ++ tests/test_metrics.py | 44 ++ tests/test_partition.py | 126 +++++ tests/test_prototype_aggregation.py | 126 ----- tests/test_prototypes.py | 58 +++ 93 files changed, 8173 insertions(+), 658 deletions(-) create mode 100644 artifacts/analysis/5gnidd__alpha0.25__seed0__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.25__seed0__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.25__seed1__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.25__seed1__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.25__seed2__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.25__seed2__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.5__seed0__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.5__seed0__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.5__seed1__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.5__seed1__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.5__seed2__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.5__seed2__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.75__seed0__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.75__seed0__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.75__seed1__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.75__seed1__rare_unseen.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.75__seed2__characterization.csv create mode 100644 artifacts/analysis/5gnidd__alpha0.75__seed2__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.25__seed0__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.25__seed0__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.25__seed1__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.25__seed1__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.25__seed2__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.25__seed2__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed0__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed0__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed1__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed1__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed2__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed2__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.75__seed0__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.75__seed0__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.75__seed1__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.75__seed1__rare_unseen.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.75__seed2__characterization.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.75__seed2__rare_unseen.csv create mode 100644 artifacts/partitions/5gnidd__alpha0.25__seed0.json create mode 100644 artifacts/partitions/5gnidd__alpha0.25__seed1.json create mode 100644 artifacts/partitions/5gnidd__alpha0.25__seed2.json create mode 100644 artifacts/partitions/5gnidd__alpha0.5__seed0.json create mode 100644 artifacts/partitions/5gnidd__alpha0.5__seed1.json create mode 100644 artifacts/partitions/5gnidd__alpha0.5__seed2.json create mode 100644 artifacts/partitions/5gnidd__alpha0.75__seed0.json create mode 100644 artifacts/partitions/5gnidd__alpha0.75__seed1.json create mode 100644 artifacts/partitions/5gnidd__alpha0.75__seed2.json create mode 100644 artifacts/partitions/xiiotid__alpha0.25__seed0.json create mode 100644 artifacts/partitions/xiiotid__alpha0.25__seed1.json create mode 100644 artifacts/partitions/xiiotid__alpha0.25__seed2.json create mode 100644 artifacts/partitions/xiiotid__alpha0.5__seed0.json create mode 100644 artifacts/partitions/xiiotid__alpha0.5__seed1.json create mode 100644 artifacts/partitions/xiiotid__alpha0.5__seed2.json create mode 100644 artifacts/partitions/xiiotid__alpha0.75__seed0.json create mode 100644 artifacts/partitions/xiiotid__alpha0.75__seed1.json create mode 100644 artifacts/partitions/xiiotid__alpha0.75__seed2.json create mode 100644 docs/baseline/BASELINE_IMPLEMENTATION.md create mode 100644 scripts/generate_partitions.py create mode 100644 scripts/run_baseline.py create mode 100644 src/protean_research/data/__init__.py create mode 100644 src/protean_research/data/characterization.py create mode 100644 src/protean_research/data/loading.py create mode 100644 src/protean_research/data/manifest.py create mode 100644 src/protean_research/data/partition.py create mode 100644 src/protean_research/data/pipeline.py create mode 100644 src/protean_research/data/preprocessing.py create mode 100644 src/protean_research/data/rare_unseen.py create mode 100644 src/protean_research/data/splits.py create mode 100644 src/protean_research/evaluation/__init__.py create mode 100644 src/protean_research/evaluation/metrics.py create mode 100644 src/protean_research/federated/__init__.py create mode 100644 src/protean_research/federated/client.py create mode 100644 src/protean_research/federated/round.py create mode 100644 src/protean_research/federated/server.py create mode 100644 src/protean_research/models/__init__.py create mode 100644 src/protean_research/models/protean_cnn.py create mode 100644 src/protean_research/models/prototypes.py create mode 100644 src/protean_research/utils/integrity.py create mode 100644 src/protean_research/utils/seeding.py create mode 100644 tests/test_baseline_aggregation.py create mode 100644 tests/test_data_pipeline.py create mode 100644 tests/test_federated_smoke.py create mode 100644 tests/test_metrics.py create mode 100644 tests/test_partition.py delete mode 100644 tests/test_prototype_aggregation.py create mode 100644 tests/test_prototypes.py diff --git a/README.md b/README.md index 0762a8a..2348c58 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,20 @@ Research code for **server-side prototype aggregation in PROTEAN** using class-specific sample support, for rare attack detection under non-IID federated intrusion detection. -The study modifies only how the server combines local class prototypes: +The planned study modifies only how the server combines local class prototypes. The +proposed support-weighted aggregation is: ``` Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta C_ij ) / ( sum_{i in S_j} N_ij^beta ) ``` with `S_j = { i | N_ij > 0 and C_ij exists }` and candidate -`beta in {0.0, 0.25, 0.5, 0.75, 1.0}`. Architecture, model aggregation, loss terms, and -inference follow the PROTEAN baseline (Chennoufi et al., 2025). +`beta in {0.0, 0.25, 0.5, 0.75, 1.0}`. + +**Current checkpoint: baseline only.** The proposed beta-weighted aggregation and the +`equal_available` control are **not implemented** in this checkpoint; they are deferred to +the later beta/controls phase, which must not begin until the baseline is verified. See +`docs/handoff/SESSION_HANDOFF.md`. ## Immutable source @@ -30,19 +35,21 @@ never hardcoded in source. Generated artifacts go under `artifacts/`. ``` docs/research/ spec interpretation, dataset inspection, implementation plan, protocol -docs/baseline/ PROTEAN baseline investigation +docs/baseline/ PROTEAN baseline investigation + baseline implementation notes docs/handoff/ session handoff (start here for a new session) configs/ datasets, experiments, model -src/protean_research/ aggregation (implemented), data/federated/models/evaluation (planned) -tests/ unit tests +src/protean_research/ data, models, aggregation, federated, evaluation, utils (baseline) +scripts/ run_baseline.py, generate_partitions.py +tests/ focused unit + integration tests artifacts/ results/ generated outputs (not source) ``` ## Status -This is a planning/bootstrap state. No full training has been run and no experimental sweep -has been launched. The original PROTEAN baseline has **no public official code**, so it must -be reproduced from the paper. +The minimum reproducible **baseline** is implemented and smoke-tested. No full-scale +training and no beta sweep have been run. The original PROTEAN baseline has **no public +official code**, so it is reproduced from the paper/thesis with explicitly documented +reproduction assumptions (`docs/baseline/BASELINE_IMPLEMENTATION.md`). Read `docs/handoff/SESSION_HANDOFF.md` first, then `planningawal/PROTEAN_Revisi_Lurus.md`. @@ -50,13 +57,20 @@ Read `docs/handoff/SESSION_HANDOFF.md` first, then ## Quickstart ```bash -# inside the repo, with a virtualenv that has pyyaml + pytest -python -m pytest -q +# inside the repo, with the project virtualenv (numpy, pandas, torch, pyyaml, pytest) +.venv/bin/python -m pytest -q + +# baseline smoke test (writes only to ignored directories) +.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ + --max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1 + +# generate + verify all deterministic partition manifests (full data) +.venv/bin/python scripts/generate_partitions.py ``` -The aggregation module is dependency-free and testable without pandas/torch: +The baseline aggregation is dependency-free and testable without pandas/torch: ```python from protean_research.aggregation import aggregate_prototypes -aggregate_prototypes([[0.0, 2.0], [2.0, 0.0]], "support_weighted", beta=0.5, supports=[1, 3]) +aggregate_prototypes([[0.0, 2.0], [2.0, 0.0]]) # equal mean over the contributor set S_j ``` diff --git a/artifacts/analysis/5gnidd__alpha0.25__seed0__characterization.csv b/artifacts/analysis/5gnidd__alpha0.25__seed0__characterization.csv new file mode 100644 index 0000000..e0f4414 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.25__seed0__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,1,185504,33441.5,54297.395750164666,1.6236531181365867,185504.0 +1,HTTPFlood,98569,9,1,1,48469,10952.111111111111,15619.095146962789,1.4261264324753735,48469.0 +2,ICMPFlood,808,9,1,2,319,89.77777777777777,99.11360236031938,1.1039881451025675,159.5 +3,SYNFlood,6805,7,3,144,4762,972.1428571428571,1553.4714516270997,1.5979867981469065,33.06944444444444 +4,SYNScan,14030,7,3,237,6524,2004.2857142857142,2116.5558758299308,1.0560150485252684,27.52742616033755 +5,SlowrateDoS,51187,9,1,1,16517,5687.444444444444,6406.6724611682275,1.1264589085219694,16517.0 +6,TCPConnectScan,14037,9,1,2,10870,1559.6666666666667,3336.037403200923,2.138942553879626,5435.0 +7,UDPFlood,320138,10,0,1,127301,32013.8,44285.809189400614,1.3833349739612484,127301.0 +8,UDPScan,11134,10,0,2,4601,1113.4,1669.91060838597,1.4998298979575801,2300.5 diff --git a/artifacts/analysis/5gnidd__alpha0.25__seed0__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.25__seed0__rare_unseen.csv new file mode 100644 index 0000000..4900830 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.25__seed0__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.25,0,0,0,Benign,6003,False,False +5gnidd,0.25,0,0,1,HTTPFlood,4887,False,False +5gnidd,0.25,0,0,2,ICMPFlood,319,False,False +5gnidd,0.25,0,0,3,SYNFlood,275,False,False +5gnidd,0.25,0,0,4,SYNScan,955,False,False +5gnidd,0.25,0,0,5,SlowrateDoS,11954,False,False +5gnidd,0.25,0,0,6,TCPConnectScan,92,True,False +5gnidd,0.25,0,0,7,UDPFlood,127301,False,False +5gnidd,0.25,0,0,8,UDPScan,2,True,False +5gnidd,0.25,0,1,0,Benign,40519,False,False +5gnidd,0.25,0,1,1,HTTPFlood,1,True,False +5gnidd,0.25,0,1,2,ICMPFlood,181,False,False +5gnidd,0.25,0,1,3,SYNFlood,0,False,True +5gnidd,0.25,0,1,4,SYNScan,289,False,False +5gnidd,0.25,0,1,5,SlowrateDoS,18,True,False +5gnidd,0.25,0,1,6,TCPConnectScan,52,False,False +5gnidd,0.25,0,1,7,UDPFlood,62527,False,False +5gnidd,0.25,0,1,8,UDPScan,4601,False,False +5gnidd,0.25,0,2,0,Benign,17462,False,False +5gnidd,0.25,0,2,1,HTTPFlood,300,False,False +5gnidd,0.25,0,2,2,ICMPFlood,2,True,False +5gnidd,0.25,0,2,3,SYNFlood,184,False,False +5gnidd,0.25,0,2,4,SYNScan,0,False,True +5gnidd,0.25,0,2,5,SlowrateDoS,1454,False,False +5gnidd,0.25,0,2,6,TCPConnectScan,63,True,False +5gnidd,0.25,0,2,7,UDPFlood,319,False,False +5gnidd,0.25,0,2,8,UDPScan,4159,False,False +5gnidd,0.25,0,3,0,Benign,4648,False,False +5gnidd,0.25,0,3,1,HTTPFlood,13072,False,False +5gnidd,0.25,0,3,2,ICMPFlood,42,False,False +5gnidd,0.25,0,3,3,SYNFlood,4762,False,False +5gnidd,0.25,0,3,4,SYNScan,3021,False,False +5gnidd,0.25,0,3,5,SlowrateDoS,16517,False,False +5gnidd,0.25,0,3,6,TCPConnectScan,1548,False,False +5gnidd,0.25,0,3,7,UDPFlood,1,True,False +5gnidd,0.25,0,3,8,UDPScan,5,True,False +5gnidd,0.25,0,4,0,Benign,5,False,False +5gnidd,0.25,0,4,1,HTTPFlood,0,False,True +5gnidd,0.25,0,4,2,ICMPFlood,15,True,False +5gnidd,0.25,0,4,3,SYNFlood,144,False,False +5gnidd,0.25,0,4,4,SYNScan,447,False,False +5gnidd,0.25,0,4,5,SlowrateDoS,14314,False,False +5gnidd,0.25,0,4,6,TCPConnectScan,10870,False,False +5gnidd,0.25,0,4,7,UDPFlood,1606,False,False +5gnidd,0.25,0,4,8,UDPScan,28,True,False +5gnidd,0.25,0,5,0,Benign,185504,False,False +5gnidd,0.25,0,5,1,HTTPFlood,48469,False,False +5gnidd,0.25,0,5,2,ICMPFlood,0,False,True +5gnidd,0.25,0,5,3,SYNFlood,0,False,True +5gnidd,0.25,0,5,4,SYNScan,0,False,True +5gnidd,0.25,0,5,5,SlowrateDoS,1,True,False +5gnidd,0.25,0,5,6,TCPConnectScan,118,True,False +5gnidd,0.25,0,5,7,UDPFlood,2869,False,False +5gnidd,0.25,0,5,8,UDPScan,732,False,False +5gnidd,0.25,0,6,0,Benign,1807,False,False +5gnidd,0.25,0,6,1,HTTPFlood,3,True,False +5gnidd,0.25,0,6,2,ICMPFlood,78,False,False +5gnidd,0.25,0,6,3,SYNFlood,559,False,False +5gnidd,0.25,0,6,4,SYNScan,2557,False,False +5gnidd,0.25,0,6,5,SlowrateDoS,23,True,False +5gnidd,0.25,0,6,6,TCPConnectScan,0,False,True +5gnidd,0.25,0,6,7,UDPFlood,94879,False,False +5gnidd,0.25,0,6,8,UDPScan,1052,False,False +5gnidd,0.25,0,7,0,Benign,1,False,False +5gnidd,0.25,0,7,1,HTTPFlood,26635,False,False +5gnidd,0.25,0,7,2,ICMPFlood,34,True,False +5gnidd,0.25,0,7,3,SYNFlood,0,False,True +5gnidd,0.25,0,7,4,SYNScan,0,False,True +5gnidd,0.25,0,7,5,SlowrateDoS,0,False,True +5gnidd,0.25,0,7,6,TCPConnectScan,81,True,False +5gnidd,0.25,0,7,7,UDPFlood,26932,False,False +5gnidd,0.25,0,7,8,UDPScan,389,False,False +5gnidd,0.25,0,8,0,Benign,63395,False,False +5gnidd,0.25,0,8,1,HTTPFlood,589,False,False +5gnidd,0.25,0,8,2,ICMPFlood,6,True,False +5gnidd,0.25,0,8,3,SYNFlood,419,False,False +5gnidd,0.25,0,8,4,SYNScan,237,False,False +5gnidd,0.25,0,8,5,SlowrateDoS,810,False,False +5gnidd,0.25,0,8,6,TCPConnectScan,2,True,False +5gnidd,0.25,0,8,7,UDPFlood,1195,False,False +5gnidd,0.25,0,8,8,UDPScan,145,False,False +5gnidd,0.25,0,9,0,Benign,15071,False,False +5gnidd,0.25,0,9,1,HTTPFlood,4613,False,False +5gnidd,0.25,0,9,2,ICMPFlood,131,True,False +5gnidd,0.25,0,9,3,SYNFlood,462,False,False +5gnidd,0.25,0,9,4,SYNScan,6524,False,False +5gnidd,0.25,0,9,5,SlowrateDoS,6096,False,False +5gnidd,0.25,0,9,6,TCPConnectScan,1211,False,False +5gnidd,0.25,0,9,7,UDPFlood,2509,False,False +5gnidd,0.25,0,9,8,UDPScan,21,True,False diff --git a/artifacts/analysis/5gnidd__alpha0.25__seed1__characterization.csv b/artifacts/analysis/5gnidd__alpha0.25__seed1__characterization.csv new file mode 100644 index 0000000..2252dff --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.25__seed1__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,30,177603,33441.5,50907.30907492558,1.5222794753502558,5920.1 +1,HTTPFlood,98569,9,1,5,45645,10952.111111111111,15449.722071893897,1.4106615532981472,9129.0 +2,ICMPFlood,808,10,0,1,257,80.8,94.58308516854375,1.1705827372344524,257.0 +3,SYNFlood,6805,9,1,10,2997,756.1111111111111,936.8245945680835,1.2390038723163486,299.7 +4,SYNScan,14030,10,0,7,3477,1403.0,1296.782711174081,0.9242927378289957,496.7142857142857 +5,SlowrateDoS,51187,8,2,211,23478,6398.375,7648.639175328837,1.1954033915375133,111.27014218009478 +6,TCPConnectScan,14037,8,2,36,3627,1754.625,1056.293039063971,0.6020050090839758,100.75 +7,UDPFlood,320138,10,0,1,128338,32013.8,41188.815061373156,1.2865956263040674,128338.0 +8,UDPScan,11134,6,4,67,5172,1855.6666666666667,1682.0645713593226,0.9064475865058321,77.19402985074628 diff --git a/artifacts/analysis/5gnidd__alpha0.25__seed1__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.25__seed1__rare_unseen.csv new file mode 100644 index 0000000..88a1a31 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.25__seed1__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.25,1,0,0,Benign,39367,False,False +5gnidd,0.25,1,0,1,HTTPFlood,0,False,True +5gnidd,0.25,1,0,2,ICMPFlood,27,True,False +5gnidd,0.25,1,0,3,SYNFlood,2997,False,False +5gnidd,0.25,1,0,4,SYNScan,3051,False,False +5gnidd,0.25,1,0,5,SlowrateDoS,0,False,True +5gnidd,0.25,1,0,6,TCPConnectScan,368,True,False +5gnidd,0.25,1,0,7,UDPFlood,16667,False,False +5gnidd,0.25,1,0,8,UDPScan,0,False,True +5gnidd,0.25,1,1,0,Benign,6206,False,False +5gnidd,0.25,1,1,1,HTTPFlood,45645,False,False +5gnidd,0.25,1,1,2,ICMPFlood,78,True,False +5gnidd,0.25,1,1,3,SYNFlood,0,False,True +5gnidd,0.25,1,1,4,SYNScan,7,True,False +5gnidd,0.25,1,1,5,SlowrateDoS,442,False,False +5gnidd,0.25,1,1,6,TCPConnectScan,1716,False,False +5gnidd,0.25,1,1,7,UDPFlood,4540,False,False +5gnidd,0.25,1,1,8,UDPScan,5172,False,False +5gnidd,0.25,1,2,0,Benign,30,False,False +5gnidd,0.25,1,2,1,HTTPFlood,28,False,False +5gnidd,0.25,1,2,2,ICMPFlood,4,True,False +5gnidd,0.25,1,2,3,SYNFlood,408,False,False +5gnidd,0.25,1,2,4,SYNScan,3477,False,False +5gnidd,0.25,1,2,5,SlowrateDoS,290,False,False +5gnidd,0.25,1,2,6,TCPConnectScan,1886,False,False +5gnidd,0.25,1,2,7,UDPFlood,13,True,False +5gnidd,0.25,1,2,8,UDPScan,2745,False,False +5gnidd,0.25,1,3,0,Benign,5914,False,False +5gnidd,0.25,1,3,1,HTTPFlood,365,False,False +5gnidd,0.25,1,3,2,ICMPFlood,225,True,False +5gnidd,0.25,1,3,3,SYNFlood,708,False,False +5gnidd,0.25,1,3,4,SYNScan,362,True,False +5gnidd,0.25,1,3,5,SlowrateDoS,11227,False,False +5gnidd,0.25,1,3,6,TCPConnectScan,0,False,True +5gnidd,0.25,1,3,7,UDPFlood,75580,False,False +5gnidd,0.25,1,3,8,UDPScan,0,False,True +5gnidd,0.25,1,4,0,Benign,42476,False,False +5gnidd,0.25,1,4,1,HTTPFlood,30225,False,False +5gnidd,0.25,1,4,2,ICMPFlood,257,True,False +5gnidd,0.25,1,4,3,SYNFlood,1743,False,False +5gnidd,0.25,1,4,4,SYNScan,1514,False,False +5gnidd,0.25,1,4,5,SlowrateDoS,6019,False,False +5gnidd,0.25,1,4,6,TCPConnectScan,1935,False,False +5gnidd,0.25,1,4,7,UDPFlood,1,True,False +5gnidd,0.25,1,4,8,UDPScan,0,False,True +5gnidd,0.25,1,5,0,Benign,3442,False,False +5gnidd,0.25,1,5,1,HTTPFlood,5,True,False +5gnidd,0.25,1,5,2,ICMPFlood,2,True,False +5gnidd,0.25,1,5,3,SYNFlood,97,False,False +5gnidd,0.25,1,5,4,SYNScan,17,False,False +5gnidd,0.25,1,5,5,SlowrateDoS,211,False,False +5gnidd,0.25,1,5,6,TCPConnectScan,2361,False,False +5gnidd,0.25,1,5,7,UDPFlood,1197,False,False +5gnidd,0.25,1,5,8,UDPScan,67,False,False +5gnidd,0.25,1,6,0,Benign,43284,False,False +5gnidd,0.25,1,6,1,HTTPFlood,660,False,False +5gnidd,0.25,1,6,2,ICMPFlood,1,True,False +5gnidd,0.25,1,6,3,SYNFlood,10,True,False +5gnidd,0.25,1,6,4,SYNScan,1401,False,False +5gnidd,0.25,1,6,5,SlowrateDoS,9026,False,False +5gnidd,0.25,1,6,6,TCPConnectScan,3627,False,False +5gnidd,0.25,1,6,7,UDPFlood,60225,False,False +5gnidd,0.25,1,6,8,UDPScan,1209,False,False +5gnidd,0.25,1,7,0,Benign,505,False,False +5gnidd,0.25,1,7,1,HTTPFlood,11422,False,False +5gnidd,0.25,1,7,2,ICMPFlood,29,True,False +5gnidd,0.25,1,7,3,SYNFlood,518,False,False +5gnidd,0.25,1,7,4,SYNScan,107,True,False +5gnidd,0.25,1,7,5,SlowrateDoS,0,False,True +5gnidd,0.25,1,7,6,TCPConnectScan,0,False,True +5gnidd,0.25,1,7,7,UDPFlood,33227,False,False +5gnidd,0.25,1,7,8,UDPScan,1006,False,False +5gnidd,0.25,1,8,0,Benign,15588,False,False +5gnidd,0.25,1,8,1,HTTPFlood,9984,False,False +5gnidd,0.25,1,8,2,ICMPFlood,11,True,False +5gnidd,0.25,1,8,3,SYNFlood,276,False,False +5gnidd,0.25,1,8,4,SYNScan,3125,False,False +5gnidd,0.25,1,8,5,SlowrateDoS,494,False,False +5gnidd,0.25,1,8,6,TCPConnectScan,36,True,False +5gnidd,0.25,1,8,7,UDPFlood,128338,False,False +5gnidd,0.25,1,8,8,UDPScan,0,False,True +5gnidd,0.25,1,9,0,Benign,177603,False,False +5gnidd,0.25,1,9,1,HTTPFlood,235,False,False +5gnidd,0.25,1,9,2,ICMPFlood,174,True,False +5gnidd,0.25,1,9,3,SYNFlood,48,True,False +5gnidd,0.25,1,9,4,SYNScan,969,False,False +5gnidd,0.25,1,9,5,SlowrateDoS,23478,False,False +5gnidd,0.25,1,9,6,TCPConnectScan,2108,False,False +5gnidd,0.25,1,9,7,UDPFlood,350,False,False +5gnidd,0.25,1,9,8,UDPScan,935,False,False diff --git a/artifacts/analysis/5gnidd__alpha0.25__seed2__characterization.csv b/artifacts/analysis/5gnidd__alpha0.25__seed2__characterization.csv new file mode 100644 index 0000000..8912f50 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.25__seed2__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,10,227468,33441.5,65962.28801709352,1.972467981911503,22746.8 +1,HTTPFlood,98569,7,3,43,36762,14081.285714285714,13455.260572029978,0.9555420467308164,854.9302325581396 +2,ICMPFlood,808,8,2,6,355,101.0,118.30574795841494,1.1713440391922272,59.166666666666664 +3,SYNFlood,6805,9,1,3,3136,756.1111111111111,1064.1515602628585,1.4074010348810768,1045.3333333333333 +4,SYNScan,14030,10,0,2,10072,1403.0,2944.64989429983,2.098823873342716,5036.0 +5,SlowrateDoS,51187,10,0,4,31740,5118.7,9212.276190497112,1.7997296560644525,7935.0 +6,TCPConnectScan,14037,9,1,3,3619,1559.6666666666667,1403.3240696448004,0.8997589675004063,1206.3333333333333 +7,UDPFlood,320138,10,0,113,82051,32013.8,25362.204812673524,0.7922272523934529,726.1150442477876 +8,UDPScan,11134,7,3,243,3899,1590.5714285714287,1346.1352996255464,0.8463218158234977,16.045267489711936 diff --git a/artifacts/analysis/5gnidd__alpha0.25__seed2__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.25__seed2__rare_unseen.csv new file mode 100644 index 0000000..e44c94c --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.25__seed2__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.25,2,0,0,Benign,46403,False,False +5gnidd,0.25,2,0,1,HTTPFlood,0,False,True +5gnidd,0.25,2,0,2,ICMPFlood,203,True,False +5gnidd,0.25,2,0,3,SYNFlood,258,False,False +5gnidd,0.25,2,0,4,SYNScan,4,True,False +5gnidd,0.25,2,0,5,SlowrateDoS,226,False,False +5gnidd,0.25,2,0,6,TCPConnectScan,2548,False,False +5gnidd,0.25,2,0,7,UDPFlood,55627,False,False +5gnidd,0.25,2,0,8,UDPScan,3899,False,False +5gnidd,0.25,2,1,0,Benign,20110,False,False +5gnidd,0.25,2,1,1,HTTPFlood,1015,False,False +5gnidd,0.25,2,1,2,ICMPFlood,6,True,False +5gnidd,0.25,2,1,3,SYNFlood,3,True,False +5gnidd,0.25,2,1,4,SYNScan,843,False,False +5gnidd,0.25,2,1,5,SlowrateDoS,692,False,False +5gnidd,0.25,2,1,6,TCPConnectScan,2980,False,False +5gnidd,0.25,2,1,7,UDPFlood,25373,False,False +5gnidd,0.25,2,1,8,UDPScan,0,False,True +5gnidd,0.25,2,2,0,Benign,12386,False,False +5gnidd,0.25,2,2,1,HTTPFlood,0,False,True +5gnidd,0.25,2,2,2,ICMPFlood,0,False,True +5gnidd,0.25,2,2,3,SYNFlood,3136,False,False +5gnidd,0.25,2,2,4,SYNScan,10072,False,False +5gnidd,0.25,2,2,5,SlowrateDoS,1818,False,False +5gnidd,0.25,2,2,6,TCPConnectScan,564,False,False +5gnidd,0.25,2,2,7,UDPFlood,113,True,False +5gnidd,0.25,2,2,8,UDPScan,436,True,False +5gnidd,0.25,2,3,0,Benign,29,False,False +5gnidd,0.25,2,3,1,HTTPFlood,13422,False,False +5gnidd,0.25,2,3,2,ICMPFlood,355,True,False +5gnidd,0.25,2,3,3,SYNFlood,221,True,False +5gnidd,0.25,2,3,4,SYNScan,448,False,False +5gnidd,0.25,2,3,5,SlowrateDoS,31740,False,False +5gnidd,0.25,2,3,6,TCPConnectScan,1136,False,False +5gnidd,0.25,2,3,7,UDPFlood,61079,False,False +5gnidd,0.25,2,3,8,UDPScan,0,False,True +5gnidd,0.25,2,4,0,Benign,3704,False,False +5gnidd,0.25,2,4,1,HTTPFlood,43,True,False +5gnidd,0.25,2,4,2,ICMPFlood,155,False,False +5gnidd,0.25,2,4,3,SYNFlood,2230,False,False +5gnidd,0.25,2,4,4,SYNScan,117,False,False +5gnidd,0.25,2,4,5,SlowrateDoS,8482,False,False +5gnidd,0.25,2,4,6,TCPConnectScan,6,True,False +5gnidd,0.25,2,4,7,UDPFlood,4837,False,False +5gnidd,0.25,2,4,8,UDPScan,243,False,False +5gnidd,0.25,2,5,0,Benign,12499,False,False +5gnidd,0.25,2,5,1,HTTPFlood,578,False,False +5gnidd,0.25,2,5,2,ICMPFlood,0,False,True +5gnidd,0.25,2,5,3,SYNFlood,4,True,False +5gnidd,0.25,2,5,4,SYNScan,2,True,False +5gnidd,0.25,2,5,5,SlowrateDoS,69,False,False +5gnidd,0.25,2,5,6,TCPConnectScan,94,False,False +5gnidd,0.25,2,5,7,UDPFlood,5864,False,False +5gnidd,0.25,2,5,8,UDPScan,1018,False,False +5gnidd,0.25,2,6,0,Benign,227468,False,False +5gnidd,0.25,2,6,1,HTTPFlood,36762,False,False +5gnidd,0.25,2,6,2,ICMPFlood,17,True,False +5gnidd,0.25,2,6,3,SYNFlood,72,False,False +5gnidd,0.25,2,6,4,SYNScan,32,False,False +5gnidd,0.25,2,6,5,SlowrateDoS,4423,False,False +5gnidd,0.25,2,6,6,TCPConnectScan,3,True,False +5gnidd,0.25,2,6,7,UDPFlood,28468,False,False +5gnidd,0.25,2,6,8,UDPScan,947,False,False +5gnidd,0.25,2,7,0,Benign,5134,False,False +5gnidd,0.25,2,7,1,HTTPFlood,27221,False,False +5gnidd,0.25,2,7,2,ICMPFlood,28,True,False +5gnidd,0.25,2,7,3,SYNFlood,0,False,True +5gnidd,0.25,2,7,4,SYNScan,1955,False,False +5gnidd,0.25,2,7,5,SlowrateDoS,4,True,False +5gnidd,0.25,2,7,6,TCPConnectScan,0,False,True +5gnidd,0.25,2,7,7,UDPFlood,27150,False,False +5gnidd,0.25,2,7,8,UDPScan,1184,False,False +5gnidd,0.25,2,8,0,Benign,10,False,False +5gnidd,0.25,2,8,1,HTTPFlood,0,False,True +5gnidd,0.25,2,8,2,ICMPFlood,11,True,False +5gnidd,0.25,2,8,3,SYNFlood,543,False,False +5gnidd,0.25,2,8,4,SYNScan,429,True,False +5gnidd,0.25,2,8,5,SlowrateDoS,2548,False,False +5gnidd,0.25,2,8,6,TCPConnectScan,3087,False,False +5gnidd,0.25,2,8,7,UDPFlood,82051,False,False +5gnidd,0.25,2,8,8,UDPScan,0,False,True +5gnidd,0.25,2,9,0,Benign,6672,False,False +5gnidd,0.25,2,9,1,HTTPFlood,19528,False,False +5gnidd,0.25,2,9,2,ICMPFlood,33,True,False +5gnidd,0.25,2,9,3,SYNFlood,338,False,False +5gnidd,0.25,2,9,4,SYNScan,128,True,False +5gnidd,0.25,2,9,5,SlowrateDoS,1185,False,False +5gnidd,0.25,2,9,6,TCPConnectScan,3619,False,False +5gnidd,0.25,2,9,7,UDPFlood,29576,False,False +5gnidd,0.25,2,9,8,UDPScan,3407,False,False diff --git a/artifacts/analysis/5gnidd__alpha0.5__seed0__characterization.csv b/artifacts/analysis/5gnidd__alpha0.5__seed0__characterization.csv new file mode 100644 index 0000000..04fcddf --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.5__seed0__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,226,108257,33441.5,35759.58709842718,1.0693176770906563,479.0132743362832 +1,HTTPFlood,98569,10,0,185,61393,9856.9,17538.944828295687,1.7793570826827592,331.8540540540541 +2,ICMPFlood,808,9,1,8,243,89.77777777777777,72.6342095369787,0.8090444131594163,30.375 +3,SYNFlood,6805,9,1,2,3250,756.1111111111111,996.1904597509281,1.3175186095162899,1625.0 +4,SYNScan,14030,10,0,11,6713,1403.0,1984.1221232575379,1.4141996601978175,610.2727272727273 +5,SlowrateDoS,51187,10,0,80,28363,5118.7,9127.314238591767,1.7831313104092381,354.5375 +6,TCPConnectScan,14037,10,0,1,4175,1403.7,1379.0263993122105,0.9824224544505311,4175.0 +7,UDPFlood,320138,10,0,1,88943,32013.8,28692.11713624493,0.8962421560778455,88943.0 +8,UDPScan,11134,10,0,14,3420,1113.4,1075.9248300880504,0.9663416832118289,244.28571428571428 diff --git a/artifacts/analysis/5gnidd__alpha0.5__seed0__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.5__seed0__rare_unseen.csv new file mode 100644 index 0000000..41204ae --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.5__seed0__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.5,0,0,0,Benign,108257,False,False +5gnidd,0.5,0,0,1,HTTPFlood,5466,False,False +5gnidd,0.5,0,0,2,ICMPFlood,173,False,False +5gnidd,0.5,0,0,3,SYNFlood,373,False,False +5gnidd,0.5,0,0,4,SYNScan,695,False,False +5gnidd,0.5,0,0,5,SlowrateDoS,455,False,False +5gnidd,0.5,0,0,6,TCPConnectScan,6,True,False +5gnidd,0.5,0,0,7,UDPFlood,1,True,False +5gnidd,0.5,0,0,8,UDPScan,1359,False,False +5gnidd,0.5,0,1,0,Benign,6399,False,False +5gnidd,0.5,0,1,1,HTTPFlood,6782,False,False +5gnidd,0.5,0,1,2,ICMPFlood,69,True,False +5gnidd,0.5,0,1,3,SYNFlood,3250,False,False +5gnidd,0.5,0,1,4,SYNScan,63,True,False +5gnidd,0.5,0,1,5,SlowrateDoS,1431,False,False +5gnidd,0.5,0,1,6,TCPConnectScan,2068,False,False +5gnidd,0.5,0,1,7,UDPFlood,57810,False,False +5gnidd,0.5,0,1,8,UDPScan,289,False,False +5gnidd,0.5,0,2,0,Benign,68144,False,False +5gnidd,0.5,0,2,1,HTTPFlood,3688,False,False +5gnidd,0.5,0,2,2,ICMPFlood,57,True,False +5gnidd,0.5,0,2,3,SYNFlood,72,True,False +5gnidd,0.5,0,2,4,SYNScan,100,False,False +5gnidd,0.5,0,2,5,SlowrateDoS,80,False,False +5gnidd,0.5,0,2,6,TCPConnectScan,2059,False,False +5gnidd,0.5,0,2,7,UDPFlood,23607,False,False +5gnidd,0.5,0,2,8,UDPScan,212,False,False +5gnidd,0.5,0,3,0,Benign,22029,False,False +5gnidd,0.5,0,3,1,HTTPFlood,4983,False,False +5gnidd,0.5,0,3,2,ICMPFlood,0,False,True +5gnidd,0.5,0,3,3,SYNFlood,367,True,False +5gnidd,0.5,0,3,4,SYNScan,667,False,False +5gnidd,0.5,0,3,5,SlowrateDoS,16850,False,False +5gnidd,0.5,0,3,6,TCPConnectScan,2108,False,False +5gnidd,0.5,0,3,7,UDPFlood,129,True,False +5gnidd,0.5,0,3,8,UDPScan,1403,False,False +5gnidd,0.5,0,4,0,Benign,1096,False,False +5gnidd,0.5,0,4,1,HTTPFlood,388,False,False +5gnidd,0.5,0,4,2,ICMPFlood,8,True,False +5gnidd,0.5,0,4,3,SYNFlood,0,False,True +5gnidd,0.5,0,4,4,SYNScan,11,True,False +5gnidd,0.5,0,4,5,SlowrateDoS,1630,False,False +5gnidd,0.5,0,4,6,TCPConnectScan,2823,False,False +5gnidd,0.5,0,4,7,UDPFlood,44903,False,False +5gnidd,0.5,0,4,8,UDPScan,3420,False,False +5gnidd,0.5,0,5,0,Benign,33517,False,False +5gnidd,0.5,0,5,1,HTTPFlood,185,False,False +5gnidd,0.5,0,5,2,ICMPFlood,12,True,False +5gnidd,0.5,0,5,3,SYNFlood,1592,False,False +5gnidd,0.5,0,5,4,SYNScan,583,False,False +5gnidd,0.5,0,5,5,SlowrateDoS,692,False,False +5gnidd,0.5,0,5,6,TCPConnectScan,1,True,False +5gnidd,0.5,0,5,7,UDPFlood,88943,False,False +5gnidd,0.5,0,5,8,UDPScan,1057,False,False +5gnidd,0.5,0,6,0,Benign,226,False,False +5gnidd,0.5,0,6,1,HTTPFlood,843,False,False +5gnidd,0.5,0,6,2,ICMPFlood,101,False,False +5gnidd,0.5,0,6,3,SYNFlood,10,True,False +5gnidd,0.5,0,6,4,SYNScan,317,False,False +5gnidd,0.5,0,6,5,SlowrateDoS,28363,False,False +5gnidd,0.5,0,6,6,TCPConnectScan,9,True,False +5gnidd,0.5,0,6,7,UDPFlood,14100,False,False +5gnidd,0.5,0,6,8,UDPScan,14,False,False +5gnidd,0.5,0,7,0,Benign,9926,False,False +5gnidd,0.5,0,7,1,HTTPFlood,2258,False,False +5gnidd,0.5,0,7,2,ICMPFlood,105,False,False +5gnidd,0.5,0,7,3,SYNFlood,2,True,False +5gnidd,0.5,0,7,4,SYNScan,1852,False,False +5gnidd,0.5,0,7,5,SlowrateDoS,185,False,False +5gnidd,0.5,0,7,6,TCPConnectScan,235,False,False +5gnidd,0.5,0,7,7,UDPFlood,733,False,False +5gnidd,0.5,0,7,8,UDPScan,92,True,False +5gnidd,0.5,0,8,0,Benign,76033,False,False +5gnidd,0.5,0,8,1,HTTPFlood,61393,False,False +5gnidd,0.5,0,8,2,ICMPFlood,243,True,False +5gnidd,0.5,0,8,3,SYNFlood,408,True,False +5gnidd,0.5,0,8,4,SYNScan,6713,False,False +5gnidd,0.5,0,8,5,SlowrateDoS,477,False,False +5gnidd,0.5,0,8,6,TCPConnectScan,553,False,False +5gnidd,0.5,0,8,7,UDPFlood,30580,False,False +5gnidd,0.5,0,8,8,UDPScan,2593,False,False +5gnidd,0.5,0,9,0,Benign,8788,False,False +5gnidd,0.5,0,9,1,HTTPFlood,12583,False,False +5gnidd,0.5,0,9,2,ICMPFlood,40,True,False +5gnidd,0.5,0,9,3,SYNFlood,731,False,False +5gnidd,0.5,0,9,4,SYNScan,3029,False,False +5gnidd,0.5,0,9,5,SlowrateDoS,1024,False,False +5gnidd,0.5,0,9,6,TCPConnectScan,4175,False,False +5gnidd,0.5,0,9,7,UDPFlood,59332,False,False +5gnidd,0.5,0,9,8,UDPScan,695,True,False diff --git a/artifacts/analysis/5gnidd__alpha0.5__seed1__characterization.csv b/artifacts/analysis/5gnidd__alpha0.5__seed1__characterization.csv new file mode 100644 index 0000000..628d73c --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.5__seed1__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,803,104938,33441.5,31374.111025652983,0.938178940108936,130.68244084682442 +1,HTTPFlood,98569,10,0,51,36928,9856.9,10966.810999100877,1.1126024408384865,724.0784313725491 +2,ICMPFlood,808,10,0,8,284,80.8,81.98999939016953,1.0147277152248704,35.5 +3,SYNFlood,6805,10,0,26,2093,680.5,611.6053057323816,0.8987587152569898,80.5 +4,SYNScan,14030,10,0,259,3621,1403.0,1206.1257811687801,0.8596762517240057,13.980694980694981 +5,SlowrateDoS,51187,10,0,26,16089,5118.7,5007.844267746353,0.978342990944254,618.8076923076923 +6,TCPConnectScan,14037,10,0,3,7188,1403.7,2021.4343447166423,1.4400757602882683,2396.0 +7,UDPFlood,320138,10,0,1907,95125,32013.8,26741.87702761345,0.8353234238863693,49.88201363398007 +8,UDPScan,11134,10,0,1,5065,1113.4,1534.664666954967,1.3783587811702596,5065.0 diff --git a/artifacts/analysis/5gnidd__alpha0.5__seed1__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.5__seed1__rare_unseen.csv new file mode 100644 index 0000000..5c4637c --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.5__seed1__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.5,1,0,0,Benign,104938,False,False +5gnidd,0.5,1,0,1,HTTPFlood,13117,False,False +5gnidd,0.5,1,0,2,ICMPFlood,18,True,False +5gnidd,0.5,1,0,3,SYNFlood,393,False,False +5gnidd,0.5,1,0,4,SYNScan,1017,False,False +5gnidd,0.5,1,0,5,SlowrateDoS,711,False,False +5gnidd,0.5,1,0,6,TCPConnectScan,1151,False,False +5gnidd,0.5,1,0,7,UDPFlood,9498,False,False +5gnidd,0.5,1,0,8,UDPScan,243,True,False +5gnidd,0.5,1,1,0,Benign,2307,False,False +5gnidd,0.5,1,1,1,HTTPFlood,36928,False,False +5gnidd,0.5,1,1,2,ICMPFlood,30,True,False +5gnidd,0.5,1,1,3,SYNFlood,151,True,False +5gnidd,0.5,1,1,4,SYNScan,2238,False,False +5gnidd,0.5,1,1,5,SlowrateDoS,1838,False,False +5gnidd,0.5,1,1,6,TCPConnectScan,1036,False,False +5gnidd,0.5,1,1,7,UDPFlood,95125,False,False +5gnidd,0.5,1,1,8,UDPScan,491,False,False +5gnidd,0.5,1,2,0,Benign,803,False,False +5gnidd,0.5,1,2,1,HTTPFlood,943,False,False +5gnidd,0.5,1,2,2,ICMPFlood,45,True,False +5gnidd,0.5,1,2,3,SYNFlood,825,False,False +5gnidd,0.5,1,2,4,SYNScan,616,False,False +5gnidd,0.5,1,2,5,SlowrateDoS,10016,False,False +5gnidd,0.5,1,2,6,TCPConnectScan,595,True,False +5gnidd,0.5,1,2,7,UDPFlood,17027,False,False +5gnidd,0.5,1,2,8,UDPScan,743,False,False +5gnidd,0.5,1,3,0,Benign,12946,False,False +5gnidd,0.5,1,3,1,HTTPFlood,51,False,False +5gnidd,0.5,1,3,2,ICMPFlood,8,True,False +5gnidd,0.5,1,3,3,SYNFlood,194,False,False +5gnidd,0.5,1,3,4,SYNScan,443,False,False +5gnidd,0.5,1,3,5,SlowrateDoS,4208,False,False +5gnidd,0.5,1,3,6,TCPConnectScan,50,False,False +5gnidd,0.5,1,3,7,UDPFlood,39604,False,False +5gnidd,0.5,1,3,8,UDPScan,17,True,False +5gnidd,0.5,1,4,0,Benign,25034,False,False +5gnidd,0.5,1,4,1,HTTPFlood,269,False,False +5gnidd,0.5,1,4,2,ICMPFlood,10,True,False +5gnidd,0.5,1,4,3,SYNFlood,1146,False,False +5gnidd,0.5,1,4,4,SYNScan,387,False,False +5gnidd,0.5,1,4,5,SlowrateDoS,7838,False,False +5gnidd,0.5,1,4,6,TCPConnectScan,162,True,False +5gnidd,0.5,1,4,7,UDPFlood,25965,False,False +5gnidd,0.5,1,4,8,UDPScan,1442,False,False +5gnidd,0.5,1,5,0,Benign,58927,False,False +5gnidd,0.5,1,5,1,HTTPFlood,6977,False,False +5gnidd,0.5,1,5,2,ICMPFlood,121,False,False +5gnidd,0.5,1,5,3,SYNFlood,104,True,False +5gnidd,0.5,1,5,4,SYNScan,3621,False,False +5gnidd,0.5,1,5,5,SlowrateDoS,8214,False,False +5gnidd,0.5,1,5,6,TCPConnectScan,7188,False,False +5gnidd,0.5,1,5,7,UDPFlood,31787,False,False +5gnidd,0.5,1,5,8,UDPScan,1,True,False +5gnidd,0.5,1,6,0,Benign,10286,False,False +5gnidd,0.5,1,6,1,HTTPFlood,8479,False,False +5gnidd,0.5,1,6,2,ICMPFlood,120,True,False +5gnidd,0.5,1,6,3,SYNFlood,2093,False,False +5gnidd,0.5,1,6,4,SYNScan,1083,True,False +5gnidd,0.5,1,6,5,SlowrateDoS,16089,False,False +5gnidd,0.5,1,6,6,TCPConnectScan,1757,False,False +5gnidd,0.5,1,6,7,UDPFlood,54891,False,False +5gnidd,0.5,1,6,8,UDPScan,2687,False,False +5gnidd,0.5,1,7,0,Benign,55826,False,False +5gnidd,0.5,1,7,1,HTTPFlood,18815,False,False +5gnidd,0.5,1,7,2,ICMPFlood,38,True,False +5gnidd,0.5,1,7,3,SYNFlood,1027,False,False +5gnidd,0.5,1,7,4,SYNScan,845,False,False +5gnidd,0.5,1,7,5,SlowrateDoS,1671,False,False +5gnidd,0.5,1,7,6,TCPConnectScan,401,True,False +5gnidd,0.5,1,7,7,UDPFlood,41124,False,False +5gnidd,0.5,1,7,8,UDPScan,5065,False,False +5gnidd,0.5,1,8,0,Benign,47693,False,False +5gnidd,0.5,1,8,1,HTTPFlood,224,False,False +5gnidd,0.5,1,8,2,ICMPFlood,284,False,False +5gnidd,0.5,1,8,3,SYNFlood,26,True,False +5gnidd,0.5,1,8,4,SYNScan,3521,False,False +5gnidd,0.5,1,8,5,SlowrateDoS,26,False,False +5gnidd,0.5,1,8,6,TCPConnectScan,3,True,False +5gnidd,0.5,1,8,7,UDPFlood,1907,False,False +5gnidd,0.5,1,8,8,UDPScan,75,False,False +5gnidd,0.5,1,9,0,Benign,15655,False,False +5gnidd,0.5,1,9,1,HTTPFlood,12766,False,False +5gnidd,0.5,1,9,2,ICMPFlood,134,True,False +5gnidd,0.5,1,9,3,SYNFlood,846,False,False +5gnidd,0.5,1,9,4,SYNScan,259,True,False +5gnidd,0.5,1,9,5,SlowrateDoS,576,False,False +5gnidd,0.5,1,9,6,TCPConnectScan,1694,False,False +5gnidd,0.5,1,9,7,UDPFlood,3210,False,False +5gnidd,0.5,1,9,8,UDPScan,370,False,False diff --git a/artifacts/analysis/5gnidd__alpha0.5__seed2__characterization.csv b/artifacts/analysis/5gnidd__alpha0.5__seed2__characterization.csv new file mode 100644 index 0000000..50cd950 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.5__seed2__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,255,225038,33441.5,65682.05261294747,1.9640881124634801,882.5019607843137 +1,HTTPFlood,98569,10,0,24,48351,9856.9,15173.43130903488,1.539371537606639,2014.625 +2,ICMPFlood,808,10,0,1,358,80.8,99.01595830975934,1.2254450285861305,358.0 +3,SYNFlood,6805,10,0,8,3530,680.5,1008.8978392285316,1.4825831583079083,441.25 +4,SYNScan,14030,10,0,5,4830,1403.0,1611.374444379704,1.1485206303490407,966.0 +5,SlowrateDoS,51187,10,0,104,17642,5118.7,5260.9995827789235,1.0277999458415072,169.6346153846154 +6,TCPConnectScan,14037,9,1,125,6952,1559.6666666666667,1980.5227143918905,1.2698371752886666,55.616 +7,UDPFlood,320138,10,0,105,138381,32013.8,41299.88736497958,1.290065139564175,1317.9142857142858 +8,UDPScan,11134,9,1,2,6727,1237.111111111111,2051.479219405919,1.6582821065792412,3363.5 diff --git a/artifacts/analysis/5gnidd__alpha0.5__seed2__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.5__seed2__rare_unseen.csv new file mode 100644 index 0000000..96f87d0 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.5__seed2__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.5,2,0,0,Benign,225038,False,False +5gnidd,0.5,2,0,1,HTTPFlood,24,True,False +5gnidd,0.5,2,0,2,ICMPFlood,1,True,False +5gnidd,0.5,2,0,3,SYNFlood,179,False,False +5gnidd,0.5,2,0,4,SYNScan,2446,False,False +5gnidd,0.5,2,0,5,SlowrateDoS,6644,False,False +5gnidd,0.5,2,0,6,TCPConnectScan,6952,False,False +5gnidd,0.5,2,0,7,UDPFlood,14179,False,False +5gnidd,0.5,2,0,8,UDPScan,0,False,True +5gnidd,0.5,2,1,0,Benign,39441,False,False +5gnidd,0.5,2,1,1,HTTPFlood,48351,False,False +5gnidd,0.5,2,1,2,ICMPFlood,2,True,False +5gnidd,0.5,2,1,3,SYNFlood,250,False,False +5gnidd,0.5,2,1,4,SYNScan,454,False,False +5gnidd,0.5,2,1,5,SlowrateDoS,104,False,False +5gnidd,0.5,2,1,6,TCPConnectScan,1256,False,False +5gnidd,0.5,2,1,7,UDPFlood,4099,False,False +5gnidd,0.5,2,1,8,UDPScan,3,True,False +5gnidd,0.5,2,2,0,Benign,4137,False,False +5gnidd,0.5,2,2,1,HTTPFlood,8055,False,False +5gnidd,0.5,2,2,2,ICMPFlood,20,False,False +5gnidd,0.5,2,2,3,SYNFlood,8,True,False +5gnidd,0.5,2,2,4,SYNScan,1510,False,False +5gnidd,0.5,2,2,5,SlowrateDoS,4532,False,False +5gnidd,0.5,2,2,6,TCPConnectScan,833,False,False +5gnidd,0.5,2,2,7,UDPFlood,990,False,False +5gnidd,0.5,2,2,8,UDPScan,2,True,False +5gnidd,0.5,2,3,0,Benign,807,False,False +5gnidd,0.5,2,3,1,HTTPFlood,28464,False,False +5gnidd,0.5,2,3,2,ICMPFlood,76,True,False +5gnidd,0.5,2,3,3,SYNFlood,122,False,False +5gnidd,0.5,2,3,4,SYNScan,319,False,False +5gnidd,0.5,2,3,5,SlowrateDoS,2592,False,False +5gnidd,0.5,2,3,6,TCPConnectScan,365,False,False +5gnidd,0.5,2,3,7,UDPFlood,7413,False,False +5gnidd,0.5,2,3,8,UDPScan,4,True,False +5gnidd,0.5,2,4,0,Benign,43342,False,False +5gnidd,0.5,2,4,1,HTTPFlood,47,True,False +5gnidd,0.5,2,4,2,ICMPFlood,358,False,False +5gnidd,0.5,2,4,3,SYNFlood,683,False,False +5gnidd,0.5,2,4,4,SYNScan,33,True,False +5gnidd,0.5,2,4,5,SlowrateDoS,1353,False,False +5gnidd,0.5,2,4,6,TCPConnectScan,1991,False,False +5gnidd,0.5,2,4,7,UDPFlood,49539,False,False +5gnidd,0.5,2,4,8,UDPScan,2253,False,False +5gnidd,0.5,2,5,0,Benign,255,False,False +5gnidd,0.5,2,5,1,HTTPFlood,582,False,False +5gnidd,0.5,2,5,2,ICMPFlood,122,True,False +5gnidd,0.5,2,5,3,SYNFlood,468,False,False +5gnidd,0.5,2,5,4,SYNScan,5,True,False +5gnidd,0.5,2,5,5,SlowrateDoS,3365,False,False +5gnidd,0.5,2,5,6,TCPConnectScan,125,False,False +5gnidd,0.5,2,5,7,UDPFlood,67573,False,False +5gnidd,0.5,2,5,8,UDPScan,827,False,False +5gnidd,0.5,2,6,0,Benign,1352,False,False +5gnidd,0.5,2,6,1,HTTPFlood,2752,False,False +5gnidd,0.5,2,6,2,ICMPFlood,44,True,False +5gnidd,0.5,2,6,3,SYNFlood,3530,False,False +5gnidd,0.5,2,6,4,SYNScan,3619,False,False +5gnidd,0.5,2,6,5,SlowrateDoS,2995,False,False +5gnidd,0.5,2,6,6,TCPConnectScan,0,False,True +5gnidd,0.5,2,6,7,UDPFlood,105,True,False +5gnidd,0.5,2,6,8,UDPScan,6727,False,False +5gnidd,0.5,2,7,0,Benign,864,False,False +5gnidd,0.5,2,7,1,HTTPFlood,4464,False,False +5gnidd,0.5,2,7,2,ICMPFlood,57,True,False +5gnidd,0.5,2,7,3,SYNFlood,32,True,False +5gnidd,0.5,2,7,4,SYNScan,807,False,False +5gnidd,0.5,2,7,5,SlowrateDoS,17642,False,False +5gnidd,0.5,2,7,6,TCPConnectScan,817,False,False +5gnidd,0.5,2,7,7,UDPFlood,27464,False,False +5gnidd,0.5,2,7,8,UDPScan,567,False,False +5gnidd,0.5,2,8,0,Benign,14286,False,False +5gnidd,0.5,2,8,1,HTTPFlood,5759,False,False +5gnidd,0.5,2,8,2,ICMPFlood,45,True,False +5gnidd,0.5,2,8,3,SYNFlood,1201,False,False +5gnidd,0.5,2,8,4,SYNScan,7,True,False +5gnidd,0.5,2,8,5,SlowrateDoS,390,False,False +5gnidd,0.5,2,8,6,TCPConnectScan,418,False,False +5gnidd,0.5,2,8,7,UDPFlood,138381,False,False +5gnidd,0.5,2,8,8,UDPScan,508,False,False +5gnidd,0.5,2,9,0,Benign,4893,False,False +5gnidd,0.5,2,9,1,HTTPFlood,71,True,False +5gnidd,0.5,2,9,2,ICMPFlood,83,True,False +5gnidd,0.5,2,9,3,SYNFlood,332,False,False +5gnidd,0.5,2,9,4,SYNScan,4830,False,False +5gnidd,0.5,2,9,5,SlowrateDoS,11570,False,False +5gnidd,0.5,2,9,6,TCPConnectScan,1280,False,False +5gnidd,0.5,2,9,7,UDPFlood,10395,False,False +5gnidd,0.5,2,9,8,UDPScan,243,False,False diff --git a/artifacts/analysis/5gnidd__alpha0.75__seed0__characterization.csv b/artifacts/analysis/5gnidd__alpha0.75__seed0__characterization.csv new file mode 100644 index 0000000..47d3058 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.75__seed0__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,1410,69384,33441.5,23619.248452268755,0.706285556935806,49.208510638297874 +1,HTTPFlood,98569,10,0,557,27843,9856.9,9709.981765688337,0.9850948843640838,49.987432675044886 +2,ICMPFlood,808,8,2,25,239,101.0,62.930517239253646,0.6230744281114222,9.56 +3,SYNFlood,6805,9,1,26,3213,756.1111111111111,956.5878532511555,1.2651419073123291,123.57692307692308 +4,SYNScan,14030,10,0,82,4335,1403.0,1341.5857035612746,0.9562264458740375,52.86585365853659 +5,SlowrateDoS,51187,10,0,2,15285,5118.7,5679.8754748673855,1.109632421292005,7642.5 +6,TCPConnectScan,14037,10,0,52,3531,1403.7,1167.4861926378403,0.8317205903240296,67.90384615384616 +7,UDPFlood,320138,10,0,2320,124158,32013.8,36460.358401420024,1.1388950515533933,53.516379310344824 +8,UDPScan,11134,10,0,74,3370,1113.4,1232.7077674777586,1.1071562488573365,45.54054054054054 diff --git a/artifacts/analysis/5gnidd__alpha0.75__seed0__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.75__seed0__rare_unseen.csv new file mode 100644 index 0000000..271ab7c --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.75__seed0__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.75,0,0,0,Benign,46676,False,False +5gnidd,0.75,0,0,1,HTTPFlood,2633,False,False +5gnidd,0.75,0,0,2,ICMPFlood,0,False,True +5gnidd,0.75,0,0,3,SYNFlood,765,False,False +5gnidd,0.75,0,0,4,SYNScan,252,True,False +5gnidd,0.75,0,0,5,SlowrateDoS,2,True,False +5gnidd,0.75,0,0,6,TCPConnectScan,2742,False,False +5gnidd,0.75,0,0,7,UDPFlood,21728,False,False +5gnidd,0.75,0,0,8,UDPScan,3082,False,False +5gnidd,0.75,0,1,0,Benign,69384,False,False +5gnidd,0.75,0,1,1,HTTPFlood,8301,False,False +5gnidd,0.75,0,1,2,ICMPFlood,239,False,False +5gnidd,0.75,0,1,3,SYNFlood,26,True,False +5gnidd,0.75,0,1,4,SYNScan,82,True,False +5gnidd,0.75,0,1,5,SlowrateDoS,11818,False,False +5gnidd,0.75,0,1,6,TCPConnectScan,386,False,False +5gnidd,0.75,0,1,7,UDPFlood,6626,False,False +5gnidd,0.75,0,1,8,UDPScan,886,False,False +5gnidd,0.75,0,2,0,Benign,7505,False,False +5gnidd,0.75,0,2,1,HTTPFlood,7904,False,False +5gnidd,0.75,0,2,2,ICMPFlood,80,True,False +5gnidd,0.75,0,2,3,SYNFlood,50,True,False +5gnidd,0.75,0,2,4,SYNScan,1089,False,False +5gnidd,0.75,0,2,5,SlowrateDoS,585,False,False +5gnidd,0.75,0,2,6,TCPConnectScan,1933,False,False +5gnidd,0.75,0,2,7,UDPFlood,124158,False,False +5gnidd,0.75,0,2,8,UDPScan,243,False,False +5gnidd,0.75,0,3,0,Benign,58004,False,False +5gnidd,0.75,0,3,1,HTTPFlood,4612,False,False +5gnidd,0.75,0,3,2,ICMPFlood,0,False,True +5gnidd,0.75,0,3,3,SYNFlood,3213,False,False +5gnidd,0.75,0,3,4,SYNScan,697,True,False +5gnidd,0.75,0,3,5,SlowrateDoS,15285,False,False +5gnidd,0.75,0,3,6,TCPConnectScan,363,True,False +5gnidd,0.75,0,3,7,UDPFlood,2320,False,False +5gnidd,0.75,0,3,8,UDPScan,2343,False,False +5gnidd,0.75,0,4,0,Benign,3818,False,False +5gnidd,0.75,0,4,1,HTTPFlood,17423,False,False +5gnidd,0.75,0,4,2,ICMPFlood,54,True,False +5gnidd,0.75,0,4,3,SYNFlood,0,False,True +5gnidd,0.75,0,4,4,SYNScan,2163,False,False +5gnidd,0.75,0,4,5,SlowrateDoS,10667,False,False +5gnidd,0.75,0,4,6,TCPConnectScan,1926,True,False +5gnidd,0.75,0,4,7,UDPFlood,2960,False,False +5gnidd,0.75,0,4,8,UDPScan,3370,False,False +5gnidd,0.75,0,5,0,Benign,21356,False,False +5gnidd,0.75,0,5,1,HTTPFlood,1375,False,False +5gnidd,0.75,0,5,2,ICMPFlood,143,True,False +5gnidd,0.75,0,5,3,SYNFlood,180,True,False +5gnidd,0.75,0,5,4,SYNScan,645,False,False +5gnidd,0.75,0,5,5,SlowrateDoS,1615,False,False +5gnidd,0.75,0,5,6,TCPConnectScan,379,False,False +5gnidd,0.75,0,5,7,UDPFlood,23593,False,False +5gnidd,0.75,0,5,8,UDPScan,198,False,False +5gnidd,0.75,0,6,0,Benign,42870,False,False +5gnidd,0.75,0,6,1,HTTPFlood,557,False,False +5gnidd,0.75,0,6,2,ICMPFlood,53,True,False +5gnidd,0.75,0,6,3,SYNFlood,706,False,False +5gnidd,0.75,0,6,4,SYNScan,4335,False,False +5gnidd,0.75,0,6,5,SlowrateDoS,612,False,False +5gnidd,0.75,0,6,6,TCPConnectScan,2301,False,False +5gnidd,0.75,0,6,7,UDPFlood,41823,False,False +5gnidd,0.75,0,6,8,UDPScan,429,True,False +5gnidd,0.75,0,7,0,Benign,1410,False,False +5gnidd,0.75,0,7,1,HTTPFlood,25878,False,False +5gnidd,0.75,0,7,2,ICMPFlood,101,True,False +5gnidd,0.75,0,7,3,SYNFlood,100,True,False +5gnidd,0.75,0,7,4,SYNScan,3133,False,False +5gnidd,0.75,0,7,5,SlowrateDoS,277,False,False +5gnidd,0.75,0,7,6,TCPConnectScan,3531,False,False +5gnidd,0.75,0,7,7,UDPFlood,4005,False,False +5gnidd,0.75,0,7,8,UDPScan,404,False,False +5gnidd,0.75,0,8,0,Benign,24950,False,False +5gnidd,0.75,0,8,1,HTTPFlood,27843,False,False +5gnidd,0.75,0,8,2,ICMPFlood,113,True,False +5gnidd,0.75,0,8,3,SYNFlood,1313,False,False +5gnidd,0.75,0,8,4,SYNScan,145,False,False +5gnidd,0.75,0,8,5,SlowrateDoS,645,False,False +5gnidd,0.75,0,8,6,TCPConnectScan,424,False,False +5gnidd,0.75,0,8,7,UDPFlood,24580,False,False +5gnidd,0.75,0,8,8,UDPScan,74,True,False +5gnidd,0.75,0,9,0,Benign,58442,False,False +5gnidd,0.75,0,9,1,HTTPFlood,2043,False,False +5gnidd,0.75,0,9,2,ICMPFlood,25,True,False +5gnidd,0.75,0,9,3,SYNFlood,452,False,False +5gnidd,0.75,0,9,4,SYNScan,1489,False,False +5gnidd,0.75,0,9,5,SlowrateDoS,9681,False,False +5gnidd,0.75,0,9,6,TCPConnectScan,52,True,False +5gnidd,0.75,0,9,7,UDPFlood,68345,False,False +5gnidd,0.75,0,9,8,UDPScan,105,False,False diff --git a/artifacts/analysis/5gnidd__alpha0.75__seed1__characterization.csv b/artifacts/analysis/5gnidd__alpha0.75__seed1__characterization.csv new file mode 100644 index 0000000..eb78b9c --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.75__seed1__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,3896,83388,33441.5,30605.998007743514,0.9152100835113112,21.403490759753595 +1,HTTPFlood,98569,10,0,621,25248,9856.9,6960.070365305224,0.7061114919807672,40.65700483091788 +2,ICMPFlood,808,9,1,15,285,89.77777777777777,99.0586557750034,1.103376116305731,19.0 +3,SYNFlood,6805,10,0,57,2358,680.5,692.489025183793,1.0176179650019002,41.36842105263158 +4,SYNScan,14030,9,1,133,3883,1558.888888888889,1236.704172337323,0.7933241305086177,29.195488721804512 +5,SlowrateDoS,51187,10,0,514,12114,5118.7,3205.4039698608976,0.6262144626293585,23.56809338521401 +6,TCPConnectScan,14037,10,0,326,2861,1403.7,788.4842484159084,0.5617184928516836,8.776073619631902 +7,UDPFlood,320138,10,0,1031,114825,32013.8,33884.0853050514,1.0584212216310278,111.37245392822503 +8,UDPScan,11134,10,0,5,2727,1113.4,1007.3150649126618,0.9047198355601417,545.4 diff --git a/artifacts/analysis/5gnidd__alpha0.75__seed1__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.75__seed1__rare_unseen.csv new file mode 100644 index 0000000..78cb9a1 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.75__seed1__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.75,1,0,0,Benign,15512,False,False +5gnidd,0.75,1,0,1,HTTPFlood,9688,False,False +5gnidd,0.75,1,0,2,ICMPFlood,27,True,False +5gnidd,0.75,1,0,3,SYNFlood,576,True,False +5gnidd,0.75,1,0,4,SYNScan,2805,False,False +5gnidd,0.75,1,0,5,SlowrateDoS,1642,False,False +5gnidd,0.75,1,0,6,TCPConnectScan,2283,False,False +5gnidd,0.75,1,0,7,UDPFlood,2744,False,False +5gnidd,0.75,1,0,8,UDPScan,1410,False,False +5gnidd,0.75,1,1,0,Benign,5328,False,False +5gnidd,0.75,1,1,1,HTTPFlood,15997,False,False +5gnidd,0.75,1,1,2,ICMPFlood,25,True,False +5gnidd,0.75,1,1,3,SYNFlood,817,False,False +5gnidd,0.75,1,1,4,SYNScan,921,False,False +5gnidd,0.75,1,1,5,SlowrateDoS,6569,False,False +5gnidd,0.75,1,1,6,TCPConnectScan,737,True,False +5gnidd,0.75,1,1,7,UDPFlood,13401,False,False +5gnidd,0.75,1,1,8,UDPScan,2727,False,False +5gnidd,0.75,1,2,0,Benign,8864,False,False +5gnidd,0.75,1,2,1,HTTPFlood,4494,False,False +5gnidd,0.75,1,2,2,ICMPFlood,30,True,False +5gnidd,0.75,1,2,3,SYNFlood,1420,False,False +5gnidd,0.75,1,2,4,SYNScan,133,True,False +5gnidd,0.75,1,2,5,SlowrateDoS,5965,False,False +5gnidd,0.75,1,2,6,TCPConnectScan,775,False,False +5gnidd,0.75,1,2,7,UDPFlood,23126,False,False +5gnidd,0.75,1,2,8,UDPScan,451,False,False +5gnidd,0.75,1,3,0,Benign,80420,False,False +5gnidd,0.75,1,3,1,HTTPFlood,621,False,False +5gnidd,0.75,1,3,2,ICMPFlood,0,False,True +5gnidd,0.75,1,3,3,SYNFlood,229,True,False +5gnidd,0.75,1,3,4,SYNScan,2650,False,False +5gnidd,0.75,1,3,5,SlowrateDoS,514,True,False +5gnidd,0.75,1,3,6,TCPConnectScan,1149,False,False +5gnidd,0.75,1,3,7,UDPFlood,114825,False,False +5gnidd,0.75,1,3,8,UDPScan,2421,False,False +5gnidd,0.75,1,4,0,Benign,3896,False,False +5gnidd,0.75,1,4,1,HTTPFlood,2280,False,False +5gnidd,0.75,1,4,2,ICMPFlood,15,True,False +5gnidd,0.75,1,4,3,SYNFlood,57,True,False +5gnidd,0.75,1,4,4,SYNScan,565,False,False +5gnidd,0.75,1,4,5,SlowrateDoS,6824,False,False +5gnidd,0.75,1,4,6,TCPConnectScan,809,False,False +5gnidd,0.75,1,4,7,UDPFlood,1533,False,False +5gnidd,0.75,1,4,8,UDPScan,1882,False,False +5gnidd,0.75,1,5,0,Benign,20027,False,False +5gnidd,0.75,1,5,1,HTTPFlood,25248,False,False +5gnidd,0.75,1,5,2,ICMPFlood,37,True,False +5gnidd,0.75,1,5,3,SYNFlood,830,True,False +5gnidd,0.75,1,5,4,SYNScan,3883,False,False +5gnidd,0.75,1,5,5,SlowrateDoS,12114,False,False +5gnidd,0.75,1,5,6,TCPConnectScan,2861,False,False +5gnidd,0.75,1,5,7,UDPFlood,48726,False,False +5gnidd,0.75,1,5,8,UDPScan,1858,False,False +5gnidd,0.75,1,6,0,Benign,13275,False,False +5gnidd,0.75,1,6,1,HTTPFlood,7039,False,False +5gnidd,0.75,1,6,2,ICMPFlood,15,True,False +5gnidd,0.75,1,6,3,SYNFlood,199,False,False +5gnidd,0.75,1,6,4,SYNScan,0,False,True +5gnidd,0.75,1,6,5,SlowrateDoS,7156,False,False +5gnidd,0.75,1,6,6,TCPConnectScan,1882,False,False +5gnidd,0.75,1,6,7,UDPFlood,22538,False,False +5gnidd,0.75,1,6,8,UDPScan,40,True,False +5gnidd,0.75,1,7,0,Benign,31700,False,False +5gnidd,0.75,1,7,1,HTTPFlood,13892,False,False +5gnidd,0.75,1,7,2,ICMPFlood,240,False,False +5gnidd,0.75,1,7,3,SYNFlood,72,True,False +5gnidd,0.75,1,7,4,SYNScan,1949,False,False +5gnidd,0.75,1,7,5,SlowrateDoS,3092,False,False +5gnidd,0.75,1,7,6,TCPConnectScan,1065,False,False +5gnidd,0.75,1,7,7,UDPFlood,64629,False,False +5gnidd,0.75,1,7,8,UDPScan,5,True,False +5gnidd,0.75,1,8,0,Benign,83388,False,False +5gnidd,0.75,1,8,1,HTTPFlood,12304,False,False +5gnidd,0.75,1,8,2,ICMPFlood,134,True,False +5gnidd,0.75,1,8,3,SYNFlood,247,False,False +5gnidd,0.75,1,8,4,SYNScan,400,False,False +5gnidd,0.75,1,8,5,SlowrateDoS,2594,False,False +5gnidd,0.75,1,8,6,TCPConnectScan,326,False,False +5gnidd,0.75,1,8,7,UDPFlood,27585,False,False +5gnidd,0.75,1,8,8,UDPScan,152,True,False +5gnidd,0.75,1,9,0,Benign,72005,False,False +5gnidd,0.75,1,9,1,HTTPFlood,7006,False,False +5gnidd,0.75,1,9,2,ICMPFlood,285,True,False +5gnidd,0.75,1,9,3,SYNFlood,2358,False,False +5gnidd,0.75,1,9,4,SYNScan,724,False,False +5gnidd,0.75,1,9,5,SlowrateDoS,4717,False,False +5gnidd,0.75,1,9,6,TCPConnectScan,2150,False,False +5gnidd,0.75,1,9,7,UDPFlood,1031,False,False +5gnidd,0.75,1,9,8,UDPScan,188,True,False diff --git a/artifacts/analysis/5gnidd__alpha0.75__seed2__characterization.csv b/artifacts/analysis/5gnidd__alpha0.75__seed2__characterization.csv new file mode 100644 index 0000000..7ec88b6 --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.75__seed2__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,Benign,334415,10,0,536,118638,33441.5,34208.390085620806,1.0229322872963476,221.33955223880596 +1,HTTPFlood,98569,10,0,1484,25410,9856.9,6717.955514142677,0.6815485106009675,17.12264150943396 +2,ICMPFlood,808,10,0,1,270,80.8,93.3892927481518,1.1558080785662352,270.0 +3,SYNFlood,6805,10,0,6,2670,680.5,810.6782654049632,1.1912979653269113,445.0 +4,SYNScan,14030,10,0,91,4022,1403.0,1290.3725818537837,0.9197238644716919,44.1978021978022 +5,SlowrateDoS,51187,10,0,99,18020,5118.7,4987.431263686748,0.9743550635291672,182.02020202020202 +6,TCPConnectScan,14037,10,0,84,3335,1403.7,1073.5125569829167,0.7647734964614353,39.70238095238095 +7,UDPFlood,320138,10,0,2637,63416,32013.8,21351.952298560427,0.6669608824494571,24.048540007584375 +8,UDPScan,11134,10,0,37,3282,1113.4,1058.092925975786,0.9503259618967003,88.70270270270271 diff --git a/artifacts/analysis/5gnidd__alpha0.75__seed2__rare_unseen.csv b/artifacts/analysis/5gnidd__alpha0.75__seed2__rare_unseen.csv new file mode 100644 index 0000000..70dac2f --- /dev/null +++ b/artifacts/analysis/5gnidd__alpha0.75__seed2__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +5gnidd,0.75,2,0,0,Benign,7083,False,False +5gnidd,0.75,2,0,1,HTTPFlood,17146,False,False +5gnidd,0.75,2,0,2,ICMPFlood,88,True,False +5gnidd,0.75,2,0,3,SYNFlood,380,False,False +5gnidd,0.75,2,0,4,SYNScan,4022,False,False +5gnidd,0.75,2,0,5,SlowrateDoS,335,True,False +5gnidd,0.75,2,0,6,TCPConnectScan,1387,False,False +5gnidd,0.75,2,0,7,UDPFlood,47750,False,False +5gnidd,0.75,2,0,8,UDPScan,600,False,False +5gnidd,0.75,2,1,0,Benign,31172,False,False +5gnidd,0.75,2,1,1,HTTPFlood,12345,False,False +5gnidd,0.75,2,1,2,ICMPFlood,8,True,False +5gnidd,0.75,2,1,3,SYNFlood,2670,False,False +5gnidd,0.75,2,1,4,SYNScan,626,False,False +5gnidd,0.75,2,1,5,SlowrateDoS,18020,False,False +5gnidd,0.75,2,1,6,TCPConnectScan,3054,False,False +5gnidd,0.75,2,1,7,UDPFlood,39353,False,False +5gnidd,0.75,2,1,8,UDPScan,562,True,False +5gnidd,0.75,2,2,0,Benign,30915,False,False +5gnidd,0.75,2,2,1,HTTPFlood,1484,False,False +5gnidd,0.75,2,2,2,ICMPFlood,35,True,False +5gnidd,0.75,2,2,3,SYNFlood,462,False,False +5gnidd,0.75,2,2,4,SYNScan,3070,False,False +5gnidd,0.75,2,2,5,SlowrateDoS,5630,False,False +5gnidd,0.75,2,2,6,TCPConnectScan,105,True,False +5gnidd,0.75,2,2,7,UDPFlood,15448,False,False +5gnidd,0.75,2,2,8,UDPScan,1283,False,False +5gnidd,0.75,2,3,0,Benign,41272,False,False +5gnidd,0.75,2,3,1,HTTPFlood,8659,False,False +5gnidd,0.75,2,3,2,ICMPFlood,25,True,False +5gnidd,0.75,2,3,3,SYNFlood,6,True,False +5gnidd,0.75,2,3,4,SYNScan,737,False,False +5gnidd,0.75,2,3,5,SlowrateDoS,8841,False,False +5gnidd,0.75,2,3,6,TCPConnectScan,84,False,False +5gnidd,0.75,2,3,7,UDPFlood,5446,False,False +5gnidd,0.75,2,3,8,UDPScan,37,False,False +5gnidd,0.75,2,4,0,Benign,7313,False,False +5gnidd,0.75,2,4,1,HTTPFlood,25410,False,False +5gnidd,0.75,2,4,2,ICMPFlood,41,True,False +5gnidd,0.75,2,4,3,SYNFlood,110,True,False +5gnidd,0.75,2,4,4,SYNScan,357,False,False +5gnidd,0.75,2,4,5,SlowrateDoS,5017,False,False +5gnidd,0.75,2,4,6,TCPConnectScan,783,False,False +5gnidd,0.75,2,4,7,UDPFlood,8809,False,False +5gnidd,0.75,2,4,8,UDPScan,1697,False,False +5gnidd,0.75,2,5,0,Benign,9497,False,False +5gnidd,0.75,2,5,1,HTTPFlood,7911,False,False +5gnidd,0.75,2,5,2,ICMPFlood,19,True,False +5gnidd,0.75,2,5,3,SYNFlood,1678,False,False +5gnidd,0.75,2,5,4,SYNScan,710,False,False +5gnidd,0.75,2,5,5,SlowrateDoS,4985,False,False +5gnidd,0.75,2,5,6,TCPConnectScan,1378,False,False +5gnidd,0.75,2,5,7,UDPFlood,45069,False,False +5gnidd,0.75,2,5,8,UDPScan,238,True,False +5gnidd,0.75,2,6,0,Benign,118638,False,False +5gnidd,0.75,2,6,1,HTTPFlood,5810,False,False +5gnidd,0.75,2,6,2,ICMPFlood,251,True,False +5gnidd,0.75,2,6,3,SYNFlood,185,True,False +5gnidd,0.75,2,6,4,SYNScan,1939,False,False +5gnidd,0.75,2,6,5,SlowrateDoS,1759,False,False +5gnidd,0.75,2,6,6,TCPConnectScan,1832,False,False +5gnidd,0.75,2,6,7,UDPFlood,58797,False,False +5gnidd,0.75,2,6,8,UDPScan,2697,False,False +5gnidd,0.75,2,7,0,Benign,67601,False,False +5gnidd,0.75,2,7,1,HTTPFlood,10568,False,False +5gnidd,0.75,2,7,2,ICMPFlood,270,False,False +5gnidd,0.75,2,7,3,SYNFlood,59,True,False +5gnidd,0.75,2,7,4,SYNScan,160,True,False +5gnidd,0.75,2,7,5,SlowrateDoS,4201,False,False +5gnidd,0.75,2,7,6,TCPConnectScan,3335,False,False +5gnidd,0.75,2,7,7,UDPFlood,2637,False,False +5gnidd,0.75,2,7,8,UDPScan,283,False,False +5gnidd,0.75,2,8,0,Benign,20388,False,False +5gnidd,0.75,2,8,1,HTTPFlood,3396,False,False +5gnidd,0.75,2,8,2,ICMPFlood,1,True,False +5gnidd,0.75,2,8,3,SYNFlood,450,False,False +5gnidd,0.75,2,8,4,SYNScan,91,True,False +5gnidd,0.75,2,8,5,SlowrateDoS,99,False,False +5gnidd,0.75,2,8,6,TCPConnectScan,1675,False,False +5gnidd,0.75,2,8,7,UDPFlood,33413,False,False +5gnidd,0.75,2,8,8,UDPScan,3282,False,False +5gnidd,0.75,2,9,0,Benign,536,False,False +5gnidd,0.75,2,9,1,HTTPFlood,5840,False,False +5gnidd,0.75,2,9,2,ICMPFlood,70,True,False +5gnidd,0.75,2,9,3,SYNFlood,805,False,False +5gnidd,0.75,2,9,4,SYNScan,2318,False,False +5gnidd,0.75,2,9,5,SlowrateDoS,2300,False,False +5gnidd,0.75,2,9,6,TCPConnectScan,404,True,False +5gnidd,0.75,2,9,7,UDPFlood,63416,False,False +5gnidd,0.75,2,9,8,UDPScan,455,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.25__seed0__characterization.csv b/artifacts/analysis/xiiotid__alpha0.25__seed0__characterization.csv new file mode 100644 index 0000000..c36726a --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.25__seed0__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,10,0,21,1065,200.4,296.149691878955,1.4777928736474801,50.714285714285715 +1,Exfiltration,15494,8,2,4,7950,1936.75,2720.187390144289,1.4045113670552674,1987.5 +2,Exploitation,793,6,4,2,595,132.16666666666666,208.87669142875237,1.5804037182503332,297.5 +3,Lateral_movement,22117,9,1,2,13857,2457.4444444444443,4448.236282470993,1.8101065489098402,6928.5 +4,Normal,292777,10,0,1,122800,29277.7,39398.78122239316,1.3456924971016562,122800.0 +5,RDOS,98883,10,0,166,31089,9888.3,10684.255463531374,1.080494671837563,187.28313253012047 +6,Reconnaissance,89161,8,2,46,65307,11145.125,20676.50504460014,1.8552062040219504,1419.7173913043478 +7,Tampering,3586,8,2,9,2272,448.25,703.1018329516713,1.5685484282245874,252.44444444444446 +8,Weaponization,45094,8,2,43,23381,5636.75,8611.380141272362,1.527720786139595,543.7441860465116 diff --git a/artifacts/analysis/xiiotid__alpha0.25__seed0__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.25__seed0__rare_unseen.csv new file mode 100644 index 0000000..a4f1d09 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.25__seed0__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.25,0,0,0,C&C,193,False,False +xiiotid,0.25,0,0,1,Exfiltration,364,False,False +xiiotid,0.25,0,0,2,Exploitation,73,False,False +xiiotid,0.25,0,0,3,Lateral_movement,6253,False,False +xiiotid,0.25,0,0,4,Normal,1,False,False +xiiotid,0.25,0,0,5,RDOS,26440,False,False +xiiotid,0.25,0,0,6,Reconnaissance,46,True,False +xiiotid,0.25,0,0,7,Tampering,159,False,False +xiiotid,0.25,0,0,8,Weaponization,43,True,False +xiiotid,0.25,0,1,0,C&C,247,False,False +xiiotid,0.25,0,1,1,Exfiltration,0,False,True +xiiotid,0.25,0,1,2,Exploitation,0,False,True +xiiotid,0.25,0,1,3,Lateral_movement,21,True,False +xiiotid,0.25,0,1,4,Normal,102,False,False +xiiotid,0.25,0,1,5,RDOS,166,True,False +xiiotid,0.25,0,1,6,Reconnaissance,65307,False,False +xiiotid,0.25,0,1,7,Tampering,2272,False,False +xiiotid,0.25,0,1,8,Weaponization,0,False,True +xiiotid,0.25,0,2,0,C&C,131,True,False +xiiotid,0.25,0,2,1,Exfiltration,187,False,False +xiiotid,0.25,0,2,2,Exploitation,0,False,True +xiiotid,0.25,0,2,3,Lateral_movement,578,False,False +xiiotid,0.25,0,2,4,Normal,1048,False,False +xiiotid,0.25,0,2,5,RDOS,3516,False,False +xiiotid,0.25,0,2,6,Reconnaissance,4027,False,False +xiiotid,0.25,0,2,7,Tampering,9,True,False +xiiotid,0.25,0,2,8,Weaponization,168,False,False +xiiotid,0.25,0,3,0,C&C,29,False,False +xiiotid,0.25,0,3,1,Exfiltration,45,False,False +xiiotid,0.25,0,3,2,Exploitation,3,True,False +xiiotid,0.25,0,3,3,Lateral_movement,2,True,False +xiiotid,0.25,0,3,4,Normal,38918,False,False +xiiotid,0.25,0,3,5,RDOS,31089,False,False +xiiotid,0.25,0,3,6,Reconnaissance,0,False,True +xiiotid,0.25,0,3,7,Tampering,0,False,True +xiiotid,0.25,0,3,8,Weaponization,3342,False,False +xiiotid,0.25,0,4,0,C&C,21,True,False +xiiotid,0.25,0,4,1,Exfiltration,0,False,True +xiiotid,0.25,0,4,2,Exploitation,2,True,False +xiiotid,0.25,0,4,3,Lateral_movement,13857,False,False +xiiotid,0.25,0,4,4,Normal,122800,False,False +xiiotid,0.25,0,4,5,RDOS,727,False,False +xiiotid,0.25,0,4,6,Reconnaissance,3644,False,False +xiiotid,0.25,0,4,7,Tampering,307,False,False +xiiotid,0.25,0,4,8,Weaponization,0,False,True +xiiotid,0.25,0,5,0,C&C,51,True,False +xiiotid,0.25,0,5,1,Exfiltration,4,True,False +xiiotid,0.25,0,5,2,Exploitation,0,False,True +xiiotid,0.25,0,5,3,Lateral_movement,808,False,False +xiiotid,0.25,0,5,4,Normal,633,False,False +xiiotid,0.25,0,5,5,RDOS,14233,False,False +xiiotid,0.25,0,5,6,Reconnaissance,4363,False,False +xiiotid,0.25,0,5,7,Tampering,453,False,False +xiiotid,0.25,0,5,8,Weaponization,347,False,False +xiiotid,0.25,0,6,0,C&C,87,True,False +xiiotid,0.25,0,6,1,Exfiltration,7950,False,False +xiiotid,0.25,0,6,2,Exploitation,57,True,False +xiiotid,0.25,0,6,3,Lateral_movement,0,False,True +xiiotid,0.25,0,6,4,Normal,6773,False,False +xiiotid,0.25,0,6,5,RDOS,1637,False,False +xiiotid,0.25,0,6,6,Reconnaissance,0,False,True +xiiotid,0.25,0,6,7,Tampering,241,False,False +xiiotid,0.25,0,6,8,Weaponization,23381,False,False +xiiotid,0.25,0,7,0,C&C,116,True,False +xiiotid,0.25,0,7,1,Exfiltration,9,True,False +xiiotid,0.25,0,7,2,Exploitation,595,False,False +xiiotid,0.25,0,7,3,Lateral_movement,573,False,False +xiiotid,0.25,0,7,4,Normal,71934,False,False +xiiotid,0.25,0,7,5,RDOS,4941,False,False +xiiotid,0.25,0,7,6,Reconnaissance,1867,False,False +xiiotid,0.25,0,7,7,Tampering,131,False,False +xiiotid,0.25,0,7,8,Weaponization,16998,False,False +xiiotid,0.25,0,8,0,C&C,1065,False,False +xiiotid,0.25,0,8,1,Exfiltration,2523,False,False +xiiotid,0.25,0,8,2,Exploitation,63,False,False +xiiotid,0.25,0,8,3,Lateral_movement,7,True,False +xiiotid,0.25,0,8,4,Normal,2928,False,False +xiiotid,0.25,0,8,5,RDOS,1704,False,False +xiiotid,0.25,0,8,6,Reconnaissance,67,False,False +xiiotid,0.25,0,8,7,Tampering,14,True,False +xiiotid,0.25,0,8,8,Weaponization,704,False,False +xiiotid,0.25,0,9,0,C&C,64,True,False +xiiotid,0.25,0,9,1,Exfiltration,4412,False,False +xiiotid,0.25,0,9,2,Exploitation,0,False,True +xiiotid,0.25,0,9,3,Lateral_movement,18,True,False +xiiotid,0.25,0,9,4,Normal,47640,False,False +xiiotid,0.25,0,9,5,RDOS,14430,False,False +xiiotid,0.25,0,9,6,Reconnaissance,9840,False,False +xiiotid,0.25,0,9,7,Tampering,0,False,True +xiiotid,0.25,0,9,8,Weaponization,111,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.25__seed1__characterization.csv b/artifacts/analysis/xiiotid__alpha0.25__seed1__characterization.csv new file mode 100644 index 0000000..a845365 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.25__seed1__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,9,1,1,986,222.66666666666666,347.19286987046394,1.559249415585916,986.0 +1,Exfiltration,15494,10,0,1,6204,1549.4,1918.3846955186025,1.238146828139023,6204.0 +2,Exploitation,793,8,2,1,321,99.125,128.38558865776173,1.2951887884767892,321.0 +3,Lateral_movement,22117,10,0,1,10094,2211.7,3172.3787305427454,1.4343621334461027,10094.0 +4,Normal,292777,8,2,198,129678,36597.125,42579.4751242823,1.16346502967876,654.939393939394 +5,RDOS,98883,9,1,168,53482,10987.0,16131.251346377318,1.4682125554179775,318.3452380952381 +6,Reconnaissance,89161,8,2,138,65101,11145.125,20636.12875176386,1.8515834278901186,471.7463768115942 +7,Tampering,3586,9,1,2,1804,398.44444444444446,536.9212876537887,1.3475436667273,902.0 +8,Weaponization,45094,9,1,1,12024,5010.444444444444,4173.732332127554,0.8330064085942251,12024.0 diff --git a/artifacts/analysis/xiiotid__alpha0.25__seed1__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.25__seed1__rare_unseen.csv new file mode 100644 index 0000000..ced45ea --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.25__seed1__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.25,1,0,0,C&C,1,True,False +xiiotid,0.25,1,0,1,Exfiltration,48,False,False +xiiotid,0.25,1,0,2,Exploitation,78,False,False +xiiotid,0.25,1,0,3,Lateral_movement,1,True,False +xiiotid,0.25,1,0,4,Normal,45538,False,False +xiiotid,0.25,1,0,5,RDOS,168,False,False +xiiotid,0.25,1,0,6,Reconnaissance,4841,False,False +xiiotid,0.25,1,0,7,Tampering,2,False,False +xiiotid,0.25,1,0,8,Weaponization,8960,False,False +xiiotid,0.25,1,1,0,C&C,21,True,False +xiiotid,0.25,1,1,1,Exfiltration,129,False,False +xiiotid,0.25,1,1,2,Exploitation,16,True,False +xiiotid,0.25,1,1,3,Lateral_movement,10094,False,False +xiiotid,0.25,1,1,4,Normal,75982,False,False +xiiotid,0.25,1,1,5,RDOS,5627,False,False +xiiotid,0.25,1,1,6,Reconnaissance,65101,False,False +xiiotid,0.25,1,1,7,Tampering,577,False,False +xiiotid,0.25,1,1,8,Weaponization,245,False,False +xiiotid,0.25,1,2,0,C&C,0,False,True +xiiotid,0.25,1,2,1,Exfiltration,3823,False,False +xiiotid,0.25,1,2,2,Exploitation,0,False,True +xiiotid,0.25,1,2,3,Lateral_movement,2,True,False +xiiotid,0.25,1,2,4,Normal,0,False,False +xiiotid,0.25,1,2,5,RDOS,0,False,True +xiiotid,0.25,1,2,6,Reconnaissance,0,False,True +xiiotid,0.25,1,2,7,Tampering,151,True,False +xiiotid,0.25,1,2,8,Weaponization,5302,False,False +xiiotid,0.25,1,3,0,C&C,21,True,False +xiiotid,0.25,1,3,1,Exfiltration,449,False,False +xiiotid,0.25,1,3,2,Exploitation,315,False,False +xiiotid,0.25,1,3,3,Lateral_movement,82,True,False +xiiotid,0.25,1,3,4,Normal,2706,False,False +xiiotid,0.25,1,3,5,RDOS,2835,False,False +xiiotid,0.25,1,3,6,Reconnaissance,6322,False,False +xiiotid,0.25,1,3,7,Tampering,170,False,False +xiiotid,0.25,1,3,8,Weaponization,0,False,True +xiiotid,0.25,1,4,0,C&C,46,True,False +xiiotid,0.25,1,4,1,Exfiltration,262,False,False +xiiotid,0.25,1,4,2,Exploitation,22,True,False +xiiotid,0.25,1,4,3,Lateral_movement,793,False,False +xiiotid,0.25,1,4,4,Normal,16624,False,False +xiiotid,0.25,1,4,5,RDOS,12854,False,False +xiiotid,0.25,1,4,6,Reconnaissance,202,False,False +xiiotid,0.25,1,4,7,Tampering,0,False,True +xiiotid,0.25,1,4,8,Weaponization,9118,False,False +xiiotid,0.25,1,5,0,C&C,181,False,False +xiiotid,0.25,1,5,1,Exfiltration,1,True,False +xiiotid,0.25,1,5,2,Exploitation,3,True,False +xiiotid,0.25,1,5,3,Lateral_movement,6163,False,False +xiiotid,0.25,1,5,4,Normal,129678,False,False +xiiotid,0.25,1,5,5,RDOS,53482,False,False +xiiotid,0.25,1,5,6,Reconnaissance,439,False,False +xiiotid,0.25,1,5,7,Tampering,18,False,False +xiiotid,0.25,1,5,8,Weaponization,3361,False,False +xiiotid,0.25,1,6,0,C&C,723,True,False +xiiotid,0.25,1,6,1,Exfiltration,1643,False,False +xiiotid,0.25,1,6,2,Exploitation,0,False,True +xiiotid,0.25,1,6,3,Lateral_movement,830,False,False +xiiotid,0.25,1,6,4,Normal,0,False,False +xiiotid,0.25,1,6,5,RDOS,998,False,False +xiiotid,0.25,1,6,6,Reconnaissance,138,True,False +xiiotid,0.25,1,6,7,Tampering,1804,False,False +xiiotid,0.25,1,6,8,Weaponization,5936,False,False +xiiotid,0.25,1,7,0,C&C,20,True,False +xiiotid,0.25,1,7,1,Exfiltration,6204,False,False +xiiotid,0.25,1,7,2,Exploitation,321,False,False +xiiotid,0.25,1,7,3,Lateral_movement,83,False,False +xiiotid,0.25,1,7,4,Normal,198,False,False +xiiotid,0.25,1,7,5,RDOS,232,False,False +xiiotid,0.25,1,7,6,Reconnaissance,2503,False,False +xiiotid,0.25,1,7,7,Tampering,314,False,False +xiiotid,0.25,1,7,8,Weaponization,1,True,False +xiiotid,0.25,1,8,0,C&C,986,False,False +xiiotid,0.25,1,8,1,Exfiltration,1919,False,False +xiiotid,0.25,1,8,2,Exploitation,1,True,False +xiiotid,0.25,1,8,3,Lateral_movement,2228,False,False +xiiotid,0.25,1,8,4,Normal,14813,False,False +xiiotid,0.25,1,8,5,RDOS,4108,False,False +xiiotid,0.25,1,8,6,Reconnaissance,0,False,True +xiiotid,0.25,1,8,7,Tampering,16,True,False +xiiotid,0.25,1,8,8,Weaponization,147,False,False +xiiotid,0.25,1,9,0,C&C,5,True,False +xiiotid,0.25,1,9,1,Exfiltration,1016,False,False +xiiotid,0.25,1,9,2,Exploitation,37,True,False +xiiotid,0.25,1,9,3,Lateral_movement,1841,False,False +xiiotid,0.25,1,9,4,Normal,7238,False,False +xiiotid,0.25,1,9,5,RDOS,18579,False,False +xiiotid,0.25,1,9,6,Reconnaissance,9615,False,False +xiiotid,0.25,1,9,7,Tampering,534,False,False +xiiotid,0.25,1,9,8,Weaponization,12024,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.25__seed2__characterization.csv b/artifacts/analysis/xiiotid__alpha0.25__seed2__characterization.csv new file mode 100644 index 0000000..3ac2bd8 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.25__seed2__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,6,4,2,1265,334.0,439.2504980076858,1.3151212515200172,632.5 +1,Exfiltration,15494,8,2,4,8269,1936.75,2589.4560118874388,1.3370109781269852,2067.25 +2,Exploitation,793,6,4,1,485,132.16666666666666,175.5395650242101,1.3281682095148306,485.0 +3,Lateral_movement,22117,7,3,1,12496,3159.5714285714284,4660.178716902324,1.4749401373747013,12496.0 +4,Normal,292777,9,1,452,118937,32530.777777777777,35898.97737255285,1.1035388584245882,263.13495575221236 +5,RDOS,98883,10,0,2,48839,9888.3,14049.173043634988,1.4207875007468411,24419.5 +6,Reconnaissance,89161,10,0,1,46541,8916.1,13537.102034408988,1.5182761559884914,46541.0 +7,Tampering,3586,10,0,1,2584,358.6,765.2960472915041,2.1341217158156836,2584.0 +8,Weaponization,45094,9,1,4,22853,5010.444444444444,7371.102105310547,1.4711473576927068,5713.25 diff --git a/artifacts/analysis/xiiotid__alpha0.25__seed2__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.25__seed2__rare_unseen.csv new file mode 100644 index 0000000..bdf9bf4 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.25__seed2__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.25,2,0,0,C&C,0,False,True +xiiotid,0.25,2,0,1,Exfiltration,430,False,False +xiiotid,0.25,2,0,2,Exploitation,0,False,True +xiiotid,0.25,2,0,3,Lateral_movement,0,False,True +xiiotid,0.25,2,0,4,Normal,118937,False,False +xiiotid,0.25,2,0,5,RDOS,2912,False,False +xiiotid,0.25,2,0,6,Reconnaissance,118,False,False +xiiotid,0.25,2,0,7,Tampering,16,True,False +xiiotid,0.25,2,0,8,Weaponization,113,True,False +xiiotid,0.25,2,1,0,C&C,2,True,False +xiiotid,0.25,2,1,1,Exfiltration,2477,False,False +xiiotid,0.25,2,1,2,Exploitation,225,False,False +xiiotid,0.25,2,1,3,Lateral_movement,1,True,False +xiiotid,0.25,2,1,4,Normal,10987,False,False +xiiotid,0.25,2,1,5,RDOS,8514,False,False +xiiotid,0.25,2,1,6,Reconnaissance,6193,False,False +xiiotid,0.25,2,1,7,Tampering,5,False,False +xiiotid,0.25,2,1,8,Weaponization,11858,False,False +xiiotid,0.25,2,2,0,C&C,1265,False,False +xiiotid,0.25,2,2,1,Exfiltration,388,False,False +xiiotid,0.25,2,2,2,Exploitation,11,True,False +xiiotid,0.25,2,2,3,Lateral_movement,839,False,False +xiiotid,0.25,2,2,4,Normal,0,False,False +xiiotid,0.25,2,2,5,RDOS,48839,False,False +xiiotid,0.25,2,2,6,Reconnaissance,121,False,False +xiiotid,0.25,2,2,7,Tampering,1,True,False +xiiotid,0.25,2,2,8,Weaponization,6425,False,False +xiiotid,0.25,2,3,0,C&C,0,False,True +xiiotid,0.25,2,3,1,Exfiltration,8269,False,False +xiiotid,0.25,2,3,2,Exploitation,0,False,True +xiiotid,0.25,2,3,3,Lateral_movement,631,False,False +xiiotid,0.25,2,3,4,Normal,452,False,False +xiiotid,0.25,2,3,5,RDOS,17375,False,False +xiiotid,0.25,2,3,6,Reconnaissance,14322,False,False +xiiotid,0.25,2,3,7,Tampering,16,True,False +xiiotid,0.25,2,3,8,Weaponization,329,True,False +xiiotid,0.25,2,4,0,C&C,428,False,False +xiiotid,0.25,2,4,1,Exfiltration,0,False,True +xiiotid,0.25,2,4,2,Exploitation,11,True,False +xiiotid,0.25,2,4,3,Lateral_movement,8040,False,False +xiiotid,0.25,2,4,4,Normal,5015,False,False +xiiotid,0.25,2,4,5,RDOS,11529,False,False +xiiotid,0.25,2,4,6,Reconnaissance,34,False,False +xiiotid,0.25,2,4,7,Tampering,18,True,False +xiiotid,0.25,2,4,8,Weaponization,0,False,True +xiiotid,0.25,2,5,0,C&C,0,False,True +xiiotid,0.25,2,5,1,Exfiltration,4,True,False +xiiotid,0.25,2,5,2,Exploitation,0,False,True +xiiotid,0.25,2,5,3,Lateral_movement,29,True,False +xiiotid,0.25,2,5,4,Normal,61418,False,False +xiiotid,0.25,2,5,5,RDOS,1181,False,False +xiiotid,0.25,2,5,6,Reconnaissance,7835,False,False +xiiotid,0.25,2,5,7,Tampering,2584,False,False +xiiotid,0.25,2,5,8,Weaponization,3257,False,False +xiiotid,0.25,2,6,0,C&C,0,False,True +xiiotid,0.25,2,6,1,Exfiltration,1201,False,False +xiiotid,0.25,2,6,2,Exploitation,1,True,False +xiiotid,0.25,2,6,3,Lateral_movement,0,False,True +xiiotid,0.25,2,6,4,Normal,35251,False,False +xiiotid,0.25,2,6,5,RDOS,1382,False,False +xiiotid,0.25,2,6,6,Reconnaissance,12411,False,False +xiiotid,0.25,2,6,7,Tampering,2,True,False +xiiotid,0.25,2,6,8,Weaponization,240,False,False +xiiotid,0.25,2,7,0,C&C,17,True,False +xiiotid,0.25,2,7,1,Exfiltration,2713,False,False +xiiotid,0.25,2,7,2,Exploitation,60,False,False +xiiotid,0.25,2,7,3,Lateral_movement,81,False,False +xiiotid,0.25,2,7,4,Normal,28172,False,False +xiiotid,0.25,2,7,5,RDOS,2,True,False +xiiotid,0.25,2,7,6,Reconnaissance,46541,False,False +xiiotid,0.25,2,7,7,Tampering,355,False,False +xiiotid,0.25,2,7,8,Weaponization,22853,False,False +xiiotid,0.25,2,8,0,C&C,128,False,False +xiiotid,0.25,2,8,1,Exfiltration,12,True,False +xiiotid,0.25,2,8,2,Exploitation,0,False,True +xiiotid,0.25,2,8,3,Lateral_movement,12496,False,False +xiiotid,0.25,2,8,4,Normal,576,False,False +xiiotid,0.25,2,8,5,RDOS,281,False,False +xiiotid,0.25,2,8,6,Reconnaissance,1585,False,False +xiiotid,0.25,2,8,7,Tampering,582,False,False +xiiotid,0.25,2,8,8,Weaponization,4,True,False +xiiotid,0.25,2,9,0,C&C,164,False,False +xiiotid,0.25,2,9,1,Exfiltration,0,False,True +xiiotid,0.25,2,9,2,Exploitation,485,False,False +xiiotid,0.25,2,9,3,Lateral_movement,0,False,True +xiiotid,0.25,2,9,4,Normal,31969,False,False +xiiotid,0.25,2,9,5,RDOS,6868,False,False +xiiotid,0.25,2,9,6,Reconnaissance,1,True,False +xiiotid,0.25,2,9,7,Tampering,7,True,False +xiiotid,0.25,2,9,8,Weaponization,15,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed0__characterization.csv b/artifacts/analysis/xiiotid__alpha0.5__seed0__characterization.csv new file mode 100644 index 0000000..5b22bf4 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed0__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,10,0,1,966,200.4,301.01302297408995,1.5020609928846804,966.0 +1,Exfiltration,15494,9,1,45,6305,1721.5555555555557,1785.267369400705,1.0370082822128788,140.11111111111111 +2,Exploitation,793,8,2,10,353,99.125,107.69451877881251,1.0864516396349306,35.3 +3,Lateral_movement,22117,10,0,49,8385,2211.7,2692.423482663899,1.217354741901659,171.12244897959184 +4,Normal,292777,10,0,604,68657,29277.7,22396.371889437807,0.7649635008705535,113.6705298013245 +5,RDOS,98883,10,0,168,49283,9888.3,13609.872137900486,1.3763611680370222,293.3511904761905 +6,Reconnaissance,89161,10,0,1137,29234,8916.1,10378.249413557183,1.1639897952644298,25.711521547933156 +7,Tampering,3586,9,1,2,1266,398.44444444444446,396.92067808487633,0.996175711869461,633.0 +8,Weaponization,45094,10,0,18,32409,4509.4,9448.019678218288,2.0951833233286665,1800.5 diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed0__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.5__seed0__rare_unseen.csv new file mode 100644 index 0000000..f0b6add --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed0__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.5,0,0,0,C&C,126,True,False +xiiotid,0.5,0,0,1,Exfiltration,901,False,False +xiiotid,0.5,0,0,2,Exploitation,0,False,True +xiiotid,0.5,0,0,3,Lateral_movement,5915,False,False +xiiotid,0.5,0,0,4,Normal,48503,False,False +xiiotid,0.5,0,0,5,RDOS,9867,False,False +xiiotid,0.5,0,0,6,Reconnaissance,29234,False,False +xiiotid,0.5,0,0,7,Tampering,0,False,True +xiiotid,0.5,0,0,8,Weaponization,18,True,False +xiiotid,0.5,0,1,0,C&C,559,False,False +xiiotid,0.5,0,1,1,Exfiltration,2212,False,False +xiiotid,0.5,0,1,2,Exploitation,69,True,False +xiiotid,0.5,0,1,3,Lateral_movement,180,True,False +xiiotid,0.5,0,1,4,Normal,34165,False,False +xiiotid,0.5,0,1,5,RDOS,6640,False,False +xiiotid,0.5,0,1,6,Reconnaissance,5054,False,False +xiiotid,0.5,0,1,7,Tampering,1266,False,False +xiiotid,0.5,0,1,8,Weaponization,486,False,False +xiiotid,0.5,0,2,0,C&C,92,True,False +xiiotid,0.5,0,2,1,Exfiltration,6305,False,False +xiiotid,0.5,0,2,2,Exploitation,353,False,False +xiiotid,0.5,0,2,3,Lateral_movement,949,False,False +xiiotid,0.5,0,2,4,Normal,604,False,False +xiiotid,0.5,0,2,5,RDOS,10123,False,False +xiiotid,0.5,0,2,6,Reconnaissance,1636,False,False +xiiotid,0.5,0,2,7,Tampering,5,True,False +xiiotid,0.5,0,2,8,Weaponization,32409,False,False +xiiotid,0.5,0,3,0,C&C,966,False,False +xiiotid,0.5,0,3,1,Exfiltration,1509,False,False +xiiotid,0.5,0,3,2,Exploitation,140,True,False +xiiotid,0.5,0,3,3,Lateral_movement,630,True,False +xiiotid,0.5,0,3,4,Normal,13892,False,False +xiiotid,0.5,0,3,5,RDOS,49283,False,False +xiiotid,0.5,0,3,6,Reconnaissance,1137,False,False +xiiotid,0.5,0,3,7,Tampering,832,False,False +xiiotid,0.5,0,3,8,Weaponization,1220,False,False +xiiotid,0.5,0,4,0,C&C,1,True,False +xiiotid,0.5,0,4,1,Exfiltration,1151,False,False +xiiotid,0.5,0,4,2,Exploitation,18,True,False +xiiotid,0.5,0,4,3,Lateral_movement,1747,False,False +xiiotid,0.5,0,4,4,Normal,733,False,False +xiiotid,0.5,0,4,5,RDOS,168,False,False +xiiotid,0.5,0,4,6,Reconnaissance,14033,False,False +xiiotid,0.5,0,4,7,Tampering,336,False,False +xiiotid,0.5,0,4,8,Weaponization,334,False,False +xiiotid,0.5,0,5,0,C&C,4,True,False +xiiotid,0.5,0,5,1,Exfiltration,554,False,False +xiiotid,0.5,0,5,2,Exploitation,53,False,False +xiiotid,0.5,0,5,3,Lateral_movement,315,False,False +xiiotid,0.5,0,5,4,Normal,11521,False,False +xiiotid,0.5,0,5,5,RDOS,1150,False,False +xiiotid,0.5,0,5,6,Reconnaissance,27596,False,False +xiiotid,0.5,0,5,7,Tampering,2,True,False +xiiotid,0.5,0,5,8,Weaponization,691,False,False +xiiotid,0.5,0,6,0,C&C,44,True,False +xiiotid,0.5,0,6,1,Exfiltration,0,False,True +xiiotid,0.5,0,6,2,Exploitation,0,False,True +xiiotid,0.5,0,6,3,Lateral_movement,588,False,False +xiiotid,0.5,0,6,4,Normal,34309,False,False +xiiotid,0.5,0,6,5,RDOS,5683,False,False +xiiotid,0.5,0,6,6,Reconnaissance,3655,False,False +xiiotid,0.5,0,6,7,Tampering,93,True,False +xiiotid,0.5,0,6,8,Weaponization,713,False,False +xiiotid,0.5,0,7,0,C&C,20,True,False +xiiotid,0.5,0,7,1,Exfiltration,2414,False,False +xiiotid,0.5,0,7,2,Exploitation,10,True,False +xiiotid,0.5,0,7,3,Lateral_movement,8385,False,False +xiiotid,0.5,0,7,4,Normal,57789,False,False +xiiotid,0.5,0,7,5,RDOS,9870,False,False +xiiotid,0.5,0,7,6,Reconnaissance,2431,False,False +xiiotid,0.5,0,7,7,Tampering,506,False,False +xiiotid,0.5,0,7,8,Weaponization,2971,False,False +xiiotid,0.5,0,8,0,C&C,4,True,False +xiiotid,0.5,0,8,1,Exfiltration,45,True,False +xiiotid,0.5,0,8,2,Exploitation,138,False,False +xiiotid,0.5,0,8,3,Lateral_movement,3359,False,False +xiiotid,0.5,0,8,4,Normal,68657,False,False +xiiotid,0.5,0,8,5,RDOS,5008,False,False +xiiotid,0.5,0,8,6,Reconnaissance,1662,False,False +xiiotid,0.5,0,8,7,Tampering,385,False,False +xiiotid,0.5,0,8,8,Weaponization,5815,False,False +xiiotid,0.5,0,9,0,C&C,188,False,False +xiiotid,0.5,0,9,1,Exfiltration,403,False,False +xiiotid,0.5,0,9,2,Exploitation,12,True,False +xiiotid,0.5,0,9,3,Lateral_movement,49,True,False +xiiotid,0.5,0,9,4,Normal,22604,False,False +xiiotid,0.5,0,9,5,RDOS,1091,False,False +xiiotid,0.5,0,9,6,Reconnaissance,2723,False,False +xiiotid,0.5,0,9,7,Tampering,161,False,False +xiiotid,0.5,0,9,8,Weaponization,437,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed1__characterization.csv b/artifacts/analysis/xiiotid__alpha0.5__seed1__characterization.csv new file mode 100644 index 0000000..b7d2948 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed1__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,8,2,2,687,250.5,246.01270292405636,0.9820866384193867,343.5 +1,Exfiltration,15494,10,0,8,6273,1549.4,1811.7618607311504,1.169331264186879,784.125 +2,Exploitation,793,9,1,1,491,88.11111111111111,144.76758725967514,1.6430117091261995,491.0 +3,Lateral_movement,22117,10,0,49,8361,2211.7,2542.799797467351,1.1497037561456576,170.6326530612245 +4,Normal,292777,10,0,260,89253,29277.7,30294.75088542568,1.0347380731896865,343.28076923076924 +5,RDOS,98883,10,0,6,47276,9888.3,14002.426425802065,1.4160600331504976,7879.333333333333 +6,Reconnaissance,89161,10,0,1842,23967,8916.1,6441.9730432531305,0.72251018306806,13.011400651465799 +7,Tampering,3586,9,1,8,1228,398.44444444444446,373.67279540353394,0.9378291016820428,153.5 +8,Weaponization,45094,10,0,917,11526,4509.4,4011.2644440375657,0.8895339610674515,12.569247546346784 diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed1__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.5__seed1__rare_unseen.csv new file mode 100644 index 0000000..60bfc3a --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed1__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.5,1,0,0,C&C,2,True,False +xiiotid,0.5,1,0,1,Exfiltration,627,False,False +xiiotid,0.5,1,0,2,Exploitation,56,False,False +xiiotid,0.5,1,0,3,Lateral_movement,2550,False,False +xiiotid,0.5,1,0,4,Normal,89253,False,False +xiiotid,0.5,1,0,5,RDOS,4878,False,False +xiiotid,0.5,1,0,6,Reconnaissance,12996,False,False +xiiotid,0.5,1,0,7,Tampering,8,True,False +xiiotid,0.5,1,0,8,Weaponization,11526,False,False +xiiotid,0.5,1,1,0,C&C,3,True,False +xiiotid,0.5,1,1,1,Exfiltration,199,False,False +xiiotid,0.5,1,1,2,Exploitation,0,False,True +xiiotid,0.5,1,1,3,Lateral_movement,3695,False,False +xiiotid,0.5,1,1,4,Normal,23474,False,False +xiiotid,0.5,1,1,5,RDOS,95,True,False +xiiotid,0.5,1,1,6,Reconnaissance,5840,False,False +xiiotid,0.5,1,1,7,Tampering,1228,False,False +xiiotid,0.5,1,1,8,Weaponization,2611,False,False +xiiotid,0.5,1,2,0,C&C,0,False,True +xiiotid,0.5,1,2,1,Exfiltration,1006,False,False +xiiotid,0.5,1,2,2,Exploitation,50,True,False +xiiotid,0.5,1,2,3,Lateral_movement,818,False,False +xiiotid,0.5,1,2,4,Normal,60061,False,False +xiiotid,0.5,1,2,5,RDOS,2816,False,False +xiiotid,0.5,1,2,6,Reconnaissance,1842,False,False +xiiotid,0.5,1,2,7,Tampering,260,True,False +xiiotid,0.5,1,2,8,Weaponization,1209,False,False +xiiotid,0.5,1,3,0,C&C,687,False,False +xiiotid,0.5,1,3,1,Exfiltration,916,False,False +xiiotid,0.5,1,3,2,Exploitation,86,True,False +xiiotid,0.5,1,3,3,Lateral_movement,1239,False,False +xiiotid,0.5,1,3,4,Normal,69735,False,False +xiiotid,0.5,1,3,5,RDOS,2508,False,False +xiiotid,0.5,1,3,6,Reconnaissance,14852,False,False +xiiotid,0.5,1,3,7,Tampering,91,True,False +xiiotid,0.5,1,3,8,Weaponization,1647,False,False +xiiotid,0.5,1,4,0,C&C,284,False,False +xiiotid,0.5,1,4,1,Exfiltration,6273,False,False +xiiotid,0.5,1,4,2,Exploitation,46,True,False +xiiotid,0.5,1,4,3,Lateral_movement,242,True,False +xiiotid,0.5,1,4,4,Normal,18842,False,False +xiiotid,0.5,1,4,5,RDOS,341,False,False +xiiotid,0.5,1,4,6,Reconnaissance,23967,False,False +xiiotid,0.5,1,4,7,Tampering,483,False,False +xiiotid,0.5,1,4,8,Weaponization,917,False,False +xiiotid,0.5,1,5,0,C&C,599,False,False +xiiotid,0.5,1,5,1,Exfiltration,2759,False,False +xiiotid,0.5,1,5,2,Exploitation,8,True,False +xiiotid,0.5,1,5,3,Lateral_movement,49,False,False +xiiotid,0.5,1,5,4,Normal,1776,False,False +xiiotid,0.5,1,5,5,RDOS,6,True,False +xiiotid,0.5,1,5,6,Reconnaissance,9632,False,False +xiiotid,0.5,1,5,7,Tampering,680,False,False +xiiotid,0.5,1,5,8,Weaponization,1375,False,False +xiiotid,0.5,1,6,0,C&C,0,False,True +xiiotid,0.5,1,6,1,Exfiltration,2518,False,False +xiiotid,0.5,1,6,2,Exploitation,9,True,False +xiiotid,0.5,1,6,3,Lateral_movement,8361,False,False +xiiotid,0.5,1,6,4,Normal,9668,False,False +xiiotid,0.5,1,6,5,RDOS,47276,False,False +xiiotid,0.5,1,6,6,Reconnaissance,4475,False,False +xiiotid,0.5,1,6,7,Tampering,0,False,True +xiiotid,0.5,1,6,8,Weaponization,951,True,False +xiiotid,0.5,1,7,0,C&C,82,False,False +xiiotid,0.5,1,7,1,Exfiltration,8,True,False +xiiotid,0.5,1,7,2,Exploitation,46,True,False +xiiotid,0.5,1,7,3,Lateral_movement,4589,False,False +xiiotid,0.5,1,7,4,Normal,17477,False,False +xiiotid,0.5,1,7,5,RDOS,20791,False,False +xiiotid,0.5,1,7,6,Reconnaissance,2025,False,False +xiiotid,0.5,1,7,7,Tampering,618,False,False +xiiotid,0.5,1,7,8,Weaponization,6976,False,False +xiiotid,0.5,1,8,0,C&C,236,False,False +xiiotid,0.5,1,8,1,Exfiltration,1113,False,False +xiiotid,0.5,1,8,2,Exploitation,1,True,False +xiiotid,0.5,1,8,3,Lateral_movement,396,False,False +xiiotid,0.5,1,8,4,Normal,260,False,False +xiiotid,0.5,1,8,5,RDOS,13932,False,False +xiiotid,0.5,1,8,6,Reconnaissance,6536,False,False +xiiotid,0.5,1,8,7,Tampering,119,True,False +xiiotid,0.5,1,8,8,Weaponization,6913,False,False +xiiotid,0.5,1,9,0,C&C,111,False,False +xiiotid,0.5,1,9,1,Exfiltration,75,True,False +xiiotid,0.5,1,9,2,Exploitation,491,False,False +xiiotid,0.5,1,9,3,Lateral_movement,178,False,False +xiiotid,0.5,1,9,4,Normal,2231,False,False +xiiotid,0.5,1,9,5,RDOS,6240,False,False +xiiotid,0.5,1,9,6,Reconnaissance,6996,False,False +xiiotid,0.5,1,9,7,Tampering,99,True,False +xiiotid,0.5,1,9,8,Weaponization,10969,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed2__characterization.csv b/artifacts/analysis/xiiotid__alpha0.5__seed2__characterization.csv new file mode 100644 index 0000000..d804a87 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed2__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,9,1,13,576,222.66666666666666,158.27683202400647,0.7108240959161968,44.30769230769231 +1,Exfiltration,15494,10,0,20,5300,1549.4,1486.1657511865894,0.9591879122154313,265.0 +2,Exploitation,793,10,0,1,298,79.3,104.09615746990855,1.3126879882712301,298.0 +3,Lateral_movement,22117,10,0,11,6341,2211.7,2001.0836089479121,0.90477171811182,576.4545454545455 +4,Normal,292777,10,0,79,92031,29277.7,32244.598171011534,1.1013364496190456,1164.9493670886077 +5,RDOS,98883,10,0,4,48860,9888.3,14020.836187973955,1.4179218053633038,12215.0 +6,Reconnaissance,89161,10,0,208,22975,8916.1,7703.846681366394,0.8640377161950172,110.45673076923077 +7,Tampering,3586,10,0,26,2437,358.6,698.4746523675716,1.9477820757600992,93.73076923076923 +8,Weaponization,45094,10,0,17,23228,4509.4,6976.612103306303,1.5471264698865266,1366.3529411764705 diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed2__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.5__seed2__rare_unseen.csv new file mode 100644 index 0000000..26237ba --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed2__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.5,2,0,0,C&C,576,False,False +xiiotid,0.5,2,0,1,Exfiltration,2126,False,False +xiiotid,0.5,2,0,2,Exploitation,10,True,False +xiiotid,0.5,2,0,3,Lateral_movement,3084,False,False +xiiotid,0.5,2,0,4,Normal,864,False,False +xiiotid,0.5,2,0,5,RDOS,17951,False,False +xiiotid,0.5,2,0,6,Reconnaissance,1651,False,False +xiiotid,0.5,2,0,7,Tampering,118,True,False +xiiotid,0.5,2,0,8,Weaponization,895,False,False +xiiotid,0.5,2,1,0,C&C,290,False,False +xiiotid,0.5,2,1,1,Exfiltration,20,False,False +xiiotid,0.5,2,1,2,Exploitation,1,True,False +xiiotid,0.5,2,1,3,Lateral_movement,11,True,False +xiiotid,0.5,2,1,4,Normal,418,False,False +xiiotid,0.5,2,1,5,RDOS,7758,False,False +xiiotid,0.5,2,1,6,Reconnaissance,208,False,False +xiiotid,0.5,2,1,7,Tampering,371,False,False +xiiotid,0.5,2,1,8,Weaponization,1014,False,False +xiiotid,0.5,2,2,0,C&C,0,False,True +xiiotid,0.5,2,2,1,Exfiltration,914,False,False +xiiotid,0.5,2,2,2,Exploitation,240,True,False +xiiotid,0.5,2,2,3,Lateral_movement,2152,False,False +xiiotid,0.5,2,2,4,Normal,79,False,False +xiiotid,0.5,2,2,5,RDOS,48860,False,False +xiiotid,0.5,2,2,6,Reconnaissance,12985,False,False +xiiotid,0.5,2,2,7,Tampering,135,True,False +xiiotid,0.5,2,2,8,Weaponization,23228,False,False +xiiotid,0.5,2,3,0,C&C,211,False,False +xiiotid,0.5,2,3,1,Exfiltration,1557,False,False +xiiotid,0.5,2,3,2,Exploitation,2,True,False +xiiotid,0.5,2,3,3,Lateral_movement,72,True,False +xiiotid,0.5,2,3,4,Normal,33716,False,False +xiiotid,0.5,2,3,5,RDOS,94,False,False +xiiotid,0.5,2,3,6,Reconnaissance,17154,False,False +xiiotid,0.5,2,3,7,Tampering,2437,False,False +xiiotid,0.5,2,3,8,Weaponization,10869,False,False +xiiotid,0.5,2,4,0,C&C,37,False,False +xiiotid,0.5,2,4,1,Exfiltration,30,True,False +xiiotid,0.5,2,4,2,Exploitation,35,True,False +xiiotid,0.5,2,4,3,Lateral_movement,566,False,False +xiiotid,0.5,2,4,4,Normal,10789,False,False +xiiotid,0.5,2,4,5,RDOS,1624,False,False +xiiotid,0.5,2,4,6,Reconnaissance,22975,False,False +xiiotid,0.5,2,4,7,Tampering,76,False,False +xiiotid,0.5,2,4,8,Weaponization,218,False,False +xiiotid,0.5,2,5,0,C&C,252,False,False +xiiotid,0.5,2,5,1,Exfiltration,1430,False,False +xiiotid,0.5,2,5,2,Exploitation,145,True,False +xiiotid,0.5,2,5,3,Lateral_movement,6341,False,False +xiiotid,0.5,2,5,4,Normal,1541,False,False +xiiotid,0.5,2,5,5,RDOS,1142,False,False +xiiotid,0.5,2,5,6,Reconnaissance,16770,False,False +xiiotid,0.5,2,5,7,Tampering,132,True,False +xiiotid,0.5,2,5,8,Weaponization,1570,False,False +xiiotid,0.5,2,6,0,C&C,212,False,False +xiiotid,0.5,2,6,1,Exfiltration,5300,False,False +xiiotid,0.5,2,6,2,Exploitation,298,False,False +xiiotid,0.5,2,6,3,Lateral_movement,3130,False,False +xiiotid,0.5,2,6,4,Normal,73646,False,False +xiiotid,0.5,2,6,5,RDOS,4,True,False +xiiotid,0.5,2,6,6,Reconnaissance,5123,False,False +xiiotid,0.5,2,6,7,Tampering,146,False,False +xiiotid,0.5,2,6,8,Weaponization,17,True,False +xiiotid,0.5,2,7,0,C&C,13,True,False +xiiotid,0.5,2,7,1,Exfiltration,2362,False,False +xiiotid,0.5,2,7,2,Exploitation,49,False,False +xiiotid,0.5,2,7,3,Lateral_movement,4431,False,False +xiiotid,0.5,2,7,4,Normal,57785,False,False +xiiotid,0.5,2,7,5,RDOS,10129,False,False +xiiotid,0.5,2,7,6,Reconnaissance,8929,False,False +xiiotid,0.5,2,7,7,Tampering,26,True,False +xiiotid,0.5,2,7,8,Weaponization,2670,False,False +xiiotid,0.5,2,8,0,C&C,296,False,False +xiiotid,0.5,2,8,1,Exfiltration,150,False,False +xiiotid,0.5,2,8,2,Exploitation,7,True,False +xiiotid,0.5,2,8,3,Lateral_movement,131,False,False +xiiotid,0.5,2,8,4,Normal,92031,False,False +xiiotid,0.5,2,8,5,RDOS,4382,False,False +xiiotid,0.5,2,8,6,Reconnaissance,2708,False,False +xiiotid,0.5,2,8,7,Tampering,64,False,False +xiiotid,0.5,2,8,8,Weaponization,38,True,False +xiiotid,0.5,2,9,0,C&C,117,False,False +xiiotid,0.5,2,9,1,Exfiltration,1605,False,False +xiiotid,0.5,2,9,2,Exploitation,6,True,False +xiiotid,0.5,2,9,3,Lateral_movement,2199,False,False +xiiotid,0.5,2,9,4,Normal,21908,False,False +xiiotid,0.5,2,9,5,RDOS,6939,False,False +xiiotid,0.5,2,9,6,Reconnaissance,658,False,False +xiiotid,0.5,2,9,7,Tampering,81,True,False +xiiotid,0.5,2,9,8,Weaponization,4575,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.75__seed0__characterization.csv b/artifacts/analysis/xiiotid__alpha0.75__seed0__characterization.csv new file mode 100644 index 0000000..eb0f628 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.75__seed0__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,8,2,5,760,250.5,222.37468381090505,0.8877232886662877,152.0 +1,Exfiltration,15494,10,0,27,4392,1549.4,1310.2802143053218,0.8456694296536219,162.66666666666666 +2,Exploitation,793,9,1,1,240,88.11111111111111,77.07347784844227,0.8747305178259526,240.0 +3,Lateral_movement,22117,10,0,190,5435,2211.7,1964.874960398244,0.8884003076358656,28.605263157894736 +4,Normal,292777,10,0,1151,92460,29277.7,26706.460263576675,0.9121775366089779,80.33014769765421 +5,RDOS,98883,10,0,347,22821,9888.3,7609.664802736058,0.7695624933240354,65.76657060518733 +6,Reconnaissance,89161,10,0,458,23824,8916.1,8755.430559943925,0.9819798521712323,52.0174672489083 +7,Tampering,3586,10,0,31,1227,358.6,421.7822186863737,1.1761913516072886,39.58064516129032 +8,Weaponization,45094,10,0,285,12915,4509.4,4085.1338337929637,0.9059151625034293,45.31578947368421 diff --git a/artifacts/analysis/xiiotid__alpha0.75__seed0__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.75__seed0__rare_unseen.csv new file mode 100644 index 0000000..c1f47f4 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.75__seed0__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.75,0,0,0,C&C,196,False,False +xiiotid,0.75,0,0,1,Exfiltration,2293,False,False +xiiotid,0.75,0,0,2,Exploitation,96,True,False +xiiotid,0.75,0,0,3,Lateral_movement,1364,False,False +xiiotid,0.75,0,0,4,Normal,24224,False,False +xiiotid,0.75,0,0,5,RDOS,22821,False,False +xiiotid,0.75,0,0,6,Reconnaissance,1040,False,False +xiiotid,0.75,0,0,7,Tampering,85,True,False +xiiotid,0.75,0,0,8,Weaponization,1114,False,False +xiiotid,0.75,0,1,0,C&C,387,False,False +xiiotid,0.75,0,1,1,Exfiltration,189,False,False +xiiotid,0.75,0,1,2,Exploitation,136,True,False +xiiotid,0.75,0,1,3,Lateral_movement,5435,False,False +xiiotid,0.75,0,1,4,Normal,8909,False,False +xiiotid,0.75,0,1,5,RDOS,347,False,False +xiiotid,0.75,0,1,6,Reconnaissance,4622,False,False +xiiotid,0.75,0,1,7,Tampering,95,True,False +xiiotid,0.75,0,1,8,Weaponization,4116,False,False +xiiotid,0.75,0,2,0,C&C,0,False,True +xiiotid,0.75,0,2,1,Exfiltration,1021,False,False +xiiotid,0.75,0,2,2,Exploitation,30,True,False +xiiotid,0.75,0,2,3,Lateral_movement,190,False,False +xiiotid,0.75,0,2,4,Normal,92460,False,False +xiiotid,0.75,0,2,5,RDOS,2604,False,False +xiiotid,0.75,0,2,6,Reconnaissance,2096,False,False +xiiotid,0.75,0,2,7,Tampering,126,True,False +xiiotid,0.75,0,2,8,Weaponization,2046,False,False +xiiotid,0.75,0,3,0,C&C,229,False,False +xiiotid,0.75,0,3,1,Exfiltration,583,False,False +xiiotid,0.75,0,3,2,Exploitation,1,True,False +xiiotid,0.75,0,3,3,Lateral_movement,1149,False,False +xiiotid,0.75,0,3,4,Normal,1151,False,False +xiiotid,0.75,0,3,5,RDOS,438,False,False +xiiotid,0.75,0,3,6,Reconnaissance,16930,False,False +xiiotid,0.75,0,3,7,Tampering,88,True,False +xiiotid,0.75,0,3,8,Weaponization,9065,False,False +xiiotid,0.75,0,4,0,C&C,0,False,True +xiiotid,0.75,0,4,1,Exfiltration,27,True,False +xiiotid,0.75,0,4,2,Exploitation,175,True,False +xiiotid,0.75,0,4,3,Lateral_movement,2323,False,False +xiiotid,0.75,0,4,4,Normal,52309,False,False +xiiotid,0.75,0,4,5,RDOS,13676,False,False +xiiotid,0.75,0,4,6,Reconnaissance,1111,False,False +xiiotid,0.75,0,4,7,Tampering,704,False,False +xiiotid,0.75,0,4,8,Weaponization,285,False,False +xiiotid,0.75,0,5,0,C&C,760,False,False +xiiotid,0.75,0,5,1,Exfiltration,998,False,False +xiiotid,0.75,0,5,2,Exploitation,30,True,False +xiiotid,0.75,0,5,3,Lateral_movement,211,False,False +xiiotid,0.75,0,5,4,Normal,19915,False,False +xiiotid,0.75,0,5,5,RDOS,13907,False,False +xiiotid,0.75,0,5,6,Reconnaissance,6989,False,False +xiiotid,0.75,0,5,7,Tampering,31,True,False +xiiotid,0.75,0,5,8,Weaponization,4276,False,False +xiiotid,0.75,0,6,0,C&C,179,True,False +xiiotid,0.75,0,6,1,Exfiltration,996,False,False +xiiotid,0.75,0,6,2,Exploitation,240,False,False +xiiotid,0.75,0,6,3,Lateral_movement,552,False,False +xiiotid,0.75,0,6,4,Normal,1197,False,False +xiiotid,0.75,0,6,5,RDOS,8278,False,False +xiiotid,0.75,0,6,6,Reconnaissance,8453,False,False +xiiotid,0.75,0,6,7,Tampering,111,True,False +xiiotid,0.75,0,6,8,Weaponization,8798,False,False +xiiotid,0.75,0,7,0,C&C,215,False,False +xiiotid,0.75,0,7,1,Exfiltration,4392,False,False +xiiotid,0.75,0,7,2,Exploitation,11,True,False +xiiotid,0.75,0,7,3,Lateral_movement,5142,False,False +xiiotid,0.75,0,7,4,Normal,13274,False,False +xiiotid,0.75,0,7,5,RDOS,5873,False,False +xiiotid,0.75,0,7,6,Reconnaissance,23824,False,False +xiiotid,0.75,0,7,7,Tampering,121,True,False +xiiotid,0.75,0,7,8,Weaponization,1393,False,False +xiiotid,0.75,0,8,0,C&C,5,True,False +xiiotid,0.75,0,8,1,Exfiltration,1887,False,False +xiiotid,0.75,0,8,2,Exploitation,74,True,False +xiiotid,0.75,0,8,3,Lateral_movement,1137,False,False +xiiotid,0.75,0,8,4,Normal,43549,False,False +xiiotid,0.75,0,8,5,RDOS,21253,False,False +xiiotid,0.75,0,8,6,Reconnaissance,458,False,False +xiiotid,0.75,0,8,7,Tampering,1227,False,False +xiiotid,0.75,0,8,8,Weaponization,12915,False,False +xiiotid,0.75,0,9,0,C&C,33,True,False +xiiotid,0.75,0,9,1,Exfiltration,3108,False,False +xiiotid,0.75,0,9,2,Exploitation,0,False,True +xiiotid,0.75,0,9,3,Lateral_movement,4614,False,False +xiiotid,0.75,0,9,4,Normal,35789,False,False +xiiotid,0.75,0,9,5,RDOS,9686,False,False +xiiotid,0.75,0,9,6,Reconnaissance,23638,False,False +xiiotid,0.75,0,9,7,Tampering,998,True,False +xiiotid,0.75,0,9,8,Weaponization,1086,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.75__seed1__characterization.csv b/artifacts/analysis/xiiotid__alpha0.75__seed1__characterization.csv new file mode 100644 index 0000000..23e44a4 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.75__seed1__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,9,1,87,538,222.66666666666666,136.2049109899411,0.6116987020506338,6.183908045977011 +1,Exfiltration,15494,10,0,239,3749,1549.4,1362.2074878666613,0.8791838697990585,15.686192468619247 +2,Exploitation,793,10,0,4,246,79.3,79.05320992850321,0.9968878931715411,61.5 +3,Lateral_movement,22117,10,0,556,4082,2211.7,1317.6872959849009,0.5957803029275675,7.341726618705036 +4,Normal,292777,10,0,4311,73956,29277.7,23958.28045186048,0.8183115631303168,17.15518441196938 +5,RDOS,98883,10,0,268,34051,9888.3,9256.126976765176,0.9360685837570842,127.05597014925372 +6,Reconnaissance,89161,10,0,520,22127,8916.1,6960.1977335417705,0.7806325336797221,42.551923076923075 +7,Tampering,3586,10,0,25,1004,358.6,307.1879554930499,0.8566312200029278,40.16 +8,Weaponization,45094,10,0,269,10726,4509.4,3590.911199124813,0.7963168490541565,39.87360594795539 diff --git a/artifacts/analysis/xiiotid__alpha0.75__seed1__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.75__seed1__rare_unseen.csv new file mode 100644 index 0000000..d9527dd --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.75__seed1__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.75,1,0,0,C&C,212,False,False +xiiotid,0.75,1,0,1,Exfiltration,3749,False,False +xiiotid,0.75,1,0,2,Exploitation,8,True,False +xiiotid,0.75,1,0,3,Lateral_movement,759,False,False +xiiotid,0.75,1,0,4,Normal,16461,False,False +xiiotid,0.75,1,0,5,RDOS,8260,False,False +xiiotid,0.75,1,0,6,Reconnaissance,2949,False,False +xiiotid,0.75,1,0,7,Tampering,102,True,False +xiiotid,0.75,1,0,8,Weaponization,1779,False,False +xiiotid,0.75,1,1,0,C&C,332,False,False +xiiotid,0.75,1,1,1,Exfiltration,705,False,False +xiiotid,0.75,1,1,2,Exploitation,39,True,False +xiiotid,0.75,1,1,3,Lateral_movement,896,False,False +xiiotid,0.75,1,1,4,Normal,73956,False,False +xiiotid,0.75,1,1,5,RDOS,6684,False,False +xiiotid,0.75,1,1,6,Reconnaissance,12303,False,False +xiiotid,0.75,1,1,7,Tampering,82,True,False +xiiotid,0.75,1,1,8,Weaponization,791,False,False +xiiotid,0.75,1,2,0,C&C,538,False,False +xiiotid,0.75,1,2,1,Exfiltration,3265,False,False +xiiotid,0.75,1,2,2,Exploitation,114,True,False +xiiotid,0.75,1,2,3,Lateral_movement,2386,False,False +xiiotid,0.75,1,2,4,Normal,6477,False,False +xiiotid,0.75,1,2,5,RDOS,10552,False,False +xiiotid,0.75,1,2,6,Reconnaissance,520,False,False +xiiotid,0.75,1,2,7,Tampering,312,True,False +xiiotid,0.75,1,2,8,Weaponization,5753,False,False +xiiotid,0.75,1,3,0,C&C,110,False,False +xiiotid,0.75,1,3,1,Exfiltration,292,False,False +xiiotid,0.75,1,3,2,Exploitation,22,True,False +xiiotid,0.75,1,3,3,Lateral_movement,688,False,False +xiiotid,0.75,1,3,4,Normal,6994,False,False +xiiotid,0.75,1,3,5,RDOS,14445,False,False +xiiotid,0.75,1,3,6,Reconnaissance,18318,False,False +xiiotid,0.75,1,3,7,Tampering,69,True,False +xiiotid,0.75,1,3,8,Weaponization,755,False,False +xiiotid,0.75,1,4,0,C&C,116,True,False +xiiotid,0.75,1,4,1,Exfiltration,239,True,False +xiiotid,0.75,1,4,2,Exploitation,246,False,False +xiiotid,0.75,1,4,3,Lateral_movement,3158,False,False +xiiotid,0.75,1,4,4,Normal,52959,False,False +xiiotid,0.75,1,4,5,RDOS,34051,False,False +xiiotid,0.75,1,4,6,Reconnaissance,8974,False,False +xiiotid,0.75,1,4,7,Tampering,754,False,False +xiiotid,0.75,1,4,8,Weaponization,9193,False,False +xiiotid,0.75,1,5,0,C&C,284,False,False +xiiotid,0.75,1,5,1,Exfiltration,977,False,False +xiiotid,0.75,1,5,2,Exploitation,194,True,False +xiiotid,0.75,1,5,3,Lateral_movement,2336,False,False +xiiotid,0.75,1,5,4,Normal,4311,False,False +xiiotid,0.75,1,5,5,RDOS,268,True,False +xiiotid,0.75,1,5,6,Reconnaissance,1523,False,False +xiiotid,0.75,1,5,7,Tampering,1004,False,False +xiiotid,0.75,1,5,8,Weaponization,10726,False,False +xiiotid,0.75,1,6,0,C&C,87,False,False +xiiotid,0.75,1,6,1,Exfiltration,1227,False,False +xiiotid,0.75,1,6,2,Exploitation,11,True,False +xiiotid,0.75,1,6,3,Lateral_movement,3567,False,False +xiiotid,0.75,1,6,4,Normal,14197,False,False +xiiotid,0.75,1,6,5,RDOS,11724,False,False +xiiotid,0.75,1,6,6,Reconnaissance,6590,False,False +xiiotid,0.75,1,6,7,Tampering,25,True,False +xiiotid,0.75,1,6,8,Weaponization,7997,False,False +xiiotid,0.75,1,7,0,C&C,131,True,False +xiiotid,0.75,1,7,1,Exfiltration,3716,False,False +xiiotid,0.75,1,7,2,Exploitation,80,True,False +xiiotid,0.75,1,7,3,Lateral_movement,3689,False,False +xiiotid,0.75,1,7,4,Normal,21292,False,False +xiiotid,0.75,1,7,5,RDOS,9898,False,False +xiiotid,0.75,1,7,6,Reconnaissance,3350,False,False +xiiotid,0.75,1,7,7,Tampering,504,False,False +xiiotid,0.75,1,7,8,Weaponization,269,False,False +xiiotid,0.75,1,8,0,C&C,0,False,True +xiiotid,0.75,1,8,1,Exfiltration,485,False,False +xiiotid,0.75,1,8,2,Exploitation,75,True,False +xiiotid,0.75,1,8,3,Lateral_movement,4082,False,False +xiiotid,0.75,1,8,4,Normal,34247,False,False +xiiotid,0.75,1,8,5,RDOS,459,False,False +xiiotid,0.75,1,8,6,Reconnaissance,12507,False,False +xiiotid,0.75,1,8,7,Tampering,297,True,False +xiiotid,0.75,1,8,8,Weaponization,3372,False,False +xiiotid,0.75,1,9,0,C&C,194,True,False +xiiotid,0.75,1,9,1,Exfiltration,839,False,False +xiiotid,0.75,1,9,2,Exploitation,4,True,False +xiiotid,0.75,1,9,3,Lateral_movement,556,False,False +xiiotid,0.75,1,9,4,Normal,61883,False,False +xiiotid,0.75,1,9,5,RDOS,2542,False,False +xiiotid,0.75,1,9,6,Reconnaissance,22127,False,False +xiiotid,0.75,1,9,7,Tampering,437,False,False +xiiotid,0.75,1,9,8,Weaponization,4459,False,False diff --git a/artifacts/analysis/xiiotid__alpha0.75__seed2__characterization.csv b/artifacts/analysis/xiiotid__alpha0.75__seed2__characterization.csv new file mode 100644 index 0000000..75fa5a2 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.75__seed2__characterization.csv @@ -0,0 +1,10 @@ +class_id,class_name,total,num_clients,n_missing_clients,min,max,mean,std,cv,max_min_ratio +0,C&C,2004,10,0,1,546,200.4,183.81414526635322,0.9172362538241179,546.0 +1,Exfiltration,15494,10,0,9,3572,1549.4,1333.091834796088,0.8603923033407048,396.8888888888889 +2,Exploitation,793,10,0,11,382,79.3,104.84374087183268,1.3221152695060867,34.72727272727273 +3,Lateral_movement,22117,10,0,3,5709,2211.7,1943.4331503810467,0.8787055886336514,1903.0 +4,Normal,292777,10,0,1930,64353,29277.7,23149.92923984866,0.7907017709672776,33.343523316062175 +5,RDOS,98883,10,0,636,18655,9888.3,5578.985284977905,0.5642006497555602,29.33176100628931 +6,Reconnaissance,89161,10,0,1528,21418,8916.1,6174.089899734211,0.692465304307288,14.017015706806284 +7,Tampering,3586,10,0,1,1321,358.6,398.6803732314898,1.1117690274163128,1321.0 +8,Weaponization,45094,10,0,169,17328,4509.4,5078.904058948151,1.1262926462385576,102.53254437869822 diff --git a/artifacts/analysis/xiiotid__alpha0.75__seed2__rare_unseen.csv b/artifacts/analysis/xiiotid__alpha0.75__seed2__rare_unseen.csv new file mode 100644 index 0000000..a00cb47 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.75__seed2__rare_unseen.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,client_id,class_id,class_name,N_ij,rare_flag,unseen_flag +xiiotid,0.75,2,0,0,C&C,226,False,False +xiiotid,0.75,2,0,1,Exfiltration,320,False,False +xiiotid,0.75,2,0,2,Exploitation,48,True,False +xiiotid,0.75,2,0,3,Lateral_movement,1483,False,False +xiiotid,0.75,2,0,4,Normal,14394,False,False +xiiotid,0.75,2,0,5,RDOS,8158,False,False +xiiotid,0.75,2,0,6,Reconnaissance,5775,False,False +xiiotid,0.75,2,0,7,Tampering,28,True,False +xiiotid,0.75,2,0,8,Weaponization,175,False,False +xiiotid,0.75,2,1,0,C&C,39,True,False +xiiotid,0.75,2,1,1,Exfiltration,443,False,False +xiiotid,0.75,2,1,2,Exploitation,43,True,False +xiiotid,0.75,2,1,3,Lateral_movement,626,False,False +xiiotid,0.75,2,1,4,Normal,10435,False,False +xiiotid,0.75,2,1,5,RDOS,14173,False,False +xiiotid,0.75,2,1,6,Reconnaissance,10388,False,False +xiiotid,0.75,2,1,7,Tampering,1321,False,False +xiiotid,0.75,2,1,8,Weaponization,3511,False,False +xiiotid,0.75,2,2,0,C&C,1,True,False +xiiotid,0.75,2,2,1,Exfiltration,271,False,False +xiiotid,0.75,2,2,2,Exploitation,382,False,False +xiiotid,0.75,2,2,3,Lateral_movement,3698,False,False +xiiotid,0.75,2,2,4,Normal,35232,False,False +xiiotid,0.75,2,2,5,RDOS,856,False,False +xiiotid,0.75,2,2,6,Reconnaissance,1528,False,False +xiiotid,0.75,2,2,7,Tampering,547,False,False +xiiotid,0.75,2,2,8,Weaponization,169,True,False +xiiotid,0.75,2,3,0,C&C,198,False,False +xiiotid,0.75,2,3,1,Exfiltration,2231,False,False +xiiotid,0.75,2,3,2,Exploitation,25,True,False +xiiotid,0.75,2,3,3,Lateral_movement,2434,False,False +xiiotid,0.75,2,3,4,Normal,61429,False,False +xiiotid,0.75,2,3,5,RDOS,12067,False,False +xiiotid,0.75,2,3,6,Reconnaissance,3527,False,False +xiiotid,0.75,2,3,7,Tampering,123,True,False +xiiotid,0.75,2,3,8,Weaponization,3814,False,False +xiiotid,0.75,2,4,0,C&C,27,True,False +xiiotid,0.75,2,4,1,Exfiltration,3050,False,False +xiiotid,0.75,2,4,2,Exploitation,17,True,False +xiiotid,0.75,2,4,3,Lateral_movement,2509,False,False +xiiotid,0.75,2,4,4,Normal,60920,False,False +xiiotid,0.75,2,4,5,RDOS,6987,False,False +xiiotid,0.75,2,4,6,Reconnaissance,21418,False,False +xiiotid,0.75,2,4,7,Tampering,416,False,False +xiiotid,0.75,2,4,8,Weaponization,3668,False,False +xiiotid,0.75,2,5,0,C&C,195,False,False +xiiotid,0.75,2,5,1,Exfiltration,3338,False,False +xiiotid,0.75,2,5,2,Exploitation,27,True,False +xiiotid,0.75,2,5,3,Lateral_movement,5036,False,False +xiiotid,0.75,2,5,4,Normal,15350,False,False +xiiotid,0.75,2,5,5,RDOS,10151,False,False +xiiotid,0.75,2,5,6,Reconnaissance,4883,False,False +xiiotid,0.75,2,5,7,Tampering,1,True,False +xiiotid,0.75,2,5,8,Weaponization,592,False,False +xiiotid,0.75,2,6,0,C&C,167,True,False +xiiotid,0.75,2,6,1,Exfiltration,551,False,False +xiiotid,0.75,2,6,2,Exploitation,11,True,False +xiiotid,0.75,2,6,3,Lateral_movement,5709,False,False +xiiotid,0.75,2,6,4,Normal,7460,False,False +xiiotid,0.75,2,6,5,RDOS,636,False,False +xiiotid,0.75,2,6,6,Reconnaissance,8572,False,False +xiiotid,0.75,2,6,7,Tampering,684,False,False +xiiotid,0.75,2,6,8,Weaponization,434,False,False +xiiotid,0.75,2,7,0,C&C,546,False,False +xiiotid,0.75,2,7,1,Exfiltration,9,True,False +xiiotid,0.75,2,7,2,Exploitation,67,False,False +xiiotid,0.75,2,7,3,Lateral_movement,3,True,False +xiiotid,0.75,2,7,4,Normal,1930,False,False +xiiotid,0.75,2,7,5,RDOS,15192,False,False +xiiotid,0.75,2,7,6,Reconnaissance,3147,False,False +xiiotid,0.75,2,7,7,Tampering,416,False,False +xiiotid,0.75,2,7,8,Weaponization,6839,False,False +xiiotid,0.75,2,8,0,C&C,526,False,False +xiiotid,0.75,2,8,1,Exfiltration,3572,False,False +xiiotid,0.75,2,8,2,Exploitation,59,True,False +xiiotid,0.75,2,8,3,Lateral_movement,145,False,False +xiiotid,0.75,2,8,4,Normal,21274,False,False +xiiotid,0.75,2,8,5,RDOS,12008,False,False +xiiotid,0.75,2,8,6,Reconnaissance,16761,False,False +xiiotid,0.75,2,8,7,Tampering,28,True,False +xiiotid,0.75,2,8,8,Weaponization,8564,False,False +xiiotid,0.75,2,9,0,C&C,79,True,False +xiiotid,0.75,2,9,1,Exfiltration,1709,False,False +xiiotid,0.75,2,9,2,Exploitation,114,False,False +xiiotid,0.75,2,9,3,Lateral_movement,474,False,False +xiiotid,0.75,2,9,4,Normal,64353,False,False +xiiotid,0.75,2,9,5,RDOS,18655,False,False +xiiotid,0.75,2,9,6,Reconnaissance,13162,False,False +xiiotid,0.75,2,9,7,Tampering,22,True,False +xiiotid,0.75,2,9,8,Weaponization,17328,False,False diff --git a/artifacts/partitions/5gnidd__alpha0.25__seed0.json b/artifacts/partitions/5gnidd__alpha0.25__seed0.json new file mode 100644 index 0000000..1178ef6 --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.25__seed0.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:12Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "e628c17369f25e355cc8917f140e43f97e37ddc77e3193b6a1b0cbdc75a9ee12", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 708356900, + "seed": 0, + "split": { + "hash": "18f89f4807de5493e83cbc19a87df51268050c5fb3a12f286e64c3115073dea7", + "seed": 0, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 6003, + 4887, + 319, + 275, + 955, + 11954, + 92, + 127301, + 2 + ], + [ + 40519, + 1, + 181, + 0, + 289, + 18, + 52, + 62527, + 4601 + ], + [ + 17462, + 300, + 2, + 184, + 0, + 1454, + 63, + 319, + 4159 + ], + [ + 4648, + 13072, + 42, + 4762, + 3021, + 16517, + 1548, + 1, + 5 + ], + [ + 5, + 0, + 15, + 144, + 447, + 14314, + 10870, + 1606, + 28 + ], + [ + 185504, + 48469, + 0, + 0, + 0, + 1, + 118, + 2869, + 732 + ], + [ + 1807, + 3, + 78, + 559, + 2557, + 23, + 0, + 94879, + 1052 + ], + [ + 1, + 26635, + 34, + 0, + 0, + 0, + 81, + 26932, + 389 + ], + [ + 63395, + 589, + 6, + 419, + 237, + 810, + 2, + 1195, + 145 + ], + [ + 15071, + 4613, + 131, + 462, + 6524, + 6096, + 1211, + 2509, + 21 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.25__seed1.json b/artifacts/partitions/5gnidd__alpha0.25__seed1.json new file mode 100644 index 0000000..c93da63 --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.25__seed1.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:13Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "05aec12ceb4c94bda85b8baed5f4fa877017aa180c543c966eacbeb620397e0e", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 902143161, + "seed": 1, + "split": { + "hash": "3d05b7fe48d5a06ce5076eb60084349adc777872a54ddd566f7461f2f0c3579e", + "seed": 1, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 39367, + 0, + 27, + 2997, + 3051, + 0, + 368, + 16667, + 0 + ], + [ + 6206, + 45645, + 78, + 0, + 7, + 442, + 1716, + 4540, + 5172 + ], + [ + 30, + 28, + 4, + 408, + 3477, + 290, + 1886, + 13, + 2745 + ], + [ + 5914, + 365, + 225, + 708, + 362, + 11227, + 0, + 75580, + 0 + ], + [ + 42476, + 30225, + 257, + 1743, + 1514, + 6019, + 1935, + 1, + 0 + ], + [ + 3442, + 5, + 2, + 97, + 17, + 211, + 2361, + 1197, + 67 + ], + [ + 43284, + 660, + 1, + 10, + 1401, + 9026, + 3627, + 60225, + 1209 + ], + [ + 505, + 11422, + 29, + 518, + 107, + 0, + 0, + 33227, + 1006 + ], + [ + 15588, + 9984, + 11, + 276, + 3125, + 494, + 36, + 128338, + 0 + ], + [ + 177603, + 235, + 174, + 48, + 969, + 23478, + 2108, + 350, + 935 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.25__seed2.json b/artifacts/partitions/5gnidd__alpha0.25__seed2.json new file mode 100644 index 0000000..b716219 --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.25__seed2.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:13Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "d9631ea6b2d3907e04389af8003ae7444fc50af30467d70853690cf5e87d5660", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 218373793, + "seed": 2, + "split": { + "hash": "fc758d060dc51dec113b118e07cda6a012b5bd825aa64fb515d595cd1684417f", + "seed": 2, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 46403, + 0, + 203, + 258, + 4, + 226, + 2548, + 55627, + 3899 + ], + [ + 20110, + 1015, + 6, + 3, + 843, + 692, + 2980, + 25373, + 0 + ], + [ + 12386, + 0, + 0, + 3136, + 10072, + 1818, + 564, + 113, + 436 + ], + [ + 29, + 13422, + 355, + 221, + 448, + 31740, + 1136, + 61079, + 0 + ], + [ + 3704, + 43, + 155, + 2230, + 117, + 8482, + 6, + 4837, + 243 + ], + [ + 12499, + 578, + 0, + 4, + 2, + 69, + 94, + 5864, + 1018 + ], + [ + 227468, + 36762, + 17, + 72, + 32, + 4423, + 3, + 28468, + 947 + ], + [ + 5134, + 27221, + 28, + 0, + 1955, + 4, + 0, + 27150, + 1184 + ], + [ + 10, + 0, + 11, + 543, + 429, + 2548, + 3087, + 82051, + 0 + ], + [ + 6672, + 19528, + 33, + 338, + 128, + 1185, + 3619, + 29576, + 3407 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.5__seed0.json b/artifacts/partitions/5gnidd__alpha0.5__seed0.json new file mode 100644 index 0000000..cfca4af --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.5__seed0.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:11Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "8352d4ff0e33f246a22da8516812bf8ff3ddbdecb6a6435fbcf08e9edcf989ab", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 673621352, + "seed": 0, + "split": { + "hash": "18f89f4807de5493e83cbc19a87df51268050c5fb3a12f286e64c3115073dea7", + "seed": 0, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 108257, + 5466, + 173, + 373, + 695, + 455, + 6, + 1, + 1359 + ], + [ + 6399, + 6782, + 69, + 3250, + 63, + 1431, + 2068, + 57810, + 289 + ], + [ + 68144, + 3688, + 57, + 72, + 100, + 80, + 2059, + 23607, + 212 + ], + [ + 22029, + 4983, + 0, + 367, + 667, + 16850, + 2108, + 129, + 1403 + ], + [ + 1096, + 388, + 8, + 0, + 11, + 1630, + 2823, + 44903, + 3420 + ], + [ + 33517, + 185, + 12, + 1592, + 583, + 692, + 1, + 88943, + 1057 + ], + [ + 226, + 843, + 101, + 10, + 317, + 28363, + 9, + 14100, + 14 + ], + [ + 9926, + 2258, + 105, + 2, + 1852, + 185, + 235, + 733, + 92 + ], + [ + 76033, + 61393, + 243, + 408, + 6713, + 477, + 553, + 30580, + 2593 + ], + [ + 8788, + 12583, + 40, + 731, + 3029, + 1024, + 4175, + 59332, + 695 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.5__seed1.json b/artifacts/partitions/5gnidd__alpha0.5__seed1.json new file mode 100644 index 0000000..80dee12 --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.5__seed1.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:11Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "0815c91ed82fe7e1502a49fa1de050fea9d5b037d051a0c3fb900d3a0f7bfa08", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 1403944623, + "seed": 1, + "split": { + "hash": "3d05b7fe48d5a06ce5076eb60084349adc777872a54ddd566f7461f2f0c3579e", + "seed": 1, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 104938, + 13117, + 18, + 393, + 1017, + 711, + 1151, + 9498, + 243 + ], + [ + 2307, + 36928, + 30, + 151, + 2238, + 1838, + 1036, + 95125, + 491 + ], + [ + 803, + 943, + 45, + 825, + 616, + 10016, + 595, + 17027, + 743 + ], + [ + 12946, + 51, + 8, + 194, + 443, + 4208, + 50, + 39604, + 17 + ], + [ + 25034, + 269, + 10, + 1146, + 387, + 7838, + 162, + 25965, + 1442 + ], + [ + 58927, + 6977, + 121, + 104, + 3621, + 8214, + 7188, + 31787, + 1 + ], + [ + 10286, + 8479, + 120, + 2093, + 1083, + 16089, + 1757, + 54891, + 2687 + ], + [ + 55826, + 18815, + 38, + 1027, + 845, + 1671, + 401, + 41124, + 5065 + ], + [ + 47693, + 224, + 284, + 26, + 3521, + 26, + 3, + 1907, + 75 + ], + [ + 15655, + 12766, + 134, + 846, + 259, + 576, + 1694, + 3210, + 370 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.5__seed2.json b/artifacts/partitions/5gnidd__alpha0.5__seed2.json new file mode 100644 index 0000000..b81b451 --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.5__seed2.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:12Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "fdcf388e31658ae11a6068e9dc2c6f27ef1567ecce60855efe6d8e3dc75cebf6", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 633461056, + "seed": 2, + "split": { + "hash": "fc758d060dc51dec113b118e07cda6a012b5bd825aa64fb515d595cd1684417f", + "seed": 2, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 225038, + 24, + 1, + 179, + 2446, + 6644, + 6952, + 14179, + 0 + ], + [ + 39441, + 48351, + 2, + 250, + 454, + 104, + 1256, + 4099, + 3 + ], + [ + 4137, + 8055, + 20, + 8, + 1510, + 4532, + 833, + 990, + 2 + ], + [ + 807, + 28464, + 76, + 122, + 319, + 2592, + 365, + 7413, + 4 + ], + [ + 43342, + 47, + 358, + 683, + 33, + 1353, + 1991, + 49539, + 2253 + ], + [ + 255, + 582, + 122, + 468, + 5, + 3365, + 125, + 67573, + 827 + ], + [ + 1352, + 2752, + 44, + 3530, + 3619, + 2995, + 0, + 105, + 6727 + ], + [ + 864, + 4464, + 57, + 32, + 807, + 17642, + 817, + 27464, + 567 + ], + [ + 14286, + 5759, + 45, + 1201, + 7, + 390, + 418, + 138381, + 508 + ], + [ + 4893, + 71, + 83, + 332, + 4830, + 11570, + 1280, + 10395, + 243 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.75__seed0.json b/artifacts/partitions/5gnidd__alpha0.75__seed0.json new file mode 100644 index 0000000..01017b6 --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.75__seed0.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.75, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:10Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "74a91845627dbe57efab9a5cf94e37bcce98ce38edb7936a8c1cbd8f040d3dc8", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 892370747, + "seed": 0, + "split": { + "hash": "18f89f4807de5493e83cbc19a87df51268050c5fb3a12f286e64c3115073dea7", + "seed": 0, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 46676, + 2633, + 0, + 765, + 252, + 2, + 2742, + 21728, + 3082 + ], + [ + 69384, + 8301, + 239, + 26, + 82, + 11818, + 386, + 6626, + 886 + ], + [ + 7505, + 7904, + 80, + 50, + 1089, + 585, + 1933, + 124158, + 243 + ], + [ + 58004, + 4612, + 0, + 3213, + 697, + 15285, + 363, + 2320, + 2343 + ], + [ + 3818, + 17423, + 54, + 0, + 2163, + 10667, + 1926, + 2960, + 3370 + ], + [ + 21356, + 1375, + 143, + 180, + 645, + 1615, + 379, + 23593, + 198 + ], + [ + 42870, + 557, + 53, + 706, + 4335, + 612, + 2301, + 41823, + 429 + ], + [ + 1410, + 25878, + 101, + 100, + 3133, + 277, + 3531, + 4005, + 404 + ], + [ + 24950, + 27843, + 113, + 1313, + 145, + 645, + 424, + 24580, + 74 + ], + [ + 58442, + 2043, + 25, + 452, + 1489, + 9681, + 52, + 68345, + 105 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.75__seed1.json b/artifacts/partitions/5gnidd__alpha0.75__seed1.json new file mode 100644 index 0000000..343c3ad --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.75__seed1.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.75, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:10Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "fc04ba00f4942bf9983ad8de50a86e066f83ba17a806195c0205498c3d34f69c", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 1817661572, + "seed": 1, + "split": { + "hash": "3d05b7fe48d5a06ce5076eb60084349adc777872a54ddd566f7461f2f0c3579e", + "seed": 1, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 15512, + 9688, + 27, + 576, + 2805, + 1642, + 2283, + 2744, + 1410 + ], + [ + 5328, + 15997, + 25, + 817, + 921, + 6569, + 737, + 13401, + 2727 + ], + [ + 8864, + 4494, + 30, + 1420, + 133, + 5965, + 775, + 23126, + 451 + ], + [ + 80420, + 621, + 0, + 229, + 2650, + 514, + 1149, + 114825, + 2421 + ], + [ + 3896, + 2280, + 15, + 57, + 565, + 6824, + 809, + 1533, + 1882 + ], + [ + 20027, + 25248, + 37, + 830, + 3883, + 12114, + 2861, + 48726, + 1858 + ], + [ + 13275, + 7039, + 15, + 199, + 0, + 7156, + 1882, + 22538, + 40 + ], + [ + 31700, + 13892, + 240, + 72, + 1949, + 3092, + 1065, + 64629, + 5 + ], + [ + 83388, + 12304, + 134, + 247, + 400, + 2594, + 326, + 27585, + 152 + ], + [ + 72005, + 7006, + 285, + 2358, + 724, + 4717, + 2150, + 1031, + 188 + ] + ] +} diff --git a/artifacts/partitions/5gnidd__alpha0.75__seed2.json b/artifacts/partitions/5gnidd__alpha0.75__seed2.json new file mode 100644 index 0000000..6f0754a --- /dev/null +++ b/artifacts/partitions/5gnidd__alpha0.75__seed2.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.75, + "benign_class": "Benign", + "class_names": [ + "Benign", + "HTTPFlood", + "ICMPFlood", + "SYNFlood", + "SYNScan", + "SlowrateDoS", + "TCPConnectScan", + "UDPFlood", + "UDPScan" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:10Z", + "dataset": "5gnidd", + "dataset_name": "5G-NIDD", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 851123, + "partition_hash": "4ee33d0a28326ae1d0e5574a8cd850068ef4ec1cc631b7366daab65b3e22ca86", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 306220443, + "seed": 2, + "split": { + "hash": "fc758d060dc51dec113b118e07cda6a012b5bd825aa64fb515d595cd1684417f", + "seed": 2, + "sizes": { + "test": 243177, + "train": 851123, + "validation": 121589 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 7083, + 17146, + 88, + 380, + 4022, + 335, + 1387, + 47750, + 600 + ], + [ + 31172, + 12345, + 8, + 2670, + 626, + 18020, + 3054, + 39353, + 562 + ], + [ + 30915, + 1484, + 35, + 462, + 3070, + 5630, + 105, + 15448, + 1283 + ], + [ + 41272, + 8659, + 25, + 6, + 737, + 8841, + 84, + 5446, + 37 + ], + [ + 7313, + 25410, + 41, + 110, + 357, + 5017, + 783, + 8809, + 1697 + ], + [ + 9497, + 7911, + 19, + 1678, + 710, + 4985, + 1378, + 45069, + 238 + ], + [ + 118638, + 5810, + 251, + 185, + 1939, + 1759, + 1832, + 58797, + 2697 + ], + [ + 67601, + 10568, + 270, + 59, + 160, + 4201, + 3335, + 2637, + 283 + ], + [ + 20388, + 3396, + 1, + 450, + 91, + 99, + 1675, + 33413, + 3282 + ], + [ + 536, + 5840, + 70, + 805, + 2318, + 2300, + 404, + 63416, + 455 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.25__seed0.json b/artifacts/partitions/xiiotid__alpha0.25__seed0.json new file mode 100644 index 0000000..911bc62 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.25__seed0.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:01Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "60faf299c0220bb6dd013fd4621babe8f6ad342e3012b983bce11445252a2b21", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 1617371214, + "seed": 0, + "split": { + "hash": "d0acc0c14e3ecbeba02dfa9e6227c983da12dec2d6111df0b714002e6e03c29d", + "seed": 0, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 193, + 364, + 73, + 6253, + 1, + 26440, + 46, + 159, + 43 + ], + [ + 247, + 0, + 0, + 21, + 102, + 166, + 65307, + 2272, + 0 + ], + [ + 131, + 187, + 0, + 578, + 1048, + 3516, + 4027, + 9, + 168 + ], + [ + 29, + 45, + 3, + 2, + 38918, + 31089, + 0, + 0, + 3342 + ], + [ + 21, + 0, + 2, + 13857, + 122800, + 727, + 3644, + 307, + 0 + ], + [ + 51, + 4, + 0, + 808, + 633, + 14233, + 4363, + 453, + 347 + ], + [ + 87, + 7950, + 57, + 0, + 6773, + 1637, + 0, + 241, + 23381 + ], + [ + 116, + 9, + 595, + 573, + 71934, + 4941, + 1867, + 131, + 16998 + ], + [ + 1065, + 2523, + 63, + 7, + 2928, + 1704, + 67, + 14, + 704 + ], + [ + 64, + 4412, + 0, + 18, + 47640, + 14430, + 9840, + 0, + 111 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.25__seed1.json b/artifacts/partitions/xiiotid__alpha0.25__seed1.json new file mode 100644 index 0000000..90c9f72 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.25__seed1.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:01Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "6ffb10aec991b9d440bad675b72f07d0807c73fa334f86b284eeb90df31687aa", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 1670706564, + "seed": 1, + "split": { + "hash": "880a9f82686bbf3af83880238aeef6f1462881ba87c5ae667b7415eb10d79892", + "seed": 1, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 1, + 48, + 78, + 1, + 45538, + 168, + 4841, + 2, + 8960 + ], + [ + 21, + 129, + 16, + 10094, + 75982, + 5627, + 65101, + 577, + 245 + ], + [ + 0, + 3823, + 0, + 2, + 0, + 0, + 0, + 151, + 5302 + ], + [ + 21, + 449, + 315, + 82, + 2706, + 2835, + 6322, + 170, + 0 + ], + [ + 46, + 262, + 22, + 793, + 16624, + 12854, + 202, + 0, + 9118 + ], + [ + 181, + 1, + 3, + 6163, + 129678, + 53482, + 439, + 18, + 3361 + ], + [ + 723, + 1643, + 0, + 830, + 0, + 998, + 138, + 1804, + 5936 + ], + [ + 20, + 6204, + 321, + 83, + 198, + 232, + 2503, + 314, + 1 + ], + [ + 986, + 1919, + 1, + 2228, + 14813, + 4108, + 0, + 16, + 147 + ], + [ + 5, + 1016, + 37, + 1841, + 7238, + 18579, + 9615, + 534, + 12024 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.25__seed2.json b/artifacts/partitions/xiiotid__alpha0.25__seed2.json new file mode 100644 index 0000000..f24a5c6 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.25__seed2.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.25, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:02Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "c8e1ad316272b858d61b083cd65a174ed674f37157c24d06efb713960b710996", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 795318573, + "seed": 2, + "split": { + "hash": "8e0db0be53055317b4df53398fad500c1a3f09491967539bf5459ba5600cee09", + "seed": 2, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 0, + 430, + 0, + 0, + 118937, + 2912, + 118, + 16, + 113 + ], + [ + 2, + 2477, + 225, + 1, + 10987, + 8514, + 6193, + 5, + 11858 + ], + [ + 1265, + 388, + 11, + 839, + 0, + 48839, + 121, + 1, + 6425 + ], + [ + 0, + 8269, + 0, + 631, + 452, + 17375, + 14322, + 16, + 329 + ], + [ + 428, + 0, + 11, + 8040, + 5015, + 11529, + 34, + 18, + 0 + ], + [ + 0, + 4, + 0, + 29, + 61418, + 1181, + 7835, + 2584, + 3257 + ], + [ + 0, + 1201, + 1, + 0, + 35251, + 1382, + 12411, + 2, + 240 + ], + [ + 17, + 2713, + 60, + 81, + 28172, + 2, + 46541, + 355, + 22853 + ], + [ + 128, + 12, + 0, + 12496, + 576, + 281, + 1585, + 582, + 4 + ], + [ + 164, + 0, + 485, + 0, + 31969, + 6868, + 1, + 7, + 15 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.5__seed0.json b/artifacts/partitions/xiiotid__alpha0.5__seed0.json new file mode 100644 index 0000000..65b2383 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.5__seed0.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:00Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "2465485539a3e923548dcc0c73d8acc5076d456536b78682d512897ea586e779", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 2009062324, + "seed": 0, + "split": { + "hash": "d0acc0c14e3ecbeba02dfa9e6227c983da12dec2d6111df0b714002e6e03c29d", + "seed": 0, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 126, + 901, + 0, + 5915, + 48503, + 9867, + 29234, + 0, + 18 + ], + [ + 559, + 2212, + 69, + 180, + 34165, + 6640, + 5054, + 1266, + 486 + ], + [ + 92, + 6305, + 353, + 949, + 604, + 10123, + 1636, + 5, + 32409 + ], + [ + 966, + 1509, + 140, + 630, + 13892, + 49283, + 1137, + 832, + 1220 + ], + [ + 1, + 1151, + 18, + 1747, + 733, + 168, + 14033, + 336, + 334 + ], + [ + 4, + 554, + 53, + 315, + 11521, + 1150, + 27596, + 2, + 691 + ], + [ + 44, + 0, + 0, + 588, + 34309, + 5683, + 3655, + 93, + 713 + ], + [ + 20, + 2414, + 10, + 8385, + 57789, + 9870, + 2431, + 506, + 2971 + ], + [ + 4, + 45, + 138, + 3359, + 68657, + 5008, + 1662, + 385, + 5815 + ], + [ + 188, + 403, + 12, + 49, + 22604, + 1091, + 2723, + 161, + 437 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.5__seed1.json b/artifacts/partitions/xiiotid__alpha0.5__seed1.json new file mode 100644 index 0000000..f0912c8 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.5__seed1.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:01Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "4061c5a15ff5c981cb0ae90beb892ad2ef730fb460a995d80fcef77bbcba52a3", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 621679668, + "seed": 1, + "split": { + "hash": "880a9f82686bbf3af83880238aeef6f1462881ba87c5ae667b7415eb10d79892", + "seed": 1, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 2, + 627, + 56, + 2550, + 89253, + 4878, + 12996, + 8, + 11526 + ], + [ + 3, + 199, + 0, + 3695, + 23474, + 95, + 5840, + 1228, + 2611 + ], + [ + 0, + 1006, + 50, + 818, + 60061, + 2816, + 1842, + 260, + 1209 + ], + [ + 687, + 916, + 86, + 1239, + 69735, + 2508, + 14852, + 91, + 1647 + ], + [ + 284, + 6273, + 46, + 242, + 18842, + 341, + 23967, + 483, + 917 + ], + [ + 599, + 2759, + 8, + 49, + 1776, + 6, + 9632, + 680, + 1375 + ], + [ + 0, + 2518, + 9, + 8361, + 9668, + 47276, + 4475, + 0, + 951 + ], + [ + 82, + 8, + 46, + 4589, + 17477, + 20791, + 2025, + 618, + 6976 + ], + [ + 236, + 1113, + 1, + 396, + 260, + 13932, + 6536, + 119, + 6913 + ], + [ + 111, + 75, + 491, + 178, + 2231, + 6240, + 6996, + 99, + 10969 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.5__seed2.json b/artifacts/partitions/xiiotid__alpha0.5__seed2.json new file mode 100644 index 0000000..320dc97 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.5__seed2.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.5, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:01Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "1118e7fa4e7d852e38b1b8a6e2789c5c6e357579698de7497a4436d02869e214", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 1438483982, + "seed": 2, + "split": { + "hash": "8e0db0be53055317b4df53398fad500c1a3f09491967539bf5459ba5600cee09", + "seed": 2, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 576, + 2126, + 10, + 3084, + 864, + 17951, + 1651, + 118, + 895 + ], + [ + 290, + 20, + 1, + 11, + 418, + 7758, + 208, + 371, + 1014 + ], + [ + 0, + 914, + 240, + 2152, + 79, + 48860, + 12985, + 135, + 23228 + ], + [ + 211, + 1557, + 2, + 72, + 33716, + 94, + 17154, + 2437, + 10869 + ], + [ + 37, + 30, + 35, + 566, + 10789, + 1624, + 22975, + 76, + 218 + ], + [ + 252, + 1430, + 145, + 6341, + 1541, + 1142, + 16770, + 132, + 1570 + ], + [ + 212, + 5300, + 298, + 3130, + 73646, + 4, + 5123, + 146, + 17 + ], + [ + 13, + 2362, + 49, + 4431, + 57785, + 10129, + 8929, + 26, + 2670 + ], + [ + 296, + 150, + 7, + 131, + 92031, + 4382, + 2708, + 64, + 38 + ], + [ + 117, + 1605, + 6, + 2199, + 21908, + 6939, + 658, + 81, + 4575 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.75__seed0.json b/artifacts/partitions/xiiotid__alpha0.75__seed0.json new file mode 100644 index 0000000..607b878 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.75__seed0.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.75, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:23:59Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "473b1c4aaaa9e55e5cc2d108a261345e6630fe481630987df60a5ac30f793048", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 957653162, + "seed": 0, + "split": { + "hash": "d0acc0c14e3ecbeba02dfa9e6227c983da12dec2d6111df0b714002e6e03c29d", + "seed": 0, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 196, + 2293, + 96, + 1364, + 24224, + 22821, + 1040, + 85, + 1114 + ], + [ + 387, + 189, + 136, + 5435, + 8909, + 347, + 4622, + 95, + 4116 + ], + [ + 0, + 1021, + 30, + 190, + 92460, + 2604, + 2096, + 126, + 2046 + ], + [ + 229, + 583, + 1, + 1149, + 1151, + 438, + 16930, + 88, + 9065 + ], + [ + 0, + 27, + 175, + 2323, + 52309, + 13676, + 1111, + 704, + 285 + ], + [ + 760, + 998, + 30, + 211, + 19915, + 13907, + 6989, + 31, + 4276 + ], + [ + 179, + 996, + 240, + 552, + 1197, + 8278, + 8453, + 111, + 8798 + ], + [ + 215, + 4392, + 11, + 5142, + 13274, + 5873, + 23824, + 121, + 1393 + ], + [ + 5, + 1887, + 74, + 1137, + 43549, + 21253, + 458, + 1227, + 12915 + ], + [ + 33, + 3108, + 0, + 4614, + 35789, + 9686, + 23638, + 998, + 1086 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.75__seed1.json b/artifacts/partitions/xiiotid__alpha0.75__seed1.json new file mode 100644 index 0000000..0241476 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.75__seed1.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.75, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:00Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "a0ec9e9c0902c64b12bbfdb2c2ec1ee814a83699b2d48c88b7edd6255aa90022", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 762050403, + "seed": 1, + "split": { + "hash": "880a9f82686bbf3af83880238aeef6f1462881ba87c5ae667b7415eb10d79892", + "seed": 1, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 212, + 3749, + 8, + 759, + 16461, + 8260, + 2949, + 102, + 1779 + ], + [ + 332, + 705, + 39, + 896, + 73956, + 6684, + 12303, + 82, + 791 + ], + [ + 538, + 3265, + 114, + 2386, + 6477, + 10552, + 520, + 312, + 5753 + ], + [ + 110, + 292, + 22, + 688, + 6994, + 14445, + 18318, + 69, + 755 + ], + [ + 116, + 239, + 246, + 3158, + 52959, + 34051, + 8974, + 754, + 9193 + ], + [ + 284, + 977, + 194, + 2336, + 4311, + 268, + 1523, + 1004, + 10726 + ], + [ + 87, + 1227, + 11, + 3567, + 14197, + 11724, + 6590, + 25, + 7997 + ], + [ + 131, + 3716, + 80, + 3689, + 21292, + 9898, + 3350, + 504, + 269 + ], + [ + 0, + 485, + 75, + 4082, + 34247, + 459, + 12507, + 297, + 3372 + ], + [ + 194, + 839, + 4, + 556, + 61883, + 2542, + 22127, + 437, + 4459 + ] + ] +} diff --git a/artifacts/partitions/xiiotid__alpha0.75__seed2.json b/artifacts/partitions/xiiotid__alpha0.75__seed2.json new file mode 100644 index 0000000..81cab16 --- /dev/null +++ b/artifacts/partitions/xiiotid__alpha0.75__seed2.json @@ -0,0 +1,154 @@ +{ + "algorithm": "per_class_dirichlet_multinomial", + "alpha": 0.75, + "benign_class": "Normal", + "class_names": [ + "C&C", + "Exfiltration", + "Exploitation", + "Lateral_movement", + "Normal", + "RDOS", + "Reconnaissance", + "Tampering", + "Weaponization" + ], + "config_hash": null, + "created_utc": "2026-09-24T17:24:00Z", + "dataset": "xiiotid", + "dataset_name": "X-IIoTID", + "manifest_version": "1", + "num_classes": 9, + "num_clients": 10, + "num_train": 569909, + "partition_hash": "00345255053affa05ebf348cee3efd6dc85ede6c4e500de881b9d96605d99b0e", + "preprocessing_hash": null, + "python": "3.12.14", + "rng_seed": 843543396, + "seed": 2, + "split": { + "hash": "8e0db0be53055317b4df53398fad500c1a3f09491967539bf5459ba5600cee09", + "seed": 2, + "sizes": { + "test": 162831, + "train": 569909, + "validation": 81414 + }, + "strategy": "stratified_by_class", + "test_fraction": 0.2, + "train_fraction": 0.7, + "validation_fraction": 0.1 + }, + "support": [ + [ + 226, + 320, + 48, + 1483, + 14394, + 8158, + 5775, + 28, + 175 + ], + [ + 39, + 443, + 43, + 626, + 10435, + 14173, + 10388, + 1321, + 3511 + ], + [ + 1, + 271, + 382, + 3698, + 35232, + 856, + 1528, + 547, + 169 + ], + [ + 198, + 2231, + 25, + 2434, + 61429, + 12067, + 3527, + 123, + 3814 + ], + [ + 27, + 3050, + 17, + 2509, + 60920, + 6987, + 21418, + 416, + 3668 + ], + [ + 195, + 3338, + 27, + 5036, + 15350, + 10151, + 4883, + 1, + 592 + ], + [ + 167, + 551, + 11, + 5709, + 7460, + 636, + 8572, + 684, + 434 + ], + [ + 546, + 9, + 67, + 3, + 1930, + 15192, + 3147, + 416, + 6839 + ], + [ + 526, + 3572, + 59, + 145, + 21274, + 12008, + 16761, + 28, + 8564 + ], + [ + 79, + 1709, + 114, + 474, + 64353, + 18655, + 13162, + 22, + 17328 + ] + ] +} diff --git a/configs/datasets/datasets.yaml b/configs/datasets/datasets.yaml index eb82db7..7ecba1f 100644 --- a/configs/datasets/datasets.yaml +++ b/configs/datasets/datasets.yaml @@ -13,6 +13,13 @@ # All preprocessed caches, partitions, manifests, checkpoints and analysis # outputs must be written under `generated_root` (default: artifacts/), never # back into the raw dataset directory. +# +# PREPROCESSING CONTRACT (frozen 2026-09-24, see docs/research/EXPERIMENT_PROTOCOL.md) +# * `categorical_columns` are one-hot encoded with categories fitted on the +# TRAIN split only; an explicit "__MISSING__" category is added. +# * every other feature column is coerced to numeric (missing tokens -> NaN), +# median-imputed and standardized using TRAIN statistics only. +# * `non_feature_columns`, the label column and its aliases are never features. generated_root: artifacts @@ -30,7 +37,13 @@ datasets: drop_labels: - crypto-ransomware # '-' is the dataset's documented missing placeholder, not an empty string. - missing_tokens: ["-", "?"] + missing_tokens: ["-", "?", "#DIV/0!", "aza"] + drop_duplicate_rows: true + # Feature columns that must be treated as categorical (one-hot). + categorical_columns: + - Protocol + - Service + - anomaly_alert # Columns that are identifiers / timestamps / network endpoints and must NOT # be used as model features without an explicit, documented decision. non_feature_columns: @@ -42,7 +55,7 @@ datasets: - Des_port notes: > Row-level missing placeholders ('-') and a small number of malformed tokens - (e.g. '?', 'aza') exist. Exact duplicate rows are present. See + (e.g. '?', 'aza', '#DIV/0!') exist. Exact duplicate rows are present. See docs/research/DATASET_INSPECTION.md. 5gnidd: @@ -57,9 +70,16 @@ datasets: benign_label: Benign drop_labels: [] missing_tokens: [""] - # First column is an unnamed row index. Seq / Offset are flow-ordering fields. + drop_duplicate_rows: true + categorical_columns: + - Proto + - sDSb + - dDSb + - Cause + - State + # The first column is an unnamed row index. Seq / Offset are flow-ordering fields. non_feature_columns: - - "" + - "Unnamed: 0" - Seq - Offset notes: > diff --git a/configs/experiments/experiment_protocol.yaml b/configs/experiments/experiment_protocol.yaml index d9a1365..a226fa8 100644 --- a/configs/experiments/experiment_protocol.yaml +++ b/configs/experiments/experiment_protocol.yaml @@ -1,6 +1,8 @@ # Frozen experiment protocol for the beta-weighted prototype aggregation study. -# Values below are the required research settings. Seed set is provisional until -# the verified baseline convention is reproduced (see docs/baseline/). +# +# ALL values below are now FROZEN for the baseline reproduction (2026-09-24 session). +# They must remain identical across every aggregation variant, beta candidate and +# seed so that comparisons are paired. See docs/research/EXPERIMENT_PROTOCOL.md. datasets: - xiiotid @@ -8,35 +10,38 @@ datasets: non_iid: partition: dirichlet + # Frozen Dirichlet algorithm: for each class, draw proportions ~ Dirichlet(alpha) + # over clients, then multinomial-allocate that class's train samples. See + # docs/research/EXPERIMENT_PROTOCOL.md section 2. + algorithm: per_class_dirichlet_multinomial alphas: [0.75, 0.50, 0.25] # required by the research specification num_clients: 10 # matches PROTEAN paper cross-silo setting aggregation: - # beta candidates under test (one beta is applied to all classes and clients) - betas: [0.00, 0.25, 0.50, 0.75, 1.00] - # Modes: - # original_protean : verified baseline semantics (NOT YET VERIFIED) - # support_weighted : w_ij = N_ij ** beta, normalized over S_j (proposed) - # equal_available : equal weights over S_j (methodological control, - # introduced only if the baseline participant rule differs) - primary_mode: support_weighted + # Baseline checkpoint: only the reproduced original-PROTEAN aggregation is + # implemented. Support-weighted beta candidates and the equal_available control + # are DEFERRED to the later beta/controls phase (see docs/research/EXPERIMENT_PROTOCOL.md). baseline_mode: original_protean - control_mode: equal_available training: rounds: 10 local_epochs: 3 - # mu is stated in the paper; lambda is NOT stated in the paper text (UNRESOLVED). - mu: 0.1 - lambda: null - batch_size: null # to be fixed after baseline reproduction - learning_rate: null # to be fixed after baseline reproduction + mu: 0.1 # model-alignment coefficient (paper, frozen) + lambda: 1.0 # prototype-alignment coefficient (PROJECT DECISION; paper gives no value) + batch_size: 256 # PROJECT DECISION; paper gives no value + learning_rate: 0.001 # PROJECT DECISION; paper gives no value + optimizer: adam # PROJECT DECISION; paper gives no value + model_aggregation: equal_1_over_m # paper Eq. (2)/(5.5) + prototype_computation: end_of_round_mean_embedding + prototype_alignment_form: per_sample_embedding_to_global_prototype # FedProto-style reading of Eq. (3) evaluation: # The paper uses 80/20 train/test and has no validation split. # This study ADDS a validation split used only for beta selection. - test_fraction: 0.20 - validation_fraction: null # finalize after baseline reproduction + test_fraction: 0.20 # paper + validation_fraction: 0.10 # PROJECT DECISION (fraction of the full cleaned dataset) + train_fraction: 0.70 # remainder + split_strategy: stratified_by_class selection_metric: rare_class_macro_f1 rare_class_rule: two_smallest_positive_attack_support report_metrics: @@ -45,12 +50,18 @@ evaluation: - overall_macro_f1 - accuracy - per_client_metrics + - per_class_precision_recall_f1 + - unseen_class_report seeds: - # Paper averaged over three random seeds. Exact seed values are UNRESOLVED; - # they must be fixed before any beta comparison and shared across all modes. - provisional: [0, 1, 2] - status: provisional + # The paper averages over three random seeds but names none, so the seed values + # are NOT source-confirmed. {0, 1, 2} is an explicit PROJECT DECISION chosen as + # the minimal three-seed set, and is identical across every baseline condition. + # Each seed deterministically controls the split, Dirichlet partition, model + # initialisation and batch order (see src/protean_research/utils/seeding.py). + frozen: [0, 1, 2] + rationale: minimal_three_seed_set_paper_reports_three_unnamed_runs + source_status: project_decision_not_source_confirmed artifacts: partition_manifest_dir: artifacts/partitions diff --git a/docs/baseline/BASELINE_IMPLEMENTATION.md b/docs/baseline/BASELINE_IMPLEMENTATION.md new file mode 100644 index 0000000..037d15f --- /dev/null +++ b/docs/baseline/BASELINE_IMPLEMENTATION.md @@ -0,0 +1,210 @@ +# Baseline Implementation + +Status: minimum reproducible baseline implemented and smoke-tested (2026-09-24). +Scope: the pipeline needed to run original-PROTEAN experiments. The proposed +beta-weighted aggregation and the `equal_available` control are **NOT implemented** +in this checkpoint; they are deferred to the later beta/controls phase. No beta +sweep was launched. + +Companion documents: + +- `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md` — evidence for baseline semantics. +- `docs/research/EXPERIMENT_PROTOCOL.md` — protocol and provenance tags. +- `docs/research/IMPLEMENTATION_PLAN.md` — architecture. +- `docs/handoff/SESSION_HANDOFF.md` — continuation point. + +## 0. Provenance legend + +Every decision below is classified as one of: + +- **CODE** — confirmed from original PROTEAN code. **None exist**: no official + implementation is public (`PROTEAN_BASELINE_INVESTIGATION.md` §2). +- **PAPER** — stated in the paper and/or the author's PhD thesis. +- **INFERRED** — not written verbatim but a defensible reading of the paper/thesis. +- **PROJECT DECISION** — unspecified by the paper; chosen here and reported as a + reproduction deviation. +- **UNRESOLVED** — still unknown. + +## 1. Implemented modules + +| Module | Responsibility | +|---|---| +| `data/loading.py` | spec resolution, read-only CSV read, label canonicalisation, duplicate drop, feature selection | +| `data/preprocessing.py` | train-fitted median imputation + standardisation, one-hot encoding, serialisable state | +| `data/splits.py` | deterministic stratified 70/10/20 split, leakage guard | +| `data/partition.py` | Dirichlet non-IID partition, `N_ij`, content hash, eval-split partition | +| `data/manifest.py` | manifest save/load, exact reproduction with hash verification | +| `data/characterization.py` | per-class and per-client non-IID statistics | +| `data/rare_unseen.py` | per-client rare/unseen class derivation | +| `data/pipeline.py` | end-to-end deterministic data wiring | +| `models/protean_cnn.py` | baseline CNN, parameter get/set/average, embedding extraction | +| `models/prototypes.py` | local prototypes, global prototype collection, nearest-prototype inference | +| `aggregation/prototype_aggregation.py` | baseline `original_protean` aggregation only | +| `federated/client.py` | local update (CE + prototype alignment + model alignment) | +| `federated/server.py` | model averaging (1/M), prototype aggregation, global inference | +| `federated/round.py` | round orchestration + per-round validation metrics | +| `evaluation/metrics.py` | accuracy, macro/weighted F1, per-class P/R/F1, rare/unseen metrics | +| `utils/seeding.py`, `utils/integrity.py` | deterministic seed derivation, hashing | +| `scripts/run_baseline.py` | one-condition baseline runner / smoke test | +| `scripts/generate_partitions.py` | generate + verify all `(dataset, alpha, seed)` manifests | + +## 2. Preprocessing decisions (all **PROJECT DECISION**) + +The paper specifies no preprocessing. Frozen recipe: + +1. Raw CSVs read read-only; paths resolve from config with env overrides. +2. Column names stripped; the empty 5G-NIDD header becomes `Unnamed: 0`. +3. Exact duplicate rows dropped (`keep="first"`): X-IIoTID 6,222; 5G-NIDD 1. +4. `crypto-ransomware` dropped from X-IIoTID (**PAPER**). +5. Labels canonicalised (whitespace collapse, whitespace around `_` removed), fixing + `Lateral _movement` -> `Lateral_movement`. +6. Feature columns exclude the label column, its aliases and configured non-feature + columns. +7. `categorical_columns` one-hot encoded with a `__MISSING__` category; all other + columns coerced to numeric, missing tokens -> NaN, median-imputed. +8. Imputation medians and standardisation statistics fitted on the **train split only**; + fitted state persisted and reused for validation/test. +9. Deterministic `float32` output matrix. + +No statistic from validation or test data enters preprocessing. + +## 3. Split decisions + +| Decision | Value | Provenance | +|---|---|---| +| Test fraction | 0.20 | PAPER (80/20) | +| Validation fraction | 0.10 | PROJECT DECISION (paper has no validation set) | +| Train fraction | 0.70 | PROJECT DECISION (remainder) | +| Stratification | per class | PROJECT DECISION (ensures rare classes in val/test) | +| Split seed | derived from `(dataset, "split", seed)` | PROJECT DECISION | + +## 4. Partition decisions + +| Decision | Value | Provenance | +|---|---|---| +| Algorithm | per-class Dirichlet + multinomial allocation | PROJECT DECISION (paper unspecified) | +| Partition source | training split only | PAPER | +| RNG seed | derived from `(dataset, "dirichlet", alpha, seed)` | PROJECT DECISION | +| Missing classes | `N_ij = 0`, no prototype | PAPER (only observed classes shared) | +| Persistence | JSON manifest + content hash, indices regenerated & verified | PROJECT DECISION | + +Observed non-IID behavior (seed 0): X-IIoTID `alpha=0.75` -> at most 1 missing class per +client; `alpha=0.25` -> up to 3. Qualitatively consistent with the paper setting. + +## 5. Model provenance + +| Element | Value | Provenance | +|---|---|---| +| Layer set | 2 Conv1D (64, 128) + ReLU + max-pool + dropout (0.2/0.5) + flatten + dense 128 + log-softmax | PAPER | +| Conv kernel / padding | 3 / 1 | PROJECT DECISION (paper silent) | +| Embedding `phi` | dense 128 output | PAPER (paper: "first eight of ten layers") | +| Prototype source | embedding | PAPER / INFERRED | + +## 6. Local training and loss provenance + +| Element | Value | Provenance | +|---|---|---| +| Objective form | CE + `lambda * prototype alignment` + `(mu/2) * model alignment` | PAPER Eq. (3) | +| `mu` | 0.1 | PAPER | +| `lambda` | 1.0 | PROJECT DECISION (paper gives no value) | +| Prototype alignment form | per-sample `\|\|phi(x) - Cbar_y\|\|^2` for classes with a global prototype | INFERRED (FedProto-style reading; paper writes class-mean to class-mean) | +| Model alignment target | global model of the round | INFERRED | +| Optimiser / lr / batch size | Adam / 1e-3 / 256 | PROJECT DECISION | +| Rounds / local epochs | 10 / 3 | PAPER | +| Local prototype | end-of-round mean embedding per observed class | PAPER Eq. (1) | +| Round 0 prototypes | none; prototype term skipped; first prototypes from round-0 aggregation | INFERRED / PROJECT DECISION | + +## 7. Aggregation provenance — `original_protean` status + +**`original_protean` is implemented, but its participant rule is a REPRODUCTION +ASSUMPTION, not verified original behavior.** + +- **PAPER**: Eq. (2)/(5.5) writes `Cbar_j = (1/M) * sum_{i} C_ij` and model averaging + `omega = (1/M) * sum_i omega_i`. +- **PAPER**: the thesis states prototypes are shared only for locally observed classes. +- **UNRESOLVED**: the paper never defines how a class missing on a client (`N_ij = 0`) is + handled, nor whether the denominator is `M` or the number of contributors. +- **REPRODUCTION ASSUMPTION (PROJECT DECISION)**: the server receives prototypes only from + `S_j = { i | N_ij > 0 and a valid C_ij exists }` and averages them with **equal weights + `1/|S_j|`**. This follows the thesis sentence above but is not source-confirmed. + +Consequences: + +- `original_protean` is therefore a *documented interpretation* of the baseline. + It must be validated in the baseline-verification phase before any between-method + comparison. +- The model-aggregation rule (equal `1/M`) is implemented separately and is closer to the + paper text. + +Whether a literal `1/M` reading with omitted classes coincides with this assumption is an +open question for the verification phase; the `equal_available` control that could +distinguish them is deliberately deferred. + +## 8. Inference provenance + +| Element | Value | Provenance | +|---|---|---| +| Rule | nearest global prototype by squared L2 | PAPER (thesis Eqs. 5.11–5.13) | +| Classes considered | those with a global prototype | INFERRED | + +## 9. Seeds + +- Exact values: **`{0, 1, 2}`**, frozen in `configs/experiments/experiment_protocol.yaml`. +- Rationale: the paper averages over three random seeds but names none, so the values are + **not source-confirmed**. `{0, 1, 2}` is the minimal three-seed set and is an explicit + **PROJECT DECISION**. +- Each seed deterministically controls the split, Dirichlet partition, model + initialisation and batch order, via + `protean_research.utils.seeding.derive_seed` (never Python `hash()`). +- The seed set is identical across all baseline conditions. + +## 10. Manifests + +There are **18** partition manifests under `artifacts/partitions/`, from +`2 datasets × 3 alphas × 3 seeds = 18` combinations of `(dataset, alpha, seed)`. Each is +content-hashed, reload-verified, and intended to be reused by every later aggregation +variant so that comparisons stay paired. Full sample-index arrays are regenerated +deterministically from the stored recipe/RNG seed and checked against the stored `N_ij` +and hash, so manifests stay small. + +## 11. Tests + +`python -m pytest -q` -> **39 tests** (see run result in the handoff). Coverage includes +dataset config resolution, deterministic label mapping, deterministic/train-only +preprocessing, deterministic stratified split, deterministic Dirichlet partition, +manifest reload identity and corruption detection, rare/unseen semantics, local prototype +means, baseline aggregation (including that `support_weighted` is not implemented), metrics +on known fixtures, and a small end-to-end federated smoke test. + +## 12. Smoke test + +``` +.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ + --max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1 +``` + +Runs on both datasets through the full path (load -> clean -> split -> preprocess -> +partition -> local training -> prototypes -> baseline aggregation -> global +redistribution -> evaluation -> artifacts). Smoke runs write only to ignored directories +(`artifacts/processed/smoke/`, `results/smoke/`) and are **not** thesis results. The +smoke test does not invoke the proposed method. + +## 13. Structured outputs + +Per run under `results//`: `metadata.json`, `history.json`, +`test_global_metrics.json`, `test_confusion_matrix.json`, `test_per_class.csv`, +`per_client.csv`, `train_support_Nij.csv`, `characterization.csv`, `rare_unseen.csv`. +Preprocessor state -> `artifacts/processed/`; round-0 prototype snapshot -> +`artifacts/processed/smoke/analysis/` (smoke) or `artifacts/analysis/` (full run); +manifests -> `artifacts/partitions/`. + +## 14. Remaining assumptions and unresolved issues + +1. **Participant rule** (`S_j`, equal weights) is a reproduction assumption (see §7). +2. `lambda = 1.0`, optimiser and schedule are unverified against the paper. +3. Prototype-alignment form and round-0 handling are inferred. +4. Per-client evaluation partitioning (val/test assigned to clients via Dirichlet) is a + project decision; the paper does not describe it. +5. No full-scale run has been timed; compute feasibility of a real sweep is unmeasured. +6. Whether the literal `1/M` reading differs from the `S_j` assumption is open and must + be tested with a deliberate control in the verification phase. diff --git a/docs/handoff/SESSION_HANDOFF.md b/docs/handoff/SESSION_HANDOFF.md index 7c7a24d..43d882f 100644 --- a/docs/handoff/SESSION_HANDOFF.md +++ b/docs/handoff/SESSION_HANDOFF.md @@ -1,11 +1,11 @@ # Session Handoff This is the canonical continuation point for the next agent session. It is written to be -readable without any prior chat history, terminal scrollback, or git history. +readable without prior chat history, terminal scrollback, or git history. -Session date: 2026-09-24 -Session type: research study, baseline investigation, architecture planning, dataset -inspection, minimal bootstrap. **No training was run and no experimental sweep was launched.** +Last updated: 2026-09-24 (Session 2, baseline-only checkpoint). +Session 2 type: minimum reproducible baseline implementation + smoke test, then cleanup so +the checkpoint contains **baseline only**. **No full-scale training and no beta sweep.** --- @@ -13,7 +13,7 @@ inspection, minimal bootstrap. **No training was run and no experimental sweep w Build the code and experiment infrastructure for a thesis study that modifies **only the server-side prototype aggregation** of PROTEAN (Chennoufi et al., 2025), a federated -intrusion-detection framework. The proposed mechanism weights each contributing client's +intrusion-detection framework. The planned mechanism weights each contributing client's class prototype by its class sample support: ``` @@ -22,325 +22,260 @@ S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } beta in {0.0, 0.25, 0.5, 0.75, 1.0} ``` -Goal: characterise when support weighting helps or hurts rare attack detection under -non-IID data, selecting beta on a validation set. `beta = 0` is a control, **not** assumed -identical to the original baseline. +**This checkpoint contains the baseline only.** The proposed beta-weighted aggregation and +the `equal_available` control are intentionally NOT implemented yet; they are deferred to +the later beta/controls phase, which must not start until the baseline is verified. ## 2. Authoritative source -- `planningawal/PROTEAN_Revisi_Lurus.md` — the authoritative research specification. -- The entire `planningawal/` directory is **immutable / read-only**. Do not edit, create, - rename, move, delete, reformat, overwrite, or generate reports inside it. It must not - appear in any "files modified" list. -- Verified unchanged this session: SHA-256 - `9ba1d7beb94354db0c4b588199fb1e2823aec8a23be1625ec09a756507fbcb89` (12,050 bytes). +- `planningawal/PROTEAN_Revisi_Lurus.md` — authoritative research specification. +- `planningawal/` is **immutable / read-only**. Verified unchanged: + SHA-256 `9ba1d7beb94354db0c4b588199fb1e2823aec8a23be1625ec09a756507fbcb89` + (only file in `planningawal/`). ## 3. Dataset locations (read-only source data) -- `~/datasets/X-IIoTID/X-IIoTID-dataset.csv` -- `~/datasets/5G-NIDD/5G-NIDD.csv` +- `~/datasets/X-IIoTID/X-IIoTID-dataset.csv` — SHA-256 + `7b9290057ee42e784da3c0d84b781815502c9205c74175c96374e71a5ffd98a0` (mtime 2021-10-07) +- `~/datasets/5G-NIDD/5G-NIDD.csv` — SHA-256 + `fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59` (mtime 2023-02-18) -Do not move, rename, overwrite, modify, or copy the full raw files into the repository. Use -configurable external paths (`configs/datasets/datasets.yaml`, with env overrides). +Re-verified unchanged. Paths resolve from `configs/datasets/datasets.yaml` with env +overrides (`PROTEAN_XIIOTID_RAW_PATH`, `PROTEAN_5GNIDD_RAW_PATH`). Raw files are never +modified or copied into the repo. -Recorded SHA-256: +## 4. Current repository state -- X-IIoTID: `7b9290057ee42e784da3c0d84b781815502c9205c74175c96374e71a5ffd98a0` -- 5G-NIDD: `fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59` - -## 4. Current repository tree +Baseline code: ``` -perdalam-protean/ -├── README.md -├── pyproject.toml -├── .gitignore -├── planningawal/ # IMMUTABLE (read-only) -│ └── PROTEAN_Revisi_Lurus.md -├── configs/ -│ ├── datasets/datasets.yaml -│ ├── experiments/experiment_protocol.yaml -│ └── models/protean_cnn.yaml -├── docs/ -│ ├── research/ -│ │ ├── RESEARCH_SPEC_INTERPRETATION.md -│ │ ├── DATASET_INSPECTION.md -│ │ ├── IMPLEMENTATION_PLAN.md -│ │ └── EXPERIMENT_PROTOCOL.md -│ ├── baseline/ -│ │ └── PROTEAN_BASELINE_INVESTIGATION.md -│ └── handoff/ -│ └── SESSION_HANDOFF.md -├── src/protean_research/ -│ ├── __init__.py -│ ├── aggregation/ -│ │ ├── __init__.py -│ │ └── prototype_aggregation.py -│ └── utils/ -│ ├── __init__.py -│ └── config.py -├── tests/ -│ └── test_prototype_aggregation.py -├── scripts/ # .gitkeep only (tracked) -├── artifacts/ -│ ├── partitions/ # tracked: deterministic manifests (.gitkeep now) -│ ├── analysis/ # tracked: small analysis outputs (.gitkeep now) -│ ├── processed/ # ignored (regenerable) -│ └── checkpoints/ # ignored (regenerable) -└── results/ # ignored (regenerable) +src/protean_research/ +├── data/ loading, preprocessing, splits, partition, manifest, +│ characterization, rare_unseen, pipeline +├── models/ protean_cnn, prototypes +├── aggregation/ prototype_aggregation (baseline original_protean ONLY) +├── federated/ client, server, round +├── evaluation/ metrics +└── utils/ config, seeding, integrity +scripts/ run_baseline.py, generate_partitions.py +tests/ 34 baseline-focused tests +artifacts/partitions/ 18 full-data manifests (tracked, small, content-hashed) +artifacts/analysis/ 18 characterization + 18 rare_unseen CSVs (tracked, small) +artifacts/processed/ ignored: preprocessor state + smoke partitions/snapshots +results/ ignored: smoke run outputs ``` -`.gitignore` deliberately keeps `artifacts/partitions/` and `artifacts/analysis/` -version-controlled (partition manifests are deterministic, content-hashed, and required for -paired comparisons), while ignoring `artifacts/processed/`, `artifacts/checkpoints/`, -`results/`, `logs/`, and `*.log`. `scripts/` is preserved with a `.gitkeep` placeholder. -`planningawal/` is never ignored. +Environment: `.venv/` exists (gitignored) with numpy 2.5.3, pandas 3.0.6, +torch 2.14.0+cpu, pyyaml 6.0.3, pytest 9.1.1. Use `.venv/bin/python`. -Note: the repository was **not** empty at session start. A partial scaffold already existed -(`configs/*`, `src/protean_research/aggregation/*`, `src/protean_research/utils/config.py`, -`__init__.py` files, and empty `docs/`, `scripts/`, `tests/`, `artifacts/`, `results/` -directories). `docs/` was empty and there was no handoff. That scaffold was kept and verified; -it is not the product of this session. +## 5. What was completed this session -## 5. Work completed this session +1. Froze every previously-PROVISIONAL protocol value in + `configs/experiments/experiment_protocol.yaml` and `docs/research/EXPERIMENT_PROTOCOL.md`. +2. Dataset config: explicit `categorical_columns`, `drop_duplicate_rows`, missing tokens, + fixed 5G-NIDD `Unnamed: 0` (quoted). +3. Data foundation (`data/`): read-only loading, deterministic canonicalisation, leak-free + train-fitted preprocessing, deterministic stratified 70/10/20 split, Dirichlet partition + with content-hashed manifests and `N_ij`, characterization, rare/unseen metadata. +4. Model (`models/`): baseline CNN, parameter averaging, embedding extraction, local + prototypes, global prototype collection, nearest-prototype inference. +5. Baseline aggregation (`aggregation/`): `original_protean` only. The proposed + `support_weighted` and the `equal_available` control were **removed** so the checkpoint + is baseline only; an unavailable-mode guard raises `ValueError`. +6. Federated loop (`federated/`): local update (CE + prototype alignment + model + alignment), server model/prototype aggregation, round orchestration, round-0 snapshot. +7. Evaluation (`evaluation/`): accuracy, macro/weighted F1, per-class P/R/F1, rare/unseen. +8. Scripts: `generate_partitions.py` (18 manifests, full data, reload-verified); + `run_baseline.py` (one baseline condition; no beta/mode options). +9. Tests: 34 baseline-only tests, all passing. +10. Smoke test: baseline-only on both datasets. +11. Docs: `docs/baseline/BASELINE_IMPLEMENTATION.md` rewritten with provenance tags. -1. Read the full specification and produced a tagged interpretation - (`docs/research/RESEARCH_SPEC_INTERPRETATION.md`). -2. Investigated authoritative PROTEAN sources (paper, ESORICS/Springer, HAL, author PhD - thesis) and searched for official code. -3. Established baseline aggregation semantics question-by-question with evidence tags - (`docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). -4. Inspected both raw datasets in full (read-only) and recorded shapes, label distributions, - missingness, duplicates, identifiers, and mismatches - (`docs/research/DATASET_INSPECTION.md`). -5. Verified the pre-existing config/path-resolution scaffold and aggregation module by running - them. -6. Wrote the implementation architecture (`docs/research/IMPLEMENTATION_PLAN.md`) and the - experiment protocol (`docs/research/EXPERIMENT_PROTOCOL.md`). -7. Added a minimal bootstrap: `pyproject.toml`, `README.md`, and dependency-free unit tests - for the aggregation math (`tests/test_prototype_aggregation.py`, 9 tests, all passing). -8. Recorded dataset and source hashes for integrity verification. -9. Wrote this handoff. +## 6. Baseline decisions and provenance -## 6. Evidence and findings +Full detail in `docs/baseline/BASELINE_IMPLEMENTATION.md`. Summary: -### 6.1 Official PROTEAN source status +- **No official PROTEAN code exists**, so no decision is source-confirmed by code. +- **PAPER**: architecture, `mu = 0.1`, 10 rounds / 3 local epochs, 80/20 test, equal `1/M` + model averaging, end-of-round mean-embedding prototypes, nearest-prototype inference. +- **REPRODUCTION ASSUMPTION (project decision)**: `original_protean` participant rule is + `S_j` with equal `1/|S_j|` weights. The paper writes `1/M` but is ambiguous for classes + missing on a client; this assumption follows the thesis sentence that prototypes are + shared only for locally observed classes. It is **not** treated as verified. +- **PROJECT DECISION**: `lambda = 1.0`; Adam `lr = 1e-3`; batch size 256; kernel size 3; + stratified 70/10/20 split with an added validation set; seeds `{0,1,2}`; + preprocessing recipe; per-class Dirichlet allocation; per-client eval partitioning. +- **INFERRED**: per-sample prototype alignment (FedProto-style reading of Eq. 3); round-0 + handling (no prototypes, prototype term skipped). +- **UNRESOLVED**: whether a literal `1/M` missing-class reading differs from `S_j`. -- **No official code exists publicly (as of 2026-09-24).** -- Author GitHub `sarachn` (Sara Chennoufi) has only `FATE-LLM` (fork) and `SecLLMHolmes` — - neither is PROTEAN. -- GitHub repository search `PROTEAN federated prototype` → 0 results; `grep.app` → none. -- The unrelated GitHub user `Chennoufi` has only a `septembre` repo — do not chase it. -- No code/artifact/supplementary link in the arXiv HTML, HAL record, or thesis text. -- **Consequence: the baseline must be reproduced from the paper/thesis.** +## 7. Seeds (explicit) -Primary sources: +- Exact values: **`{0, 1, 2}`**. +- Rationale: the paper averages over three random seeds but names none, so the values are + **not source-confirmed**. `{0, 1, 2}` is the minimal three-seed set and is an explicit + **project decision**. +- Each seed deterministically controls the split, Dirichlet partition, model + initialisation and batch order, via `protean_research.utils.seeding.derive_seed`. +- The set is identical across all baseline conditions. -- arXiv:2507.05524v1 (7 Jul 2025), ESORICS 2025 pp. 103–125, - DOI `10.1007/978-3-032-07884-1_6`. -- HAL `hal-05379203`. -- Author PhD thesis `tel-05490927` (195 pages, 2025) — restates PROTEAN with extra detail. +## 8. Manifests (18) -### 6.2 Baseline semantics discovered +`artifacts/partitions/` holds **18** manifests from `2 datasets × 3 alphas × 3 seeds`. +They are content-hashed, reload-verified, and reused by later variants for pairing. Sample +indices are regenerated deterministically from the stored recipe/RNG seed and checked +against the stored `N_ij` and hash, keeping manifests small. -Confirmed from the paper/thesis: +## 9. Verification performed this session -- Local prototype (Eq. 1 / 5.4): mean embedding over the client's class-`j` samples. -- Server aggregation (Eq. 2 / 5.5): `omega = (1/M) sum_i omega_i`, - `Cbar_j = (1/M) sum_i C_ij` (equal, no `N_ij`). -- Local loss (Eq. 3 / 5.6): cross-entropy + `lambda * ||C_ij - Cbar_j||^2` + - `(mu/2) * ||omega_i - omega^{t-1}||^2`, with `mu = 0.1`. -- Inference: nearest global prototype by squared L2 (thesis Eqs. 5.11–5.13). -- Model: 2 Conv1D (64, 128 filters) + ReLU + maxpool + dropout (0.2, 0.5), flatten, - dense 128, log-softmax head. -- Setup: 10 clients, 80/20 train/test, 10 rounds, 3 local epochs, - `alpha in {0.75, 0.5, 0.25}`. -- Thesis (Sec. 5.3.6): "prototypes are shared only for locally observed classes". +- `python -m pytest -q` -> **34 passed**. +- Full partition generation on both datasets: 18 manifests, each reload-verified. +- Non-IID check (seed 0): X-IIoTID `alpha=0.75` max 1 missing class/client, `alpha=0.25` + up to 3; 5G-NIDD similar. Qualitatively consistent with the paper setting. +- Baseline-only smoke on X-IIoTID (3000 rows, 4 clients, 2 rounds, 1 epoch): full path + succeeds, ~4.5 s. +- Baseline-only smoke on 5G-NIDD (4000 rows): succeeds. +- No beta/support_weighted code is invoked or present. +- Integrity: `planningawal/` and both raw datasets retain their recorded SHA-256 values. -### 6.3 Missing-class findings (critical) +## 10. Files added this session -- Equation (2)/5.5 sums over all `M` participants, but the paper never states how a missing - class prototype is handled. The thesis implies absent classes are not shared, but the - server denominator (`M` vs number of contributors) is undefined. -- Therefore `beta = 0` over `S_j` is **not** automatically equal to the original baseline; - equality requires the same participant rule. This is the central baseline ambiguity. -- `original_protean` is deliberately **not implemented** in - `src/protean_research/aggregation/prototype_aggregation.py`; it raises - `NotImplementedError` with an explanatory message. -- Additional unknowns: round-0 global-prototype initialisation; how the alignment loss sums - over `j = 1..K` when a client has `N_ij = 0`; per-batch vs end-of-round prototypes; - whether model aggregation is equal `1/M` or FedAvg sample-weighted (the thesis intro says - FedAvg is a weighted average, contradicting Eq. 5.5); numeric value of `lambda`; seed values; - feature preprocessing. +Source: `src/protean_research/data/{__init__,loading,preprocessing,splits,partition,manifest, +characterization,rare_unseen,pipeline}.py`, +`src/protean_research/models/{__init__,protean_cnn,prototypes}.py`, +`src/protean_research/federated/{__init__,client,server,round}.py`, +`src/protean_research/evaluation/{__init__,metrics}.py`, +`src/protean_research/utils/{seeding,integrity}.py`. -### 6.4 Dataset findings +Scripts: `scripts/run_baseline.py`, `scripts/generate_partitions.py`. -- X-IIoTID: 820,834 × 68; labels `class1` (19), `class2` (10 general), `class3` (binary). - `class2` benign = `Normal`. After dropping `crypto-ransomware`: 9 classes. `'-'` - placeholders in 42 columns; 6,222 duplicate rows; label typo `Lateral _movement`. - Identifiers: `Date`, `Timestamp`, `Scr_IP`, `Scr_port`, `Des_IP`, `Des_port`. -- 5G-NIDD: 1,215,890 × 52; `Attack Type` (9 values, benign = `Benign`), `Label` binary, - `Attack Tool` (6). 8 attack classes + Benign = 9. Heavy structural missingness (e.g. `dVid` - 99.83%, `dTos` 77.56%); 1 duplicate row. Identifiers: empty-header index, `Seq`, `Offset`. -- Both map to PROTEAN's 9-class setup, but PROTEAN's exact preprocessing is unspecified and - must be frozen here. +Tests: `tests/test_data_pipeline.py`, `tests/test_partition.py`, `tests/test_prototypes.py`, +`tests/test_baseline_aggregation.py`, `tests/test_metrics.py`, `tests/test_federated_smoke.py`. -## 7. Decisions made +Docs: `docs/baseline/BASELINE_IMPLEMENTATION.md`. -1. **Treat `planningawal/` as immutable** and keep all outputs elsewhere. -2. **Do not implement `original_protean` until its semantics are verified.** Encode only the - unambiguous modes (`support_weighted`, `equal_available`). Rationale: avoid silently - assuming baseline behaviour. -3. **Keep `equal_available` as a dormant control**, introduced only if baseline investigation - shows the participant rule differs from `S_j` (per the specification). -4. **Centralise dataset paths** in `configs/datasets/datasets.yaml` with env overrides; never - hardcode in source. -5. **Separate raw and generated data**; generated outputs under `artifacts/`. -6. **Use `class2` for X-IIoTID and `Attack Type` for 5G-NIDD** as the multi-class labels, drop - `crypto-ransomware`, treat `Normal`/`Benign` as benign. Matches the paper. -7. **Keep the pre-existing scaffold** (configs + aggregation + utils) and extend it minimally - rather than restructuring; no evidence justified a different layout. -8. **Add dependency-free unit tests** for aggregation invariants now; defer data/federated - tests until those modules exist. -9. **Do not initialize git during the planning session** (not requested at the time). A - follow-up checkpoint task then initialized Git and added remote `origin`; see Section 11. -10. **Mark seeds, `lambda`, validation fraction, Dirichlet allocation, and preprocessing as - PROVISIONAL**; they must be frozen before any sweep. +Artifacts (tracked, small): `artifacts/partitions/*.json` (18), +`artifacts/analysis/*.csv` (36). -## 8. Unresolved questions +## 11. Files modified this session -1. Baseline missing-class rule and prototype-aggregation denominator (`M` vs `|S_j|`). -2. Round-0 initialisation of global prototypes for all `K` classes. -3. `lambda` value (paper gives `mu = 0.1` only). -4. Whether model-parameter aggregation is equal or FedAvg sample-weighted. -5. How the prototype alignment loss treats `N_ij = 0` classes. -6. Per-batch vs end-of-round local prototype computation. -7. Validation split fraction and procedure (the paper has no validation set). -8. Exact Dirichlet allocation algorithm (affects realised `N_ij`). -9. Exact seed values and whether metrics are averaged per client then per seed. -10. Feature preprocessing: encoding, scaling, `'-'`/NaN handling, duplicate-row policy. -11. ~~Whether to initialize git to protect the immutable source.~~ Resolved: Git was - initialized in the follow-up checkpoint task (see Section 11). -12. Compute budget: no timing measured; a profiling smoke test is required before any sweep. - -## 9. Files created this session - -- `docs/research/RESEARCH_SPEC_INTERPRETATION.md` -- `docs/research/DATASET_INSPECTION.md` -- `docs/research/IMPLEMENTATION_PLAN.md` +- `configs/datasets/datasets.yaml` +- `configs/experiments/experiment_protocol.yaml` - `docs/research/EXPERIMENT_PROTOCOL.md` -- `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md` -- `docs/handoff/SESSION_HANDOFF.md` +- `docs/research/IMPLEMENTATION_PLAN.md` - `README.md` -- `pyproject.toml` -- `.gitignore` -- `tests/test_prototype_aggregation.py` +- `src/protean_research/aggregation/__init__.py` +- `src/protean_research/aggregation/prototype_aggregation.py` +- `src/protean_research/models/prototypes.py` +- `src/protean_research/federated/server.py` +- `src/protean_research/federated/round.py` +- `scripts/run_baseline.py` +- tests (baseline-only aggregation suite; beta tests removed) -## 10. Files modified this session +`planningawal/` was not touched. -- None. No existing file was edited. `planningawal/` was not touched (and must not be). +## 12. Removed from this checkpoint (deferred to beta/controls phase) -Pre-existing files (present at session start, verified but not modified): -`configs/datasets/datasets.yaml`, `configs/experiments/experiment_protocol.yaml`, -`configs/models/protean_cnn.yaml`, `src/protean_research/__init__.py`, -`src/protean_research/aggregation/__init__.py`, -`src/protean_research/aggregation/prototype_aggregation.py`, -`src/protean_research/utils/__init__.py`, `src/protean_research/utils/config.py`. +- `support_weighted` aggregation code and `beta` parameters. +- `equal_available` mode (redundant with the baseline `S_j` rule; only needed if a later + investigation shows the baseline participant rule differs). +- `--mode` / `--beta` CLI options in `run_baseline.py`. +- Beta-specific tests (`beta = 0/1`, single-contributor/equal-support beta invariance, + normalised beta weights, the `original_protean == support_weighted(beta=0)` equivalence + test). The former `tests/test_prototype_aggregation.py` was removed and folded into a + baseline-only `tests/test_baseline_aggregation.py`. +- Any claim that `original_protean == support_weighted(beta=0)` demonstrates equivalence + with original PROTEAN. That was at most an internal implementation-equivalence check and + is no longer present in code or docs. -## 11. Git status +## 13. Git status -The planning session ran with **no Git repository**. A follow-up checkpoint task then -initialized Git: +Not committed (no commit was requested). `git status --short` shows modified tracked files +(configs, docs, aggregation package, federated modules, README, run script) plus untracked +source/tests/scripts/docs, the 18 partition manifests and 36 analysis CSVs. `AGENTS.md` was +already untracked before this session. `.venv/`, `artifacts/processed/`, `results/`, +`logs/`, and `*.log` are ignored. No checkpoints or raw datasets are tracked. -- Branch: `main` -- Remote `origin`: `git@github.com:willtanoe/prototype-agregation.git` -- Initial checkpoint commit: `chore: checkpoint PROTEAN research planning and architecture` - (exact hash reported in the checkpoint task's terminal summary; deliberately not embedded - here to avoid a recursive "hash inside the commit" loop). -- `.gitkeep` placeholders were added for `scripts/`, `artifacts/partitions/`, and - `artifacts/analysis/`. `artifacts/processed/`, `artifacts/checkpoints/`, `results/`, - `logs/`, and `*.log` are ignored. -- `planningawal/PROTEAN_Revisi_Lurus.md` is tracked exactly as-is and was not modified. -- Raw datasets are outside the repository and are not tracked. +## 14. Unresolved issues / blockers -Because this is the initial commit, `planningawal/` was verified with SHA-256 hashing and -directory listing rather than `git diff`. +1. **Baseline is implemented but not yet VALIDATED.** The `original_protean` participant + rule is a reproduction assumption, not verified behavior. +2. `lambda`, optimiser/schedule, and round-0 handling are unverified. +3. No full-scale run has been timed; compute feasibility of a real sweep is unmeasured + (X-IIoTID ~570k train rows, 5G-NIDD ~851k; CPU-only). +4. Per-client evaluation partitioning is a project decision. +5. Prototype-alignment form (per-sample vs class-mean) is a documented inference. +6. Whether the literal `1/M` missing-class reading differs from the `S_j` assumption is + open; a deliberate control is needed in the verification phase. -## 12. Verification performed +## 15. Exact next task -- Read the full specification (118 lines). -- Read the full paper HTML and extracted the author PhD thesis text (195 pages). -- GitHub API searches (author account, repositories), DuckDuckGo, `grep.app`. -- Full pandas load of both datasets; recorded shape, dtypes, label counts, missingness, - duplicates, identifier columns. -- SHA-256 of both datasets and of `planningawal/PROTEAN_Revisi_Lurus.md`. -- Ran the pre-existing aggregation module and config resolver (modes, weights, path - resolution, `original_protean` raises `NotImplementedError`). -- Ran `python -m pytest -q` in a temporary venv: **9 passed**. -- Confirmed no training or sweep was launched. -- Final check: `planningawal/` contains only `PROTEAN_Revisi_Lurus.md` with the same hash; - raw datasets retain the same hashes and mtimes. +**Baseline validation and reproduction verification — NOT beta weighting.** -Checkpoint task (follow-up): +Next session should: -- `git rev-parse --is-inside-work-tree` before init: not a repository. -- Searched the whole tree for `*.csv`, `*.parquet`, `*.feather`, `*.pkl`, `*.pickle`, model - checkpoints: none present; largest file is the 12 KB spec. -- Scanned tracked text for secret patterns (`API_KEY`, `SECRET`, `TOKEN`, `PASSWORD`, - `PRIVATE KEY`, `BEGIN OPENSSH`, `BEGIN RSA`, `sk-`, `ghp_`, `github_pat_`): none found. -- `find planningawal -maxdepth 2 -type f` → only `PROTEAN_Revisi_Lurus.md`. -- Re-ran `pytest`: 9 passed. +1. Run one full-data baseline condition (e.g. X-IIoTID, `alpha=0.5`, `seed=0`, + `original_protean`) with measured wall-clock and memory; record it. +2. Inspect the run: per-class precision/recall/F1, per-client rare metrics, confusion + matrix. Check plausibility and missing-class-client behavior. +3. Stress-test the `S_j` reproduction assumption: compare against a literal `1/M` + omitted-class reading to quantify the effect of the assumption (introduce the control + only for this investigation, clearly marked). +4. Add a deterministic end-to-end baseline regression test pinning a small result. +5. Only after the baseline is demonstrably verified and recorded, begin the paired beta + comparison on a single `(dataset, alpha, seed)` before any grid. -## 13. Exact next task +Do **not** launch the full grid, and do not change the architecture, loss coefficients, or +inference rule. -**Implement the deterministic data foundation, and freeze the remaining protocol choices -before coding.** +## 16. Important commands -Concretely, the next session should: +```bash +# Tests +.venv/bin/python -m pytest -q -1. In `docs/research/EXPERIMENT_PROTOCOL.md`, replace the PROVISIONAL entries with explicit - frozen decisions and rationale for: preprocessing recipe, duplicate-row policy, - `'-'`/NaN handling, the Dirichlet allocation algorithm, the train/validation/test split - procedure and fraction, the seed set, `lambda`, batch size, and learning rate. These are - project decisions because the paper does not specify them; label them as reproduction - deviations. -2. Create `src/protean_research/data/` with `loading.py`, `preprocessing.py`, `splits.py`, - `partition.py`, `manifest.py`, `characterization.py`, `rare_unseen.py` implementing the - flow in `docs/research/IMPLEMENTATION_PLAN.md` Section 3. -3. Persist one partition manifest per `(dataset, alpha, seed)` under `artifacts/partitions/` - with a content hash, and derive rare/unseen metadata from the training split. -4. Add tests for determinism of partition generation (strategy item 8) and identical manifest - reload (item 9), plus the non-IID characterization fields. +# Generate / verify all partition manifests (full data, ~30 s) +.venv/bin/python scripts/generate_partitions.py -Do **not** start model training or a beta sweep. The baseline missing-class rule is still -unresolved, so `original_protean` remains unimplemented. +# Baseline smoke on both datasets (writes only to ignored dirs) +.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ + --max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1 +.venv/bin/python scripts/run_baseline.py --dataset 5gnidd --alpha 0.25 --seed 1 \ + --max-rows 4000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1 -## 14. Recommended next-session prompt +# Full-data baseline for one condition (next session; expect long CPU runtime) +.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ + --clients 10 --rounds 10 --local-epochs 3 +``` + +## 17. Recommended next-session prompt ``` -You are continuing the perdalam-protean research repository. +You are continuing the perdalam-protean research repository (Session 3). -FIRST, read these two files completely: +FIRST, read completely: 1. docs/handoff/SESSION_HANDOFF.md - 2. planningawal/PROTEAN_Revisi_Lurus.md + 2. docs/baseline/BASELINE_IMPLEMENTATION.md + 3. docs/research/EXPERIMENT_PROTOCOL.md -Then read, as needed: - - docs/research/IMPLEMENTATION_PLAN.md - - docs/research/EXPERIMENT_PROTOCOL.md - - docs/research/DATASET_INSPECTION.md +Then read as needed: + - planningawal/PROTEAN_Revisi_Lurus.md - docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md + - docs/research/IMPLEMENTATION_PLAN.md Critical rules: - planningawal/ is immutable. Do not edit/create/move/rename/delete anything inside it. - ~/datasets/X-IIoTID/X-IIoTID-dataset.csv and ~/datasets/5G-NIDD/5G-NIDD.csv are read-only external source data. Do not modify or copy them into the repo. - - Do NOT run full training or an experimental sweep. + - This checkpoint is BASELINE ONLY. Do NOT add the proposed support_weighted/beta code + or the equal_available control until the baseline is verified. + - Do NOT launch the full alpha x seed x dataset x beta sweep. -Your task is Section 13 of the handoff: first freeze the PROVISIONAL protocol decisions in -docs/research/EXPERIMENT_PROTOCOL.md, then implement the deterministic data foundation in -src/protean_research/data/ with partition-manifest persistence, rare/unseen metadata, and -tests for partition determinism and manifest reload. +Your task is Section 15 of the handoff: BASELINE VALIDATION AND REPRODUCTION +VERIFICATION. Run one full-data baseline condition with measured runtime, inspect its +per-class/per-client behavior, stress-test the S_j participant-rule assumption, and add a +deterministic end-to-end baseline regression test. Record everything in +docs/baseline/BASELINE_IMPLEMENTATION.md and update docs/handoff/SESSION_HANDOFF.md. -Start by reading the handoff, then continue from the documented state. Update -docs/handoff/SESSION_HANDOFF.md at the end of your session. +Use .venv/bin/python for all Python commands. ``` diff --git a/docs/research/EXPERIMENT_PROTOCOL.md b/docs/research/EXPERIMENT_PROTOCOL.md index 1f35dfe..830fd6d 100644 --- a/docs/research/EXPERIMENT_PROTOCOL.md +++ b/docs/research/EXPERIMENT_PROTOCOL.md @@ -1,10 +1,15 @@ # Experiment Protocol -Status: frozen design (to be executed in later sessions) -Machine-readable counterpart: `configs/experiments/experiment_protocol.yaml` +Status: **FROZEN** reproduction protocol (2026-09-24). +Machine-readable counterpart: `configs/experiments/experiment_protocol.yaml`. -Values marked **PROVISIONAL** require confirmation before any scientific sweep. Values -marked **FROZEN** must remain identical across every aggregation variant. +Values marked **FROZEN** must remain identical across every aggregation variant, beta +candidate and seed. Values marked **PROJECT DECISION** are choices this project had to make +because the paper/thesis leave them unspecified; they are reproduction deviations and must +be reported as such. No value is PROVISIONAL any more. + +See `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md` for the evidence behind each choice and +`docs/baseline/BASELINE_IMPLEMENTATION.md` for how the protocol is implemented. --- @@ -24,59 +29,79 @@ Details and mismatches: `docs/research/DATASET_INSPECTION.md`. FROZEN: symmetric Dirichlet over classes, `num_clients = 10`, `alpha ∈ {0.75, 0.50, 0.25}`. -- Partitions are generated from the **training split only**. -- Class proportions are drawn per client from `Dirichlet(alpha)` and used to allocate - training samples per class to clients. -- The exact Dirichlet implementation (per-class client allocation vs per-client class - sampling) must be fixed and documented, because it changes realised `N_ij`. `PROVISIONAL`. -- Success criterion for reproducing the paper's setting: at low alpha, several clients miss - several classes, consistent with the paper's reported counts (e.g. X-IIoTID `alpha=0.25`: - 8/10 clients miss 1–5 classes). +FROZEN algorithm (**PROJECT DECISION**, paper does not specify the allocation procedure): + +1. Partitions are generated from the **training split only**. +2. For each class `c`, draw client proportions `p ~ Dirichlet(alpha * 1_K_clients)`. +3. Allocate that class's training samples to clients with + `counts ~ Multinomial(N_c, p)`, so the counts sum exactly to `N_c`. +4. Split the class's shuffled sample positions into consecutive blocks of size `counts`. + The shuffle is a permutation drawn from the same deterministic RNG. +5. A client may receive zero samples of a class; that class is then *missing* on the client + (`N_ij = 0`), which is the intended source of non-IIDness. + +Determinism: the RNG seed is derived as +`rng_seed = sha256(f"{dataset}|dirichlet|alpha={alpha}|seed={seed}")[:16] mod 2**31`. +The manifest stores `rng_seed`, so the same partition is reproduced exactly on reload. + +Success criterion for reproducing the paper's setting: at low alpha several clients miss +several classes. ## 3. Splits -- Test: 20% of each dataset (paper uses 80/20). FROZEN. -- Validation: a separate split used only for beta selection. The paper has **no** validation - set, so this is an explicit project addition. `PROVISIONAL` fraction and procedure. -- Requirement: train/validation/test come from a single deterministic split with a recorded - seed, and the same split is reused for every beta variant. +FROZEN: a single deterministic **stratified** split per `(dataset, seed)`. + +- `test = 0.20` of the cleaned dataset (paper uses 80/20). FROZEN. +- `validation = 0.10`; `train = 0.70`. **PROJECT DECISION**: the paper has no validation set; + this project adds one for beta selection. The test fraction keeps the paper's 20%. +- Stratification is per class: each class's samples are shuffled with a deterministic RNG and + split 70/10/20. This guarantees every class is represented in validation and test, which is + required for rare-class evaluation. +- Split seed: `sha256(f"{dataset}|split|seed={seed}")[:16] mod 2**31`. +- The exact same split is reused for every aggregation variant and beta candidate. ## 4. Seeds -- The paper averages over three runs with unnamed seeds. `PROVISIONAL` set: `{0, 1, 2}`. -- Once frozen, the seed set is identical across all aggregation variants. -- Each seed deterministically controls: split, Dirichlet partition, model initialisation, - batch order/shuffling. -- A beta run MUST NOT derive a new partition; it loads the manifest for - `(dataset, alpha, seed)`. +FROZEN (`PROJECT DECISION`): `{0, 1, 2}` (the paper averages over three unnamed seeds). -## 5. Beta candidates and modes +Each seed deterministically controls: split, Dirichlet partition, model initialisation, batch +order/shuffling. All RNG use is isolated via `protean_research.utils.seeding.derive_seed`. +A beta run MUST NOT derive a new partition; it loads the manifest for `(dataset, alpha, seed)`. -FROZEN: `beta ∈ {0.00, 0.25, 0.50, 0.75, 1.00}`, one beta for all classes and clients. +## 5. Aggregation modes -Modes: +**Baseline checkpoint: only `original_protean` is implemented.** The proposed +support-weighted aggregation (`support_weighted`) and the `equal_available` control are +**DEFERRED** to the later beta/controls phase and are not present in executable code. -- `original_protean` — verified baseline semantics. Not yet encodable (see baseline - investigation). This is the baseline to reproduce. -- `support_weighted` — `w_ij = N_ij^beta`, normalised over `S_j` (proposed mechanism). -- `equal_available` — equal weights over `S_j`; a methodological control introduced only if - the baseline participant rule differs from `S_j`. Not to be presented as a novelty. +- `original_protean` — reproduced baseline. **REPRODUCTION ASSUMPTION (project decision)**: + because Eq. (2)/(5.5) is internally inconsistent about missing classes, the baseline + participant rule is assumed to be `S_j = { i | N_ij > 0 and a valid C_ij exists }` with + **equal weights `1/|S_j|`**, following the thesis statement that prototypes are shared + only for locally observed classes. This is not verified against code and must be + validated in the baseline-verification phase. +- `support_weighted` (deferred) — `w_ij = N_ij^beta`, normalised over `S_j`. +- `equal_available` (deferred) — equal weights over `S_j`; only introduced later if the + baseline participant rule is shown to differ from `S_j`. -`S_j = { i | N_ij > 0 and a valid local prototype C_ij exists }`. +FROZEN candidates for the later phase: `beta ∈ {0.00, 0.25, 0.50, 0.75, 1.00}`, one beta for +all classes and clients. They are **not** part of the baseline checkpoint. ## 6. Partition persistence -Each `(dataset, alpha, seed)` produces one persisted manifest containing at least: +Each `(dataset, alpha, seed)` produces one persisted JSON manifest under +`artifacts/partitions/` containing: -- dataset name, alpha, seed, num_clients, K; -- per-client sample indices or a reproducible reference plus a content hash; -- per-client `N_ij` matrix `[client][class]`; -- class label mapping; -- split indices for train/validation/test; -- preprocessing/config hash and code version. +- dataset key, alpha, seed, num_clients, num_classes, class names; +- the Dirichlet `algorithm` id and `rng_seed`; +- per-client `N_ij` matrix `[client][class]` (training split); +- split metadata (seed, per-split sizes, content hash) and partition content hash; +- preprocessing/config/code hashes. -Manifests live under `artifacts/partitions/`. Reloading a manifest must reproduce the same -partition (tested). Training reads manifests; it does not regenerate them. +Manifests are small and version-controlled. Full sample-index arrays are **not** stored by +default (they would be tens of MB per manifest); they are regenerated deterministically from +the stored recipe and RNG seed and verified against the stored `N_ij` and content hash. This +is the tested reload path: `load_manifest` -> `reproduce_partition` -> hash match. ## 7. Rare and unseen classes @@ -96,18 +121,17 @@ Persisted metadata (minimum one row per `(dataset, alpha, seed, client, class)`) Procedure, strictly ordered: -1. **Candidate evaluation (validation only).** For each beta candidate, train with the - paired manifest and compute validation metrics. Primary criterion: rare-class macro-F1, - computed per client that has rare classes, then averaged over those clients. -2. **Beta selection.** Choose the beta with the best mean validation rare-class macro-F1 - (ties broken by a pre-declared rule, e.g. smaller beta). The result is - "best among tested candidates under this procedure", never a global optimum. +1. **Candidate evaluation (validation only).** For each beta candidate, train with the paired + manifest and compute validation metrics. Primary criterion: rare-class macro-F1, computed + per client that has rare classes, then averaged over those clients. +2. **Beta selection.** Choose the beta with the best mean validation rare-class macro-F1 (ties + broken by smaller beta). The result is "best among tested candidates under this + procedure", never a global optimum. 3. **Final test evaluation.** Evaluate the selected beta on the test split. Report overall macro-F1, rare-class recall, rare-class macro-F1, and per-client results, plus the test results of all candidates for transparency. -Test data is never used in steps 1–2. The selection code must not have access to test -outputs; test evaluation is a separate step keyed on the selected beta. +Test data is never used in steps 1–2. ## 9. Required metrics @@ -117,29 +141,28 @@ Per dataset/alpha/seed/beta and per client, record: - rare-class recall; - overall macro-F1; - accuracy (for comparison with the paper); -- per-class precision/recall/F1 and confusion counts; +- per-class precision/recall/F1 and support (confusion counts persisted); - per-client metrics. -Also record the validation/test rare-class support counts, so rare-class metrics are never -read without their sample sizes. Results are reported as mean and variation across seeds. +Also record the validation/test rare-class support counts. Results are reported as mean and +variation across seeds. ## 10. Non-IID characterization For every manifest, record at least: `N_ij`, class total, number of clients containing class `j`, `min N_ij`, `max N_ij`, `mean N_ij`, `std N_ij`, coefficient of variation, and -`max/min` ratio where defined. Alpha alone is insufficient; different seeds realise -different distributions. +`max/min` ratio where defined. Alpha alone is insufficient. ## 11. Direct prototype analysis Separate the direct mathematical aggregation effect from the end-to-end training effect. -- During a run, persist at least one round's fixed `C_ij` and `N_ij` snapshot. +- Persist at least one round's fixed `C_ij` and `N_ij` snapshot. - Reusing those snapshots, compute `Cbar_j(beta)` for several beta without retraining. -- Report `|| Cbar_j(beta) - Cbar_j(0) ||`; optionally the normalised aggregation weights, - distances to contributing prototypes, pairwise global-prototype displacement, and support +- Report `|| Cbar_j(beta) - Cbar_j(0) ||`; optionally normalised aggregation weights, + distances to contributing prototypes, pairwise global-prototype displacement, support imbalance. -- This analysis complements, and never replaces, end-to-end evaluation. +- This complements, and never replaces, end-to-end evaluation. ## 12. Paired comparison guarantee @@ -154,24 +177,66 @@ For each `(dataset, alpha, seed)`, every beta variant and every mode reuses: A beta run that regenerates a fresh Dirichlet partition is a protocol violation. -## 13. Frozen vs provisional summary +## 13. Preprocessing (frozen) -| Item | Value | Status | +**PROJECT DECISION** — the paper specifies no preprocessing. Frozen recipe: + +1. Read the raw CSV read-only (`pandas.read_csv`, `low_memory=False`). +2. Column names are stripped; the empty 5G-NIDD header becomes `Unnamed: 0`. +3. Exact duplicate rows are dropped (`keep="first"`) before splitting. +4. Rows whose label is in `drop_labels` are dropped. +5. Labels are canonicalised: surrounding whitespace removed, runs of whitespace collapsed, + and whitespace around `_` removed (fixes `Lateral _movement`). +6. Feature columns = all columns except the label column, its aliases, and + `non_feature_columns`. +7. `missing_tokens` are replaced with NaN. Categorical missing values become an explicit + `__MISSING__` category. +8. Categorical columns (`categorical_columns`) are one-hot encoded with categories fitted on + the **train split only**; unseen categories at transform time become all-zero. +9. Numeric columns are coerced to numeric, median-imputed and standardized using **train + statistics only**. +10. Output is a deterministic `float32` matrix with a fixed column order. + +No statistic from validation or test data ever enters preprocessing. + +## 14. Model, local training, inference (frozen) + +- Model: 2 Conv1D (64, 128 filters, ReLU, max-pool, dropout 0.2/0.5), flatten, dense 128 + (embedding), log-softmax head. **PROJECT DECISION**: kernel size 3, padding 1. +- Local loss: `cross_entropy + lambda * ||phi(x) - Cbar_{y}||^2 + (mu/2) * ||omega_i - omega||^2`, + with `mu = 0.1` (paper), `lambda = 1.0` (**PROJECT DECISION**, paper silent). The prototype + term is applied per sample only for classes that currently have a global prototype. +- Optimizer Adam, `lr = 1e-3`, batch size 256, 3 local epochs, 10 rounds + (**PROJECT DECISION** where not stated). +- Local prototypes: mean embedding over the client's full class-`j` training samples, + computed at the **end of the local round** (paper Eq. 1). Missing classes produce no + prototype. +- Server: model parameters averaged with equal `1/M` weights (paper Eq. 2/5.5); class + prototypes aggregated with the active mode. +- Round 0: no global prototypes exist; the prototype-alignment term is skipped; the first + global prototypes are produced by aggregating round-0 local prototypes. +- Inference: embedding `z = phi(x)`, prediction `argmin_j ||z - Cbar_j||^2` over classes that + have a global prototype. + +## 15. Frozen vs decision summary + +| Item | Value | Source | |---|---|---| -| Datasets | X-IIoTID, 5G-NIDD | FROZEN | -| Clients | 10 | FROZEN | -| Alpha | 0.75, 0.50, 0.25 | FROZEN | -| Beta candidates | 0, 0.25, 0.50, 0.75, 1.00 | FROZEN | -| Test fraction | 0.20 | FROZEN | -| Rounds / local epochs | 10 / 3 | FROZEN | -| `mu` | 0.1 | FROZEN | -| `lambda` | unknown | PROVISIONAL | -| Batch size / learning rate | unknown | PROVISIONAL | -| Validation fraction / procedure | unknown | PROVISIONAL | -| Seeds | 0, 1, 2 | PROVISIONAL | -| Dirichlet allocation algorithm | unknown | PROVISIONAL | -| Preprocessing recipe | unknown | PROVISIONAL | -| Missing-class baseline rule | unknown | PROVISIONAL / blocked on baseline | +| Datasets | X-IIoTID, 5G-NIDD | paper | +| Clients | 10 | paper | +| Alpha | 0.75, 0.50, 0.25 | paper | +| Beta candidates | 0, 0.25, 0.50, 0.75, 1.00 | spec | +| Test fraction | 0.20 | paper | +| Rounds / local epochs | 10 / 3 | paper | +| `mu` | 0.1 | paper | +| `lambda` | 1.0 | project decision | +| Batch size / learning rate | 256 / 1e-3 (Adam) | project decision | +| Validation fraction | 0.10 (stratified) | project decision | +| Seeds | 0, 1, 2 | project decision | +| Dirichlet allocation | per-class Dirichlet + multinomial | project decision | +| Preprocessing recipe | §13 | project decision | +| Baseline participant rule | `S_j`, equal `1/|S_j|` | project decision | +| Model aggregation | equal `1/M` | paper Eq. 2/5.5 | +| Prototype computation | end-of-round mean embedding | paper Eq. 1 | -The provisional items must be resolved and recorded here before the first sweep. Until then -only smoke tests are permitted. +Until the baseline is verified, only smoke tests are permitted; no sweep may be launched. diff --git a/docs/research/IMPLEMENTATION_PLAN.md b/docs/research/IMPLEMENTATION_PLAN.md index 1f535a7..025fe0d 100644 --- a/docs/research/IMPLEMENTATION_PLAN.md +++ b/docs/research/IMPLEMENTATION_PLAN.md @@ -1,6 +1,9 @@ # Implementation Plan -Status: design document (no full training implemented in this session) +Status: design document. The data, model, federated and evaluation modules +described below are now IMPLEMENTED for the baseline (see +`docs/baseline/BASELINE_IMPLEMENTATION.md`); the analysis/direct-prototype module +is still future work and beta experiments were not run in this session. Scope: architecture for the server-side prototype-aggregation study on top of a reproduced PROTEAN baseline. @@ -107,19 +110,18 @@ regenerate a partition. - `round.py`: one communication round; records per-round prototype snapshots for direct analysis. -### `aggregation/` (exists) +### `aggregation/` (baseline implemented; proposed method deferred) -Pure, dependency-free functions implementing only the unambiguous modes: +Pure, dependency-free functions implementing only the reproduced baseline: -- `beta_weights(supports, beta)` → `N_ij ** beta` (unnormalised). -- `aggregation_weights(prototypes, mode, beta, supports)` → normalised weights summing to 1. -- `aggregate_prototypes(...)` → `Cbar_j` for one class. -- `AggregationMode`: `original_protean` (raises `NotImplementedError` until verified), - `support_weighted`, `equal_available`. +- `aggregate_prototypes(prototypes, mode)` → `Cbar_j` for one class (equal mean over `S_j`). +- `aggregate_class_prototypes(contributions, mode)` → per-class global prototypes; builds + `S_j` by dropping clients with `N_ij = 0` or a missing prototype. +- `AggregationMode`: only `original_protean` in this checkpoint. -Planned addition: a server-facing helper that accepts `{client_id: (C_ij, N_ij)}` for class -`j`, constructs `S_j` (drop clients with `N_ij = 0` or missing prototype), and delegates to -`aggregate_prototypes`. This keeps all weighting logic in one tested place. +The proposed `support_weighted` mode (`N_ij ** beta`) and the `equal_available` control are +**deferred** to the later beta/controls phase and are not present in executable code. The +baseline participant rule (`S_j`, equal weights) is a documented reproduction assumption. ### `evaluation/` (planned) @@ -139,20 +141,23 @@ Planned addition: a server-facing helper that accepts `{client_id: (C_ij, N_ij)} `Cbar_j(beta)` for several beta without retraining and report `||Cbar_j(beta) - Cbar_j(0)||`, weights, distances, displacement, imbalance. -## 5. Aggregation API concept +## 5. Aggregation API (baseline) ``` -aggregate_prototypes(prototypes, mode, beta=None, supports=None) -> list[float] +aggregate_prototypes(prototypes, mode="original_protean") -> list[float] +aggregate_class_prototypes(contributions, mode="original_protean") -> dict[int, list[float]] ``` -Contract: +Contract (implemented baseline): -- `prototypes[i]` is `C_ij`; `supports[i]` is `N_ij > 0`; order is client order. -- `support_weighted`: weights `N_ij ** beta`, normalised over `S_j`. -- `equal_available`: equal weights over `S_j` (control; not for `beta > 0` claims). -- `original_protean`: only after the baseline missing-class rule is verified and encoded. -- Validates finiteness, strictly positive supports, equal dimensions; normalised weights sum - to 1; never emits NaN/Inf for valid inputs. +- `prototypes[i]` is `C_ij`; order is client order. +- `original_protean`: equal mean over the supplied contributor set `S_j`. +- `aggregate_class_prototypes` builds `S_j` by dropping clients with `N_ij <= 0` or a + missing prototype; missing classes are never fabricated. +- Validates finiteness and equal dimensions; never emits NaN/Inf for valid inputs. + +The proposed `support_weighted(beta, supports)` and `equal_available` modes are **planned** +for the later beta/controls phase and are not implemented in this checkpoint. ## 6. Experiment orchestration @@ -169,22 +174,24 @@ Contract: ## 7. Testing strategy -Required tests (from the specification), to be implemented as the code is built: +Implemented in the baseline checkpoint (see `tests/`): -1. `beta = 0` equals the arithmetic mean over the same `S_j`. -2. `beta = 1` equals sample-count weighting (equals pooling of included embeddings). -3. One contributing client gives the same output for all beta. -4. Equal supports give the same output for all beta. -5. A missing class does not enter `S_j`. -6. Normalised weights sum to one. -7. Valid inputs never produce NaN/Inf. -8. Partition generation is deterministic for a fixed seed. -9. Saved partition manifests reload identically. -10. Baseline behaviour is unchanged by adding the proposed mode. -11. Validation/test separation is maintained. +- baseline aggregation is the equal mean over `S_j`; +- a missing class does not enter `S_j` and is never fabricated; +- valid inputs never produce NaN/Inf; dimension/finiteness validation; +- partition generation is deterministic for a fixed seed; +- saved partition manifests reload identically (hash-verified); +- rare/unseen semantics; prototype means; metrics on known fixtures; +- a small end-to-end federated smoke test. -This session adds the dependency-free subset (1–7) in `tests/test_prototype_aggregation.py`. -Items 8–11 require the future data/federated modules and are deferred. +Deferred to the beta/controls phase (proposed-method invariants): + +- `beta = 0` equals the arithmetic mean over the same `S_j`; +- `beta = 1` equals sample-count weighting; +- single-contributor and equal-support invariance across beta; +- normalised beta weights sum to one; +- baseline behaviour unchanged by adding the proposed mode; +- paired beta/validation/test separation. ## 8. Compute feasibility diff --git a/docs/research/RESEARCH_SPEC_INTERPRETATION.md b/docs/research/RESEARCH_SPEC_INTERPRETATION.md index 41123f7..fa1e731 100644 --- a/docs/research/RESEARCH_SPEC_INTERPRETATION.md +++ b/docs/research/RESEARCH_SPEC_INTERPRETATION.md @@ -88,9 +88,10 @@ must still be verified. the participant rule from the effect of the weights. The spec says such a control is only introduced if needed. -`IMPLEMENTATION DECISION` `equal_available` is kept as a dormant, documented control mode. -It must not be reported as a result unless baseline investigation demonstrates its -necessity (see `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). +`IMPLEMENTATION DECISION` The `equal_available` control is **deferred** to the later +beta/controls phase and is not implemented in the baseline-only checkpoint. It is +introduced only if baseline investigation demonstrates that the baseline participant rule +differs from `S_j` (see `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). ## 5. Rare class and unseen class definitions diff --git a/scripts/generate_partitions.py b/scripts/generate_partitions.py new file mode 100644 index 0000000..54a9724 --- /dev/null +++ b/scripts/generate_partitions.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +"""Generate deterministic Dirichlet partition manifests for all conditions. + +This does NOT train anything. It loads each dataset once (labels only), computes +the frozen stratified split, generates the Dirichlet partition for every +``(alpha, seed)`` and persists a manifest plus characterization/rare-unseen +metadata. Training later loads these manifests; it never regenerates a partition. + +Usage:: + + python scripts/generate_partitions.py --datasets xiiotid 5gnidd \ + --alphas 0.75 0.5 0.25 --seeds 0 1 2 --clients 10 +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from protean_research.data import loading, manifest as manifest_mod, partition as partition_mod # noqa: E402 +from protean_research.data import rare_unseen, splits as splits_mod # noqa: E402 +from protean_research.data.characterization import characterize_support # noqa: E402 +from protean_research.data.pipeline import ( # noqa: E402 + TEST_FRACTION, + TRAIN_FRACTION, + VALIDATION_FRACTION, +) +from protean_research.utils.config import expand_path, load_datasets_config, load_yaml # noqa: E402 + + +def write_csv(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8") + return + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--datasets", nargs="+", default=["xiiotid", "5gnidd"]) + parser.add_argument("--alphas", nargs="+", type=float, default=[0.75, 0.5, 0.25]) + parser.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) + parser.add_argument("--clients", type=int, default=10) + parser.add_argument("--partitions-dir", default=None) + parser.add_argument("--analysis-dir", default=None) + args = parser.parse_args(argv) + + datasets_cfg = load_datasets_config() + protocol = load_yaml(REPO_ROOT / "configs" / "experiments" / "experiment_protocol.yaml") + artifacts_cfg = protocol.get("artifacts", {}) + partitions_dir = expand_path(args.partitions_dir or artifacts_cfg.get("partition_manifest_dir", "artifacts/partitions")) + analysis_dir = expand_path(args.analysis_dir or artifacts_cfg.get("analysis_dir", "artifacts/analysis")) + + created = [] + for dataset_key in args.datasets: + spec = loading.load_spec(dataset_key, datasets_cfg) + cleaned = loading.clean_frame(loading.read_raw(spec), spec) + for alpha in args.alphas: + for seed in args.seeds: + split = splits_mod.stratified_split( + cleaned.labels, dataset_key, seed, + train_fraction=TRAIN_FRACTION, + validation_fraction=VALIDATION_FRACTION, + test_fraction=TEST_FRACTION, + ) + labels_train = cleaned.labels[split.train] + class_indices = partition_mod.class_index_map(labels_train, len(cleaned.class_names)) + rng_seed = partition_mod.partition_rng_seed(dataset_key, alpha, seed) + assignments = partition_mod.dirichlet_partition( + class_indices, args.clients, alpha, rng_seed + ) + manifest = manifest_mod.build_manifest( + dataset_key=dataset_key, + dataset_name=spec.name, + alpha=alpha, + seed=seed, + num_clients=args.clients, + class_names=cleaned.class_names, + benign_class=spec.benign_label, + labels=cleaned.labels, + split=split, + assignments=assignments, + rng_seed=rng_seed, + train_fraction=TRAIN_FRACTION, + validation_fraction=VALIDATION_FRACTION, + test_fraction=TEST_FRACTION, + ) + path = manifest_mod.manifest_path(partitions_dir, dataset_key, alpha, seed) + manifest_mod.save_manifest(path, manifest) + + # Verify exact reload before declaring success. + manifest_mod.reproduce_train_assignments(manifest, cleaned.labels) + + support = manifest["support"] + import numpy as np + + support_arr = np.asarray(support, dtype=np.int64) + tag = f"{dataset_key}__alpha{alpha:g}__seed{seed}" + write_csv(analysis_dir / f"{tag}__characterization.csv", + characterize_support(support_arr, cleaned.class_names)) + write_csv(analysis_dir / f"{tag}__rare_unseen.csv", + rare_unseen.rare_unseen_rows( + support_arr, cleaned.class_names, cleaned.benign_index, + dataset_key, alpha, seed)) + created.append({"dataset": dataset_key, "alpha": alpha, "seed": seed, + "path": str(path), "partition_hash": manifest["partition_hash"]}) + print(f"created {path.name}") + + print(json.dumps({"created": len(created)}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_baseline.py b/scripts/run_baseline.py new file mode 100644 index 0000000..580293b --- /dev/null +++ b/scripts/run_baseline.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python +"""Baseline PROTEAN smoke/run entry point. + +Runs the frozen baseline pipeline end to end for one ``(dataset, alpha, seed)``: +load -> clean -> split -> preprocess -> Dirichlet partition -> federated training +-> local prototypes -> server aggregation -> global redistribution -> evaluation +-> structured artifacts. + +This runner defaults to a small configuration suitable for a smoke test. A full +scientific sweep must be launched separately once the baseline is verified. + +Usage (smoke):: + + python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ + --max-rows 4000 --clients 4 --rounds 2 --local-epochs 1 +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from protean_research.data import manifest as manifest_mod # noqa: E402 +from protean_research.data import partition as partition_mod # noqa: E402 +from protean_research.data import rare_unseen # noqa: E402 +from protean_research.data.characterization import characterize_support # noqa: E402 +from protean_research.data.pipeline import ( # noqa: E402 + create_partition, + prepare_dataset, + preprocessing_content_hash, +) +from protean_research.evaluation.metrics import ( # noqa: E402 + classification_metrics, + rare_class_metrics, +) +from protean_research.federated import TrainingConfig, run_federated # noqa: E402 +from protean_research.models.protean_cnn import extract_embeddings # noqa: E402 +from protean_research.models.prototypes import nearest_prototype_predict # noqa: E402 +from protean_research.utils.config import ( # noqa: E402 + expand_path, + load_datasets_config, + load_yaml, +) + + +def parse_args(argv=None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset", required=True) + parser.add_argument("--alpha", type=float, required=True) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--clients", type=int, default=10) + parser.add_argument("--rounds", type=int, default=10) + parser.add_argument("--local-epochs", type=int, default=3) + parser.add_argument("--batch-size", type=int, default=256) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--mu", type=float, default=0.1) + parser.add_argument("--lam", type=float, default=1.0) + parser.add_argument( + "--max-rows", + type=int, + default=None, + help="SMOKE ONLY: limit how many raw rows are read; never use for science.", + ) + parser.add_argument("--sample-from", type=int, default=200000, + help="SMOKE ONLY: rows to read before random subsampling.") + parser.add_argument("--partitions-dir", default=None) + parser.add_argument("--processed-dir", default=None) + parser.add_argument("--results-dir", default=None) + parser.add_argument("--analysis-dir", default=None) + return parser.parse_args(argv) + + +def write_json(path: Path, payload) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def write_csv(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8") + return + fieldnames = list(rows[0].keys()) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def main(argv=None) -> int: + args = parse_args(argv) + datasets_cfg = load_datasets_config() + protocol = load_yaml(REPO_ROOT / "configs" / "experiments" / "experiment_protocol.yaml") + + artifacts_cfg = protocol.get("artifacts", {}) + partitions_dir = expand_path(args.partitions_dir or artifacts_cfg.get("partition_manifest_dir", "artifacts/partitions")) + processed_dir = expand_path(args.processed_dir or artifacts_cfg.get("processed_dir", "artifacts/processed")) + analysis_dir = expand_path(args.analysis_dir or artifacts_cfg.get("analysis_dir", "artifacts/analysis")) + results_root = expand_path(args.results_dir or artifacts_cfg.get("results_dir", "results")) + + # Smoke runs must never pollute the tracked partition/analysis directories. + if args.max_rows is not None: + if args.partitions_dir is None: + partitions_dir = processed_dir / "smoke" / "partitions" + if args.analysis_dir is None: + analysis_dir = processed_dir / "smoke" / "analysis" + if args.results_dir is None: + results_root = results_root / "smoke" + + if args.max_rows is not None: + from protean_research.data import loading as loading_mod + import numpy as np + + spec = loading_mod.load_spec(args.dataset, datasets_cfg) + pool = loading_mod.read_raw(spec, nrows=args.sample_from) + rng = np.random.default_rng(args.seed) + take = min(args.max_rows, len(pool)) + chosen = np.sort(rng.choice(len(pool), size=take, replace=False)) + pool = pool.iloc[chosen].reset_index(drop=True) + bundle = prepare_dataset( + args.dataset, seed=args.seed, datasets_cfg=datasets_cfg, raw_frame=pool + ) + else: + bundle = prepare_dataset(args.dataset, seed=args.seed, datasets_cfg=datasets_cfg) + + # ---- partition (create or reload) ------------------------------------- + pre_hash = preprocessing_content_hash(bundle.preprocessor) + manifest_file = manifest_mod.manifest_path(partitions_dir, args.dataset, args.alpha, args.seed) + if manifest_file.exists(): + manifest = manifest_mod.load_manifest(manifest_file) + split, assignments, support = manifest_mod.reproduce_train_assignments(manifest, bundle.labels) + # Reuse the manifest's split for pairing; verify it matches the bundle. + bundle.split = split + else: + manifest, assignments, support = create_partition( + bundle, alpha=args.alpha, seed=args.seed, num_clients=args.clients, + partition_dir=partitions_dir, preprocessing_hash=pre_hash, + ) + manifest_reload_ok = True + try: + manifest_mod.reproduce_train_assignments(manifest, bundle.labels) + except Exception: # pragma: no cover - surfaced in metadata + manifest_reload_ok = False + + labels = bundle.labels + train_features = bundle.features[bundle.split.train] + train_labels = labels[bundle.split.train] + val_features = bundle.features[bundle.split.validation] + val_labels = labels[bundle.split.validation] + test_features = bundle.features[bundle.split.test] + test_labels = labels[bundle.split.test] + + benign_index = bundle.cleaned.benign_index + per_client_rare = rare_unseen.rare_class_ids(support, benign_index) + union_rare = sorted({c for row in per_client_rare for c in row}) + + cfg = TrainingConfig( + rounds=args.rounds, + local_epochs=args.local_epochs, + batch_size=args.batch_size, + learning_rate=args.lr, + mu=args.mu, + lam=args.lam, + num_clients=args.clients, + ) + result = run_federated( + train_features=train_features, + train_labels=train_labels, + assignments=assignments, + val_features=val_features, + val_labels=val_labels, + num_classes=bundle.num_classes, + input_dim=bundle.features.shape[1], + config=cfg, + init_seed_parts=(args.dataset, f"seed={args.seed}"), + rare_class_ids=union_rare, + ) + + model = result["model"] + global_prototypes = result["global_prototypes"] + + # ---- global test evaluation ------------------------------------------ + test_embeddings = extract_embeddings(model, test_features) + test_pred = nearest_prototype_predict(test_embeddings, global_prototypes) + test_metrics = classification_metrics(test_labels, test_pred, bundle.num_classes) + test_rare = rare_class_metrics(test_labels, test_pred, bundle.num_classes, union_rare) + + # ---- per-client evaluation ------------------------------------------- + eval_assignments, eval_rng = partition_mod.partition_eval_split( + test_labels, args.clients, args.alpha, args.dataset, args.seed, "test" + ) + per_client_rows = [] + for client_id in range(args.clients): + idx = eval_assignments[client_id] + if len(idx) == 0: + per_client_rows.append({ + "client_id": client_id, "n_samples": 0, "accuracy": None, + "macro_f1": None, "rare_class_macro_f1": None, "rare_class_recall": None, + }) + continue + emb = extract_embeddings(model, test_features[idx]) + pred = nearest_prototype_predict(emb, global_prototypes) + metrics = classification_metrics(test_labels[idx], pred, bundle.num_classes) + rare_ids = per_client_rare[client_id] + if rare_ids: + rare = rare_class_metrics(test_labels[idx], pred, bundle.num_classes, rare_ids) + else: + rare = {"rare_class_macro_f1": None, "rare_class_recall": None} + per_client_rows.append({ + "client_id": client_id, + "n_samples": int(len(idx)), + "accuracy": metrics["accuracy"], + "macro_f1": metrics["macro_f1"], + "rare_class_macro_f1": rare["rare_class_macro_f1"], + "rare_class_recall": rare["rare_class_recall"], + }) + + # ---- structured artifacts -------------------------------------------- + run_name = f"{args.dataset}__alpha{args.alpha:g}__seed{args.seed}__original_protean" + run_dir = results_root / run_name + run_dir.mkdir(parents=True, exist_ok=True) + + metadata = { + "run_name": run_name, + "dataset": args.dataset, + "alpha": args.alpha, + "seed": args.seed, + "num_clients": args.clients, + "num_classes": bundle.num_classes, + "class_names": bundle.class_names, + "benign_class": bundle.spec.benign_label, + "aggregation_mode": "original_protean", + "training": { + "rounds": args.rounds, + "local_epochs": args.local_epochs, + "batch_size": args.batch_size, + "learning_rate": args.lr, + "mu": args.mu, + "lambda": args.lam, + }, + "smoke": args.max_rows is not None, + "max_rows": args.max_rows, + "manifest_path": str(manifest_file), + "manifest_content_hash": manifest_mod.manifest_content_hash(manifest), + "preprocessing_hash": pre_hash, + "manifest_reload_ok": manifest_reload_ok, + "repo_root": str(REPO_ROOT), + } + write_json(run_dir / "metadata.json", metadata) + write_json(run_dir / "history.json", result["history"]) + write_json(run_dir / "test_global_metrics.json", { + "accuracy": test_metrics["accuracy"], + "macro_f1": test_metrics["macro_f1"], + "weighted_f1": test_metrics["weighted_f1"], + **test_rare, + }) + write_json(run_dir / "test_confusion_matrix.json", test_metrics["confusion_matrix"]) + write_csv(run_dir / "test_per_class.csv", [ + {**row, "class_name": bundle.class_names[row["class_id"]]} + for row in test_metrics["per_class"] + ]) + write_csv(run_dir / "per_client.csv", per_client_rows) + write_csv(run_dir / "train_support_Nij.csv", [ + {"client_id": i, **{bundle.class_names[c]: int(support[i, c]) for c in range(bundle.num_classes)}} + for i in range(args.clients) + ]) + write_csv(run_dir / "characterization.csv", characterize_support(support, bundle.class_names)) + write_csv(run_dir / "rare_unseen.csv", rare_unseen.rare_unseen_rows( + support, bundle.class_names, benign_index, args.dataset, args.alpha, args.seed + )) + + processed_dir.mkdir(parents=True, exist_ok=True) + bundle.preprocessor.save(processed_dir / f"{args.dataset}__seed{args.seed}__preprocessor.json") + + snapshot = result["snapshot"] + if snapshot is not None: + write_json(analysis_dir / f"{run_name}__prototype_snapshot.json", snapshot) + + summary = { + "run": run_name, + "test_accuracy": test_metrics["accuracy"], + "test_macro_f1": test_metrics["macro_f1"], + "test_rare_class_macro_f1": test_rare["rare_class_macro_f1"], + "n_global_prototypes": len(global_prototypes), + "history_last": result["history"][-1] if result["history"] else None, + "results_dir": str(run_dir), + } + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/protean_research/aggregation/__init__.py b/src/protean_research/aggregation/__init__.py index 0fd4296..458ad15 100644 --- a/src/protean_research/aggregation/__init__.py +++ b/src/protean_research/aggregation/__init__.py @@ -1,20 +1,18 @@ """Server-side prototype aggregation. -This package is the single locus of the proposed modification. Local model -training, the embedding architecture, the loss terms and the inference rule are -intentionally left untouched (see docs/research/IMPLEMENTATION_PLAN.md). +This package is the single locus of the proposed modification. In the baseline +checkpoint it implements only the reproduced original-PROTEAN aggregation. The +proposed support-weighted aggregation is deferred to the beta/controls phase. """ from .prototype_aggregation import ( AggregationMode, + aggregate_class_prototypes, aggregate_prototypes, - aggregation_weights, - beta_weights, ) __all__ = [ "AggregationMode", + "aggregate_class_prototypes", "aggregate_prototypes", - "aggregation_weights", - "beta_weights", ] diff --git a/src/protean_research/aggregation/prototype_aggregation.py b/src/protean_research/aggregation/prototype_aggregation.py index 4b7fc37..71d1d08 100644 --- a/src/protean_research/aggregation/prototype_aggregation.py +++ b/src/protean_research/aggregation/prototype_aggregation.py @@ -1,73 +1,40 @@ -"""Pure server-side prototype aggregation. +"""Baseline server-side prototype aggregation. -Formulation under study ------------------------ -For a class ``j`` with contributing-client set ``S_j = {i | N_ij > 0 and C_ij exists}``:: +This module implements **only the reproduced original-PROTEAN prototype +aggregation** used by the baseline. The proposed support-weighted aggregation +(``N_ij ** beta``) is deliberately NOT present in this checkpoint; it belongs to +the later beta/controls phase and must not be introduced before the baseline is +verified. - C_bar_j(beta) = sum_{i in S_j} N_ij**beta * C_ij - -------------------------------- - sum_{i in S_j} N_ij**beta +Participant rule (reproduction assumption, not verified against code) +--------------------------------------------------------------------- +The published Equation (2) / thesis Eq. (5.5) writes ``C_bar_j = (1/M) * sum_i +C_ij`` over all ``M`` clients, but never states how a class that is missing on a +client (``N_ij = 0``) is handled. The thesis separately states that "prototypes +are shared only for locally observed classes", so this baseline assumes the +server receives prototypes only from the contributor set:: -with candidate ``beta in {0.0, 0.25, 0.5, 0.75, 1.0}``. + S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } -The functions in this module are pure and dependency-free so they can be unit -tested in a bare interpreter. They implement only the two *unambiguous* modes: - -``equal_available`` - equal weight for every contributing client in ``S_j`` (``beta == 0`` control). - -``support_weighted`` - ``w_ij = N_ij ** beta`` normalized over ``S_j`` (the proposed mechanism). - -``original_protean`` is deliberately NOT implemented here. The published -Equation (2) writes ``(1/M) * sum_{i=1}^{M} C_ij`` but the paper does not define -what happens to ``C_ij`` when ``N_ij == 0`` (omitted / zero vector / stale value / -different denominator). Encoding any choice here would silently assume baseline -semantics that must first be verified from official code. See -``docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md``. +and averages them with **equal weights ``1/|S_j|``**. This is an explicit +REPRODUCTION ASSUMPTION of this project, not a source-confirmed fact. It must be +validated in the baseline-verification phase; see +``docs/baseline/BASELINE_IMPLEMENTATION.md``. """ from __future__ import annotations import math from enum import Enum -from typing import Optional, Sequence +from typing import Mapping, Sequence Vector = Sequence[float] class AggregationMode(str, Enum): - """Supported prototype-aggregation modes.""" + """Prototype-aggregation modes implemented in the baseline checkpoint.""" ORIGINAL_PROTEAN = "original_protean" - SUPPORT_WEIGHTED = "support_weighted" - EQUAL_AVAILABLE = "equal_available" - - -def _validate_beta(beta: Optional[float]) -> float: - if beta is None: - raise ValueError("beta is required") - beta = float(beta) - if math.isnan(beta) or math.isinf(beta): - raise ValueError(f"beta must be finite, got {beta!r}") - if not 0.0 <= beta <= 1.0: - raise ValueError(f"beta must be in the closed interval [0, 1], got {beta!r}") - return beta - - -def _validate_supports(supports: Optional[Sequence[float]]) -> list[float]: - if supports is None: - raise ValueError("supports (N_ij) are required") - result = [float(n) for n in supports] - if not result: - raise ValueError("supports must not be empty") - for n in result: - if math.isnan(n) or math.isinf(n) or n <= 0.0: - raise ValueError( - "supports must be finite and strictly positive: S_j contains " - f"only clients with N_ij > 0, got {n!r}" - ) - return result def _validate_vectors(prototypes: Sequence[Vector]) -> int: @@ -89,75 +56,61 @@ def _validate_vectors(prototypes: Sequence[Vector]) -> int: return dim -def beta_weights(supports: Sequence[float], beta: float) -> list[float]: - """Return the unnormalized weights ``N_ij ** beta``.""" - beta = _validate_beta(beta) - return [n ** beta for n in _validate_supports(supports)] - - -def aggregation_weights( - prototypes: Sequence[Vector], - mode: AggregationMode | str, - beta: Optional[float] = None, - supports: Optional[Sequence[float]] = None, -) -> list[float]: - """Return normalized aggregation weights that sum to 1. - - ``prototypes`` is used for length/dimension validation and to mirror the - required shape of the weight vector. Weights are returned in the same client - order as ``prototypes``. - """ - _validate_vectors(prototypes) - mode = AggregationMode(mode) - - if mode is AggregationMode.ORIGINAL_PROTEAN: - raise NotImplementedError( - "original_protean aggregation semantics are not yet verified. " - "Missing-class behavior (N_ij == 0) is undefined in the paper and no " - "official implementation was found. Verify the baseline first; see " - "docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md." - ) - - if mode is AggregationMode.SUPPORT_WEIGHTED: - weights = beta_weights(_validate_supports(supports), _validate_beta(beta)) - else: # EQUAL_AVAILABLE - weights = [1.0] * len(prototypes) - - total = sum(weights) - if not math.isfinite(total) or total <= 0.0: - raise ValueError(f"aggregation weights must sum to a positive value, got {total!r}") - return [w / total for w in weights] - - def aggregate_prototypes( prototypes: Sequence[Vector], - mode: AggregationMode | str, - beta: Optional[float] = None, - supports: Optional[Sequence[float]] = None, + mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, ) -> list[float]: - """Aggregate ``C_ij`` for one class into ``C_bar_j``. + """Aggregate the local prototypes ``C_ij`` of one class into ``C_bar_j``. Parameters ---------- prototypes: Local prototypes ``C_ij`` for the contributing clients in ``S_j``. mode: - One of :class:`AggregationMode`. - beta: - Weighting exponent, required for ``support_weighted`` (and meaningful - for ``equal_available`` as a documented ``beta == 0`` equivalent). - supports: - Sample counts ``N_ij`` (> 0), required for ``support_weighted``. + Only :attr:`AggregationMode.ORIGINAL_PROTEAN` is implemented in this + baseline checkpoint. Other modes raise ``NotImplementedError``. + + The baseline uses equal weights over the supplied contributor set, so the + result is the arithmetic mean of ``prototypes``. """ + # ``AggregationMode`` contains only the baseline mode, so any other value + # raises ``ValueError`` here. This prevents the proposed support-weighted + # aggregation from being used before the baseline is verified. + AggregationMode(mode) dim = _validate_vectors(prototypes) - if supports is not None and len(supports) != len(prototypes): - raise ValueError( - f"supports length ({len(supports)}) must match prototypes " - f"({len(prototypes)})" - ) - weights = aggregation_weights(prototypes, mode, beta=beta, supports=supports) + count = len(prototypes) result = [0.0] * dim - for weight, vec in zip(weights, prototypes): + for vec in prototypes: for d in range(dim): - result[d] += weight * float(vec[d]) + result[d] += float(vec[d]) / count + return result + + +def aggregate_class_prototypes( + contributions, + mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, +): + """Aggregate per-class local prototypes into global prototypes. + + ``contributions`` maps ``class_id -> sequence of (C_ij, N_ij)``. The + contributing set ``S_j`` is constructed here by dropping any entry with a + missing prototype (``None``) or ``N_ij <= 0``; a missing class is never + fabricated. Classes with no valid contributor are omitted from the result. + + Returns ``{class_id: list[float]}`` with one entry per aggregated class. + """ + if not isinstance(contributions, Mapping): + raise TypeError("contributions must be a mapping class_id -> [(C_ij, N_ij), ...]") + result = {} + for class_id in sorted(contributions, key=lambda c: int(c)): + member_prototypes: list[Vector] = [] + for proto, support in contributions[class_id]: + if proto is None: + continue + if support is None or float(support) <= 0.0: + continue + member_prototypes.append(proto) + if not member_prototypes: + continue + result[int(class_id)] = aggregate_prototypes(member_prototypes, mode) return result diff --git a/src/protean_research/data/__init__.py b/src/protean_research/data/__init__.py new file mode 100644 index 0000000..0bc57f9 --- /dev/null +++ b/src/protean_research/data/__init__.py @@ -0,0 +1 @@ +"""Data loading, preprocessing, splitting and non-IID partitioning.""" diff --git a/src/protean_research/data/characterization.py b/src/protean_research/data/characterization.py new file mode 100644 index 0000000..f05b4a4 --- /dev/null +++ b/src/protean_research/data/characterization.py @@ -0,0 +1,66 @@ +"""Non-IID characterization of a partition's class-support matrix ``N_ij``.""" + +from __future__ import annotations + +from typing import Sequence + + +def characterize_support(support, class_names: Sequence[str]) -> list[dict]: + """Per-class inequality statistics of ``N_ij``. + + Returns one row per class with totals, contributor counts and the inequality + measures required by the protocol (min/max/mean/std/CV/max-min ratio). + """ + import numpy as np + + support = np.asarray(support) + if support.ndim != 2: + raise ValueError("support must be a 2D [clients, classes] matrix") + if support.shape[1] != len(class_names): + raise ValueError("class_names length must match support columns") + + rows: list[dict] = [] + for c, name in enumerate(class_names): + column = support[:, c] + positive = column[column > 0] + n_clients = int((column > 0).sum()) + mean = float(positive.mean()) if positive.size else 0.0 + std = float(positive.std(ddof=0)) if positive.size else 0.0 + cv = float(std / mean) if mean > 0 else 0.0 + max_min = ( + float(positive.max() / positive.min()) if positive.size and positive.min() > 0 else 0.0 + ) + rows.append( + { + "class_id": c, + "class_name": name, + "total": int(column.sum()), + "num_clients": n_clients, + "n_missing_clients": int((column == 0).sum()), + "min": int(positive.min()) if positive.size else 0, + "max": int(positive.max()) if positive.size else 0, + "mean": mean, + "std": std, + "cv": cv, + "max_min_ratio": max_min, + } + ) + return rows + + +def client_totals(support, client_ids: Sequence[int] | None = None) -> list[dict]: + """Per-client total sample counts and number of missing classes.""" + import numpy as np + + support = np.asarray(support) + rows: list[dict] = [] + for client in range(support.shape[0]): + row = support[client] + rows.append( + { + "client_id": int(client_ids[client]) if client_ids else client, + "n_samples": int(row.sum()), + "n_missing_classes": int((row == 0).sum()), + } + ) + return rows diff --git a/src/protean_research/data/loading.py b/src/protean_research/data/loading.py new file mode 100644 index 0000000..da88a74 --- /dev/null +++ b/src/protean_research/data/loading.py @@ -0,0 +1,225 @@ +"""Raw dataset loading and cleaning (read-only with respect to the source CSV). + +The raw datasets are external, immutable source data. This module only reads +them. Cleaning here is limited to label canonicalisation, duplicate removal and +identification of feature columns; numeric/categorical encoding lives in +:mod:`protean_research.data.preprocessing`. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping + +from ..utils.config import ( + expand_path, + load_datasets_config, + resolve_raw_path, +) + +LABEL_ALIAS_KEYS = ("fine_grained", "binary", "tool") + + +def canonical_label(value: object) -> str: + """Canonicalise a class label. + + Strips surrounding whitespace, collapses internal whitespace runs and + removes whitespace around underscores. This fixes the X-IIoTID typo + ``"Lateral _movement"`` -> ``"Lateral_movement"``. + """ + text = str(value).strip() + text = re.sub(r"\s+", " ", text) + text = re.sub(r"\s*_\s*", "_", text) + return text + + +def _string_tuple(values, field: str) -> tuple[str, ...]: + result = [] + for value in values: + if not isinstance(value, str): + raise TypeError( + f"{field} entries must be strings; got {value!r}. " + "Quote YAML values that contain ':' (e.g. \"Unnamed: 0\")." + ) + result.append(value) + return tuple(result) + + +@dataclass(frozen=True) +class DatasetSpec: + """Validated configuration for one dataset.""" + + key: str + name: str + label_column: str + label_aliases: tuple[str, ...] + benign_label: str + drop_labels: tuple[str, ...] + missing_tokens: tuple[str, ...] + drop_duplicate_rows: bool + categorical_columns: tuple[str, ...] + non_feature_columns: tuple[str, ...] + raw_path: Path + env_override: str | None = None + aliases: Mapping[str, str] = field(default_factory=dict) + + @property + def non_feature_set(self) -> set[str]: + return ( + set(self.non_feature_columns) + | {self.label_column} + | set(self.label_aliases) + ) + + +def list_dataset_keys(datasets_cfg: Mapping[str, Any] | None = None) -> list[str]: + cfg = datasets_cfg or load_datasets_config() + return list(cfg.get("datasets", {}).keys()) + + +def load_spec( + key: str, + datasets_cfg: Mapping[str, Any] | None = None, +) -> DatasetSpec: + """Build a :class:`DatasetSpec` for dataset *key*. + + Raises ``KeyError`` with a clear message if the key or a required field is + missing, rather than silently defaulting. + """ + cfg = datasets_cfg or load_datasets_config() + datasets = cfg.get("datasets", {}) + if key not in datasets: + raise KeyError( + f"unknown dataset {key!r}; available: {sorted(datasets)}" + ) + entry = datasets[key] + aliases = entry.get("label_column_aliases", {}) or {} + alias_values = tuple( + str(aliases[k]) for k in LABEL_ALIAS_KEYS if k in aliases + ) + spec = DatasetSpec( + key=key, + name=str(entry.get("name", key)), + label_column=str(entry["label_column"]), + label_aliases=alias_values, + benign_label=canonical_label(entry["benign_label"]), + drop_labels=tuple(canonical_label(v) for v in entry.get("drop_labels", [])), + missing_tokens=tuple(str(t) for t in entry.get("missing_tokens", [])), + drop_duplicate_rows=bool(entry.get("drop_duplicate_rows", True)), + categorical_columns=_string_tuple(entry.get("categorical_columns", []), "categorical_columns"), + non_feature_columns=_string_tuple(entry.get("non_feature_columns", []), "non_feature_columns"), + raw_path=resolve_raw_path(entry), + env_override=entry.get("env_override"), + aliases=aliases, + ) + return spec + + +def resolve_spec_raw_path(spec: DatasetSpec) -> Path: + """Return the resolved raw path, honouring the env override.""" + import os + + if spec.env_override and os.environ.get(spec.env_override): + return expand_path(os.environ[spec.env_override]) + return spec.raw_path + + +@dataclass +class CleanedData: + """Cleaned dataset: features, integer labels, class metadata.""" + + spec: DatasetSpec + features: Any # pandas.DataFrame + labels: Any # numpy.ndarray[int64] + class_names: list[str] + benign_index: int + feature_columns: list[str] + categorical_columns: list[str] + numeric_columns: list[str] + n_rows_raw: int + n_duplicates_dropped: int + n_rows_dropped_label: int + + +def read_raw(spec: DatasetSpec, nrows: int | None = None): + """Read the raw CSV. Pass ``nrows`` only for smoke tests / inspection.""" + import pandas as pd + + path = resolve_spec_raw_path(spec) + if not path.exists(): + raise FileNotFoundError( + f"raw dataset for {spec.key!r} not found at {path}. " + "Set the environment override or fix configs/datasets/datasets.yaml." + ) + frame = pd.read_csv(path, low_memory=False, nrows=nrows) + # Normalise the unnamed first column header pandas assigns to 5G-NIDD. + frame.columns = [str(c).strip() for c in frame.columns] + return frame + + +def clean_frame(frame, spec: DatasetSpec) -> CleanedData: + """Apply the frozen cleaning recipe and return a :class:`CleanedData`.""" + import numpy as np + import pandas as pd + + if spec.label_column not in frame.columns: + raise KeyError( + f"label column {spec.label_column!r} not found in {spec.key!r} dataset" + ) + + n_rows_raw = int(len(frame)) + duplicates = int(frame.duplicated().sum()) + if spec.drop_duplicate_rows: + frame = frame.drop_duplicates(keep="first").reset_index(drop=True) + + labels = frame[spec.label_column].map(canonical_label) + mask = ~labels.isin(set(spec.drop_labels)) + n_dropped_label = int((~mask).sum()) + frame = frame.loc[mask].reset_index(drop=True) + labels = labels.loc[mask].reset_index(drop=True) + + feature_columns = [ + c for c in frame.columns if c not in spec.non_feature_set + ] + class_names = sorted(labels.unique().tolist()) + if spec.benign_label not in class_names: + raise ValueError( + f"benign label {spec.benign_label!r} absent from {spec.key!r} after cleaning; " + f"present classes: {class_names}" + ) + class_to_index = {name: i for i, name in enumerate(class_names)} + labels_int = labels.map(class_to_index).to_numpy(dtype=np.int64) + + categorical_columns = [ + c for c in spec.categorical_columns if c in feature_columns + ] + numeric_columns = [ + c for c in feature_columns if c not in set(categorical_columns) + ] + + return CleanedData( + spec=spec, + features=frame[feature_columns].reset_index(drop=True), + labels=labels_int, + class_names=class_names, + benign_index=class_to_index[spec.benign_label], + feature_columns=feature_columns, + categorical_columns=categorical_columns, + numeric_columns=numeric_columns, + n_rows_raw=n_rows_raw, + n_duplicates_dropped=duplicates, + n_rows_dropped_label=n_dropped_label, + ) + + +def load_cleaned( + key: str, + datasets_cfg: Mapping[str, Any] | None = None, + nrows: int | None = None, +) -> CleanedData: + """Convenience: resolve spec, read raw and clean in one call.""" + spec = load_spec(key, datasets_cfg) + frame = read_raw(spec, nrows=nrows) + return clean_frame(frame, spec) diff --git a/src/protean_research/data/manifest.py b/src/protean_research/data/manifest.py new file mode 100644 index 0000000..46fcfc7 --- /dev/null +++ b/src/protean_research/data/manifest.py @@ -0,0 +1,159 @@ +"""Partition manifest persistence and exact reproduction. + +Manifests are small, deterministic, version-controlled JSON records. They store +the partition recipe (algorithm id + RNG seed), the split description and the +resulting class-support matrix ``N_ij`` plus a content hash. Full sample-index +arrays are regenerated deterministically and verified against the stored hash, +which keeps manifests small while making reload exact. +""" + +from __future__ import annotations + +import json +import platform +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from ..utils.integrity import hash_index_arrays, hash_json +from . import partition as partition_mod +from . import splits as splits_mod + +MANIFEST_VERSION = "1" + + +def manifest_path(partition_dir: str | Path, dataset_key: str, alpha: float, seed: int) -> Path: + return Path(partition_dir) / f"{dataset_key}__alpha{alpha:g}__seed{seed}.json" + + +def build_manifest( + *, + dataset_key: str, + dataset_name: str, + alpha: float, + seed: int, + num_clients: int, + class_names: list[str], + benign_class: str, + labels, + split: splits_mod.SplitIndices, + assignments, + rng_seed: int, + train_fraction: float, + validation_fraction: float, + test_fraction: float, + config_hash: str | None = None, + preprocessing_hash: str | None = None, +) -> dict[str, Any]: + import numpy as np + + labels = np.asarray(labels) + num_classes = len(class_names) + support = partition_mod.support_matrix( + assignments, labels[split.train], num_classes + ) + manifest = { + "manifest_version": MANIFEST_VERSION, + "dataset": dataset_key, + "dataset_name": dataset_name, + "alpha": float(alpha), + "seed": int(seed), + "num_clients": int(num_clients), + "num_classes": num_classes, + "class_names": list(class_names), + "benign_class": benign_class, + "algorithm": partition_mod.ALGORITHM, + "rng_seed": int(rng_seed), + "num_train": int(len(split.train)), + "support": support.astype(int).tolist(), + "partition_hash": partition_mod.partition_hash(assignments), + "split": { + "strategy": "stratified_by_class", + "seed": int(seed), + "train_fraction": float(train_fraction), + "validation_fraction": float(validation_fraction), + "test_fraction": float(test_fraction), + "sizes": split.sizes, + "hash": hash_index_arrays([split.train, split.validation, split.test]), + }, + "preprocessing_hash": preprocessing_hash, + "config_hash": config_hash, + "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "python": platform.python_version(), + } + return manifest + + +def save_manifest(path: str | Path, manifest: Mapping[str, Any]) -> Path: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(manifest, handle, indent=2, sort_keys=True) + handle.write("\n") + return path + + +def load_manifest(path: str | Path) -> dict[str, Any]: + with Path(path).open("r", encoding="utf-8") as handle: + manifest = json.load(handle) + if manifest.get("manifest_version") != MANIFEST_VERSION: + raise ValueError( + f"unsupported manifest version {manifest.get('manifest_version')!r}" + ) + return manifest + + +def reproduce_split(manifest: Mapping[str, Any], labels): + """Recompute the split described by *manifest* and verify its hash.""" + split_cfg = manifest["split"] + split = splits_mod.stratified_split( + labels, + dataset_key=str(manifest["dataset"]), + seed=int(split_cfg["seed"]), + train_fraction=float(split_cfg["train_fraction"]), + validation_fraction=float(split_cfg["validation_fraction"]), + test_fraction=float(split_cfg["test_fraction"]), + ) + expected_hash = split_cfg["hash"] + actual_hash = hash_index_arrays([split.train, split.validation, split.test]) + if actual_hash != expected_hash: + raise ValueError( + "split mismatch: manifest does not reproduce on this dataset; " + "the raw data or split recipe changed" + ) + if split.sizes != split_cfg["sizes"]: + raise ValueError( + f"split sizes mismatch: {split.sizes} != {split_cfg['sizes']}" + ) + return split + + +def reproduce_train_assignments(manifest: Mapping[str, Any], labels): + """Regenerate train assignments and verify ``N_ij`` and content hash.""" + import numpy as np + + split = reproduce_split(manifest, labels) + labels_train = np.asarray(labels)[split.train] + class_indices = partition_mod.class_index_map(labels_train, int(manifest["num_classes"])) + assignments = partition_mod.dirichlet_partition( + class_indices, + int(manifest["num_clients"]), + float(manifest["alpha"]), + int(manifest["rng_seed"]), + ) + support = partition_mod.support_matrix( + assignments, labels_train, int(manifest["num_classes"]) + ) + expected_support = np.asarray(manifest["support"], dtype=np.int64) + if not np.array_equal(support, expected_support): + raise ValueError("support matrix N_ij mismatch on manifest reload") + if partition_mod.partition_hash(assignments) != manifest["partition_hash"]: + raise ValueError("partition content hash mismatch on manifest reload") + return split, assignments, support + + +def manifest_content_hash(manifest: Mapping[str, Any]) -> str: + payload = dict(manifest) + payload.pop("created_utc", None) + payload.pop("python", None) + return hash_json(payload) diff --git a/src/protean_research/data/partition.py b/src/protean_research/data/partition.py new file mode 100644 index 0000000..797fca0 --- /dev/null +++ b/src/protean_research/data/partition.py @@ -0,0 +1,129 @@ +"""Dirichlet non-IID client partitioning. + +Frozen algorithm (``docs/research/EXPERIMENT_PROTOCOL.md`` §2): for each class, +draw client proportions from ``Dirichlet(alpha)`` and multinomial-allocate that +class's samples. Partitions are generated from the training split only and are +persisted in a manifest; a beta run must load the manifest, never regenerate. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ..utils.integrity import hash_index_arrays +from ..utils.seeding import derive_seed + +ALGORITHM = "per_class_dirichlet_multinomial" + + +def partition_rng_seed(dataset_key: str, alpha: float, seed: int, tag: str = "dirichlet") -> int: + """Deterministic RNG seed for a ``(dataset, alpha, seed)`` partition.""" + return derive_seed(dataset_key, tag, f"alpha={alpha}", f"seed={seed}") + + +def class_index_map(labels, num_classes: int) -> dict[int, object]: + """Map class id -> positional indices within *labels*.""" + import numpy as np + + labels = np.asarray(labels) + return { + c: np.flatnonzero(labels == c).astype(np.int64) + for c in range(num_classes) + } + + +def dirichlet_partition( + class_indices: dict[int, object], + num_clients: int, + alpha: float, + rng_seed: int, +) -> list[object]: + """Allocate samples to clients and return a client->indices list. + + ``class_indices`` maps class id -> positions. The output list has length + ``num_clients``; each element is an ``int64`` array (possibly empty). + """ + import numpy as np + + if num_clients <= 0: + raise ValueError("num_clients must be positive") + if alpha <= 0: + raise ValueError("alpha must be positive") + + rng = np.random.default_rng(rng_seed) + buckets: list[list[np.ndarray]] = [[] for _ in range(num_clients)] + + for class_id in sorted(class_indices): + positions = np.asarray(class_indices[class_id], dtype=np.int64) + n = len(positions) + if n == 0: + continue + shuffled = positions[rng.permutation(n)] + proportions = rng.dirichlet(np.full(num_clients, alpha, dtype=float)) + counts = rng.multinomial(n, proportions) + start = 0 + for client in range(num_clients): + k = int(counts[client]) + if k > 0: + buckets[client].append(shuffled[start : start + k]) + start += k + + return [ + ( + np.sort(np.concatenate(parts)).astype(np.int64, copy=False) + if parts + else np.empty(0, dtype=np.int64) + ) + for parts in buckets + ] + + +def support_matrix(assignments, labels, num_classes: int): + """Return the ``[num_clients, num_classes]`` count matrix ``N_ij``.""" + import numpy as np + + labels = np.asarray(labels) + matrix = np.zeros((len(assignments), num_classes), dtype=np.int64) + for client, idx in enumerate(assignments): + if len(idx): + counts = np.bincount(labels[idx], minlength=num_classes) + matrix[client, : len(counts)] = counts + return matrix + + +def partition_hash(assignments) -> str: + return hash_index_arrays(assignments) + + +def partition_eval_split( + labels, + num_clients: int, + alpha: float, + dataset_key: str, + seed: int, + split_name: str, +): + """Partition a validation/test split into per-client local eval sets. + + The baseline evaluation is per client (each client scores its local held-out + data). This uses the same frozen Dirichlet algorithm with a split-specific + seed, so it is deterministic and paired across variants. + """ + import numpy as np + + labels = np.asarray(labels) + # Labels are globally encoded; class space size is max(label) + 1. + num_classes = int(labels.max()) + 1 if len(labels) else 0 + class_indices = class_index_map(labels, num_classes) + rng_seed = derive_seed(dataset_key, f"eval_partition:{split_name}", f"alpha={alpha}", f"seed={seed}") + return dirichlet_partition(class_indices, num_clients, alpha, rng_seed), rng_seed + + +@dataclass(frozen=True) +class PartitionView: + """Convenience bundle for a generated partition.""" + + assignments: list + support: object # np.ndarray [clients, classes] + rng_seed: int + content_hash: str diff --git a/src/protean_research/data/pipeline.py b/src/protean_research/data/pipeline.py new file mode 100644 index 0000000..7694f33 --- /dev/null +++ b/src/protean_research/data/pipeline.py @@ -0,0 +1,131 @@ +"""Deterministic data pipeline: raw CSV -> cleaned -> split -> preprocess -> partition. + +This module wires the frozen protocol together so that experiment runners do not +duplicate orchestration logic. It never modifies the raw datasets. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from ..utils.integrity import hash_json +from . import loading, manifest as manifest_mod, partition as partition_mod, splits as splits_mod +from .preprocessing import Preprocessor, fit_preprocessor, transform + +TRAIN_FRACTION = 0.70 +VALIDATION_FRACTION = 0.10 +TEST_FRACTION = 0.20 + + +@dataclass +class DatasetBundle: + """Fully prepared dataset for a fixed ``(dataset, seed)``.""" + + spec: loading.DatasetSpec + cleaned: loading.CleanedData + split: splits_mod.SplitIndices + preprocessor: Preprocessor + features: Any # float32 matrix for all cleaned rows + train_fraction: float = TRAIN_FRACTION + validation_fraction: float = VALIDATION_FRACTION + test_fraction: float = TEST_FRACTION + + @property + def class_names(self) -> list[str]: + return list(self.cleaned.class_names) + + @property + def num_classes(self) -> int: + return len(self.cleaned.class_names) + + @property + def labels(self): + return self.cleaned.labels + + +def prepare_dataset( + dataset_key: str, + *, + seed: int, + datasets_cfg: Mapping[str, Any] | None = None, + nrows: int | None = None, + raw_frame: Any | None = None, +) -> DatasetBundle: + """Run the deterministic, leak-free preprocessing pipeline. + + ``nrows`` is a smoke-test-only override that limits how many raw rows are + read; ``raw_frame`` lets a smoke caller pass an already-subsampled frame. + Neither must be used for a scientific run. + """ + spec = loading.load_spec(dataset_key, datasets_cfg) + frame = raw_frame if raw_frame is not None else loading.read_raw(spec, nrows=nrows) + cleaned = loading.clean_frame(frame, spec) + split = splits_mod.stratified_split( + cleaned.labels, + dataset_key=dataset_key, + seed=seed, + train_fraction=TRAIN_FRACTION, + validation_fraction=VALIDATION_FRACTION, + test_fraction=TEST_FRACTION, + ) + splits_mod.check_disjoint(split, n_rows=len(cleaned.labels)) + preprocessor = fit_preprocessor(cleaned, split.train) + features = transform(preprocessor, cleaned) + return DatasetBundle( + spec=spec, + cleaned=cleaned, + split=split, + preprocessor=preprocessor, + features=features, + ) + + +def create_partition( + bundle: DatasetBundle, + *, + alpha: float, + seed: int, + num_clients: int, + partition_dir: str | Path, + preprocessing_hash: str | None = None, + config_hash: str | None = None, + save: bool = True, +) -> tuple[dict, list, Any]: + """Create and (optionally) persist the partition manifest for one condition.""" + import numpy as np + + labels_train = np.asarray(bundle.labels)[bundle.split.train] + class_indices = partition_mod.class_index_map(labels_train, bundle.num_classes) + rng_seed = partition_mod.partition_rng_seed(bundle.spec.key, alpha, seed) + assignments = partition_mod.dirichlet_partition( + class_indices, num_clients, alpha, rng_seed + ) + manifest = manifest_mod.build_manifest( + dataset_key=bundle.spec.key, + dataset_name=bundle.spec.name, + alpha=alpha, + seed=seed, + num_clients=num_clients, + class_names=bundle.class_names, + benign_class=bundle.spec.benign_label, + labels=bundle.labels, + split=bundle.split, + assignments=assignments, + rng_seed=rng_seed, + train_fraction=bundle.train_fraction, + validation_fraction=bundle.validation_fraction, + test_fraction=bundle.test_fraction, + preprocessing_hash=preprocessing_hash, + config_hash=config_hash, + ) + if save: + path = manifest_mod.manifest_path(partition_dir, bundle.spec.key, alpha, seed) + manifest_mod.save_manifest(path, manifest) + support = np.asarray(manifest["support"], dtype=np.int64) + return manifest, assignments, support + + +def preprocessing_content_hash(preprocessor: Preprocessor) -> str: + return hash_json(preprocessor.to_dict()) diff --git a/src/protean_research/data/preprocessing.py b/src/protean_research/data/preprocessing.py new file mode 100644 index 0000000..c66d1d3 --- /dev/null +++ b/src/protean_research/data/preprocessing.py @@ -0,0 +1,187 @@ +"""Deterministic, leak-free feature preprocessing. + +The frozen recipe is described in ``docs/research/EXPERIMENT_PROTOCOL.md`` §13. +All preprocessing statistics (medians, means, stds, one-hot categories) are +fitted on the **train split only** and then applied unchanged to validation and +test. The fitted state is serialisable so future rounds and beta variants reuse +exactly the same transform. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Sequence + +MISSING_CATEGORY = "__MISSING__" + + +@dataclass +class Preprocessor: + numeric_columns: list[str] + categorical_columns: list[str] + medians: dict[str, float] + means: dict[str, float] + stds: dict[str, float] + categories: dict[str, list[str]] + output_columns: list[str] = field(default_factory=list) + + # ---- persistence ------------------------------------------------- + def to_dict(self) -> dict[str, Any]: + return { + "numeric_columns": self.numeric_columns, + "categorical_columns": self.categorical_columns, + "medians": self.medians, + "means": self.means, + "stds": self.stds, + "categories": self.categories, + "output_columns": self.output_columns, + } + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "Preprocessor": + return cls( + numeric_columns=list(payload["numeric_columns"]), + categorical_columns=list(payload["categorical_columns"]), + medians=dict(payload["medians"]), + means=dict(payload["means"]), + stds=dict(payload["stds"]), + categories={k: list(v) for k, v in payload["categories"].items()}, + output_columns=list(payload.get("output_columns", [])), + ) + + def save(self, path: str | Path) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(self.to_dict(), handle, indent=2, sort_keys=True) + + @classmethod + def load(cls, path: str | Path) -> "Preprocessor": + with Path(path).open("r", encoding="utf-8") as handle: + return cls.from_dict(json.load(handle)) + + +def _replace_missing(series, missing_tokens: Sequence[str]): + """Map configured missing tokens to NaN.""" + import numpy as np + + if not missing_tokens: + return series + result = series.astype(object) + for token in missing_tokens: + result = result.mask(result == token, np.nan) + return result + + +def _output_columns(pre: Preprocessor) -> list[str]: + cols = list(pre.numeric_columns) + for col in pre.categorical_columns: + for cat in pre.categories[col]: + cols.append(f"{col}=={cat}") + return cols + + +def fit_preprocessor(cleaned, train_idx) -> Preprocessor: + """Fit preprocessing state on the train split only. + + ``cleaned`` is a :class:`~protean_research.data.loading.CleanedData`; + ``train_idx`` are positional indices into ``cleaned.features``. + """ + import numpy as np + import pandas as pd + + features = cleaned.features + spec = cleaned.spec + train = features.iloc[train_idx] + + medians: dict[str, float] = {} + means: dict[str, float] = {} + stds: dict[str, float] = {} + + for col in cleaned.numeric_columns: + values = pd.to_numeric( + _replace_missing(train[col], spec.missing_tokens), errors="coerce" + ).astype("float64") + median = float(np.nanmedian(values.to_numpy())) if values.notna().any() else 0.0 + median = 0.0 if not np.isfinite(median) else median + filled = values.fillna(median).to_numpy(dtype="float64") + mean = float(filled.mean()) if filled.size else 0.0 + std = float(filled.std(ddof=0)) if filled.size else 0.0 + if not np.isfinite(std) or std == 0.0: + std = 1.0 + medians[col] = median + means[col] = mean + stds[col] = std + + categories: dict[str, list[str]] = {} + for col in cleaned.categorical_columns: + values = _replace_missing(train[col], spec.missing_tokens) + observed = {str(v) for v in values.dropna().unique().tolist()} + cats = sorted(observed) + if values.isna().any(): + cats.append(MISSING_CATEGORY) + categories[col] = cats + + pre = Preprocessor( + numeric_columns=list(cleaned.numeric_columns), + categorical_columns=list(cleaned.categorical_columns), + medians=medians, + means=means, + stds=stds, + categories=categories, + ) + pre.output_columns = _output_columns(pre) + return pre + + +def transform(pre: Preprocessor, cleaned, indices=None): + """Apply fitted preprocessing and return a ``float32`` matrix. + + ``indices`` selects rows of ``cleaned.features`` (all rows if ``None``). + """ + import numpy as np + import pandas as pd + + features = cleaned.features + spec = cleaned.spec + frame = features if indices is None else features.iloc[indices] + n = len(frame) + + blocks: list[np.ndarray] = [] + for col in pre.numeric_columns: + values = pd.to_numeric( + _replace_missing(frame[col], spec.missing_tokens), errors="coerce" + ).astype("float64") + values = values.fillna(pre.medians[col]).to_numpy(dtype="float64") + values = (values - pre.means[col]) / pre.stds[col] + blocks.append(values.astype(np.float32).reshape(n, 1)) + + for col in pre.categorical_columns: + raw = _replace_missing(frame[col], spec.missing_tokens) + missing = raw.isna().to_numpy() + text = raw.astype("object").where(~missing, other=None) + cats = pre.categories[col] + index = {cat: i for i, cat in enumerate(cats)} + onehot = np.zeros((n, len(cats)), dtype=np.float32) + for row, value in enumerate(text.to_numpy()): + if value is None: + if MISSING_CATEGORY in index: + onehot[row, index[MISSING_CATEGORY]] = 1.0 + continue + j = index.get(str(value)) + if j is not None: + onehot[row, j] = 1.0 + blocks.append(onehot) + + if not blocks: + raise ValueError("no feature columns available after preprocessing") + matrix = np.concatenate(blocks, axis=1) + return np.ascontiguousarray(matrix, dtype=np.float32) + + +def fit_transform(cleaned, train_idx): + """Fit on ``train_idx`` and return ``(preprocessor, full_matrix)``.""" + pre = fit_preprocessor(cleaned, train_idx) + return pre, transform(pre, cleaned) diff --git a/src/protean_research/data/rare_unseen.py b/src/protean_research/data/rare_unseen.py new file mode 100644 index 0000000..41597b6 --- /dev/null +++ b/src/protean_research/data/rare_unseen.py @@ -0,0 +1,92 @@ +"""Rare-class and unseen-class definitions. + +Frozen research definitions (``planningawal/PROTEAN_Revisi_Lurus.md`` §9): +per client, rare classes are the two attack classes with the smallest positive +training support; unseen classes are attack classes with ``N_ij = 0``. The +benign class is never rare or unseen. Rare and unseen are never merged. +""" + +from __future__ import annotations + +from typing import Sequence + + +def attack_class_ids(num_classes: int, benign_index: int) -> list[int]: + return [c for c in range(num_classes) if c != benign_index] + + +def rare_class_ids_for_client( + support_row: Sequence[int], + benign_index: int, + max_rare: int = 2, +) -> list[int]: + """Two smallest positive-support attack classes (ties broken by class id).""" + support_row = list(support_row) + candidates = [ + (int(support_row[c]), c) + for c in range(len(support_row)) + if c != benign_index and int(support_row[c]) > 0 + ] + candidates.sort(key=lambda item: (item[0], item[1])) + return [c for _, c in candidates[:max_rare]] + + +def unseen_class_ids_for_client( + support_row: Sequence[int], + benign_index: int, +) -> list[int]: + support_row = list(support_row) + return [ + c + for c in range(len(support_row)) + if c != benign_index and int(support_row[c]) == 0 + ] + + +def rare_unseen_rows( + support, + class_names: Sequence[str], + benign_index: int, + dataset_key: str, + alpha: float, + seed: int, +) -> list[dict]: + """One row per ``(client, class)`` with rare/unseen flags and support.""" + import numpy as np + + support = np.asarray(support) + num_classes = support.shape[1] + rows: list[dict] = [] + for client in range(support.shape[0]): + row = support[client] + rare = set(rare_class_ids_for_client(row, benign_index)) + unseen = set(unseen_class_ids_for_client(row, benign_index)) + for c in range(num_classes): + rows.append( + { + "dataset": dataset_key, + "alpha": float(alpha), + "seed": int(seed), + "client_id": client, + "class_id": c, + "class_name": class_names[c], + "N_ij": int(row[c]), + "rare_flag": c in rare, + "unseen_flag": c in unseen, + } + ) + return rows + + +def rare_class_ids( + support, + benign_index: int, +) -> list[list[int]]: + """Per-client list of rare class ids.""" + import numpy as np + + support = np.asarray(support) + return [ + rare_class_ids_for_client(support[client], benign_index) + for client in range(support.shape[0]) + ] diff --git a/src/protean_research/data/splits.py b/src/protean_research/data/splits.py new file mode 100644 index 0000000..7f16b2a --- /dev/null +++ b/src/protean_research/data/splits.py @@ -0,0 +1,109 @@ +"""Deterministic stratified train/validation/test splitting. + +Frozen protocol (``docs/research/EXPERIMENT_PROTOCOL.md`` §3): a single +stratified 70/10/20 split per ``(dataset, seed)``, reused by every aggregation +variant and beta candidate. Test data never influences preprocessing, model +selection or beta selection. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ..utils.seeding import numpy_rng + + +@dataclass(frozen=True) +class SplitIndices: + """Positional indices into the cleaned dataset for each split.""" + + train: object # np.ndarray[int64] + validation: object + test: object + + @property + def sizes(self) -> dict[str, int]: + return { + "train": int(len(self.train)), + "validation": int(len(self.validation)), + "test": int(len(self.test)), + } + + +def stratified_split( + labels, + dataset_key: str, + seed: int, + train_fraction: float = 0.70, + validation_fraction: float = 0.10, + test_fraction: float = 0.20, +) -> SplitIndices: + """Return a deterministic per-class stratified split. + + Every class is shuffled with the seeded RNG and split into train/validation/ + test so that all classes are represented in validation and test, which the + rare-class evaluation requires. + """ + import numpy as np + + fractions = np.array([train_fraction, validation_fraction, test_fraction], dtype=float) + if fractions.min() < 0 or abs(fractions.sum() - 1.0) > 1e-9: + raise ValueError( + "train/validation/test fractions must be non-negative and sum to 1, " + f"got {fractions.tolist()}" + ) + + labels = np.asarray(labels) + rng = numpy_rng(dataset_key, "split", f"seed={seed}") + train_parts: list[np.ndarray] = [] + val_parts: list[np.ndarray] = [] + test_parts: list[np.ndarray] = [] + + for class_id in sorted(np.unique(labels).tolist()): + positions = np.flatnonzero(labels == class_id) + positions = positions[rng.permutation(len(positions))] + n = len(positions) + n_val = int(round(n * validation_fraction)) + n_test = int(round(n * test_fraction)) + # Keep at least one sample in train when a class is tiny. + if n_val + n_test >= n and n >= 1: + n_val = min(n_val, n - 1) + n_test = min(n_test, n - n_val - 1) + val_parts.append(positions[:n_val]) + test_parts.append(positions[n_val : n_val + n_test]) + train_parts.append(positions[n_val + n_test :]) + + def _concat(parts): + if not parts: + return np.empty(0, dtype=np.int64) + joined = np.concatenate(parts) + joined.sort() + return joined.astype(np.int64, copy=False) + + return SplitIndices( + train=_concat(train_parts), + validation=_concat(val_parts), + test=_concat(test_parts), + ) + + +def check_disjoint(splits: SplitIndices, n_rows: int | None = None) -> None: + """Raise ``ValueError`` if the three splits overlap or miss rows. + + This is a leakage guard used by tests and the smoke runner. + """ + import numpy as np + + train = np.asarray(splits.train) + val = np.asarray(splits.validation) + test = np.asarray(splits.test) + if len(np.intersect1d(train, val)) or len(np.intersect1d(train, test)) or len( + np.intersect1d(val, test) + ): + raise ValueError("train/validation/test splits overlap") + if n_rows is not None: + union = np.union1d(np.union1d(train, val), test) + if len(union) != n_rows: + raise ValueError( + f"splits cover {len(union)} rows but dataset has {n_rows}" + ) diff --git a/src/protean_research/evaluation/__init__.py b/src/protean_research/evaluation/__init__.py new file mode 100644 index 0000000..5e6c6d1 --- /dev/null +++ b/src/protean_research/evaluation/__init__.py @@ -0,0 +1,17 @@ +"""Evaluation metrics for the baseline and future beta comparisons.""" + +from .metrics import ( + classification_metrics, + confusion_matrix, + mean_metric, + rare_class_metrics, + unseen_class_report, +) + +__all__ = [ + "classification_metrics", + "confusion_matrix", + "rare_class_metrics", + "unseen_class_report", + "mean_metric", +] diff --git a/src/protean_research/evaluation/metrics.py b/src/protean_research/evaluation/metrics.py new file mode 100644 index 0000000..6acbbe5 --- /dev/null +++ b/src/protean_research/evaluation/metrics.py @@ -0,0 +1,127 @@ +"""Classification metrics (pure NumPy, no scikit-learn dependency).""" + +from __future__ import annotations + +from typing import Mapping, Sequence + + +def confusion_matrix(y_true, y_pred, num_classes: int): + import numpy as np + + y_true = np.asarray(y_true, dtype=np.int64) + y_pred = np.asarray(y_pred, dtype=np.int64) + if y_true.shape != y_pred.shape: + raise ValueError("y_true and y_pred must have the same shape") + matrix = np.zeros((num_classes, num_classes), dtype=np.int64) + valid = (y_true >= 0) & (y_true < num_classes) & (y_pred >= 0) & (y_pred < num_classes) + np.add.at(matrix, (y_true[valid], y_pred[valid]), 1) + return matrix + + +def _prf_from_confusion(matrix, class_id: int) -> tuple[float, float, float, int]: + import numpy as np + + tp = float(matrix[class_id, class_id]) + fp = float(matrix[:, class_id].sum() - tp) + fn = float(matrix[class_id, :].sum() - tp) + support = int(matrix[class_id, :].sum()) + precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + f1 = ( + 2 * precision * recall / (precision + recall) + if (precision + recall) > 0 + else 0.0 + ) + return precision, recall, f1, support + + +def classification_metrics(y_true, y_pred, num_classes: int) -> dict: + """Accuracy, macro-F1 (classes with support), weighted-F1 and per-class stats.""" + import numpy as np + + matrix = confusion_matrix(y_true, y_pred, num_classes) + total = int(matrix.sum()) + accuracy = float(np.trace(matrix) / total) if total else 0.0 + + per_class = [] + supports = [] + for c in range(num_classes): + precision, recall, f1, support = _prf_from_confusion(matrix, c) + per_class.append( + { + "class_id": c, + "precision": precision, + "recall": recall, + "f1": f1, + "support": support, + } + ) + supports.append(support) + + supports = np.asarray(supports, dtype=np.float64) + f1s = np.asarray([row["f1"] for row in per_class], dtype=np.float64) + present = supports > 0 + macro_f1 = float(f1s[present].mean()) if present.any() else 0.0 + weighted_f1 = ( + float((f1s * supports).sum() / supports.sum()) if supports.sum() else 0.0 + ) + return { + "accuracy": accuracy, + "macro_f1": macro_f1, + "weighted_f1": weighted_f1, + "n_samples": total, + "per_class": per_class, + "confusion_matrix": matrix.astype(int).tolist(), + } + + +def rare_class_metrics( + y_true, + y_pred, + num_classes: int, + rare_class_ids: Sequence[int], +) -> dict: + """Macro-F1 and recall restricted to *rare_class_ids*. + + Classes with no evaluation samples are skipped. Returns the mean over the + evaluated rare classes together with their support. + """ + import numpy as np + + matrix = confusion_matrix(y_true, y_pred, num_classes) + f1s, recalls, evaluated, supports = [], [], [], [] + for c in sorted(set(int(v) for v in rare_class_ids)): + _, recall, f1, support = _prf_from_confusion(matrix, c) + if support == 0: + continue + f1s.append(f1) + recalls.append(recall) + supports.append(support) + evaluated.append(c) + return { + "rare_class_macro_f1": float(np.mean(f1s)) if f1s else 0.0, + "rare_class_recall": float(np.mean(recalls)) if recalls else 0.0, + "evaluated_rare_classes": evaluated, + "rare_support": int(sum(supports)), + } + + +def unseen_class_report(y_true, y_pred, num_classes: int, unseen_class_ids: Sequence[int]) -> dict: + """Recall over samples whose true class was unseen during training.""" + import numpy as np + + matrix = confusion_matrix(y_true, y_pred, num_classes) + rows = [] + for c in sorted(set(int(v) for v in unseen_class_ids)): + _, recall, f1, support = _prf_from_confusion(matrix, c) + if support == 0: + continue + rows.append( + {"class_id": c, "recall": recall, "f1": f1, "support": support} + ) + return {"unseen_class_rows": rows} + + +def mean_metric(per_client: Sequence[Mapping[str, float]], key: str) -> float: + values = [float(m[key]) for m in per_client if key in m] + return float(sum(values) / len(values)) if values else 0.0 diff --git a/src/protean_research/federated/__init__.py b/src/protean_research/federated/__init__.py new file mode 100644 index 0000000..1ac2de1 --- /dev/null +++ b/src/protean_research/federated/__init__.py @@ -0,0 +1,15 @@ +"""Federated client updates, server aggregation and round orchestration.""" + +from .client import ClientUpdate, local_update +from .round import TrainingConfig, run_federated +from .server import aggregate_models, aggregate_prototypes, predict_global + +__all__ = [ + "ClientUpdate", + "local_update", + "TrainingConfig", + "run_federated", + "aggregate_models", + "aggregate_prototypes", + "predict_global", +] diff --git a/src/protean_research/federated/client.py b/src/protean_research/federated/client.py new file mode 100644 index 0000000..8a64548 --- /dev/null +++ b/src/protean_research/federated/client.py @@ -0,0 +1,130 @@ +"""Local client training (baseline PROTEAN objective, frozen). + +Loss (paper Eq. 3 / thesis Eq. 5.6):: + + CE(logits, y) + lambda * ||phi(x) - Cbar_y||^2 + + (mu / 2) * ||omega_i - omega||^2 + +The prototype-alignment term is applied per sample for classes that currently +have a global prototype (FedProto-style reading of Eq. 3); at round 0 no global +prototypes exist and the term is skipped. Local prototypes are recomputed at the +end of the local round as mean embeddings over the client's class samples +(paper Eq. 1). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Mapping, Optional + +from ..models.protean_cnn import get_parameters, set_parameters +from ..models.prototypes import compute_local_prototypes + + +@dataclass +class ClientUpdate: + client_id: int + parameters: list + prototypes: dict = field(default_factory=dict) + supports: dict = field(default_factory=dict) + n_samples: int = 0 + loss: float = 0.0 + + +def local_update( + client_id: int, + model, + global_parameters, + global_prototypes: Optional[Mapping[int, object]], + features, + labels, + indices, + *, + num_classes: int, + local_epochs: int, + batch_size: int, + learning_rate: float, + mu: float, + lam: float, + seed: int, +) -> ClientUpdate: + import numpy as np + import torch + + if len(indices) == 0: + return ClientUpdate( + client_id=client_id, + parameters=[ + torch.as_tensor(p, dtype=p.dtype).clone() for p in global_parameters + ], + prototypes={}, + supports={}, + n_samples=0, + ) + + device = next(model.parameters()).device + set_parameters(model, global_parameters) + + x = torch.as_tensor(features[indices], dtype=torch.float32, device=device) + y = torch.as_tensor(np.asarray(labels)[indices], dtype=torch.long, device=device) + + global_tensors = [torch.as_tensor(p, dtype=torch.float32, device=device) for p in global_parameters] + proto_tensors = { + int(c): torch.as_tensor(v, dtype=torch.float32, device=device) + for c, v in (global_prototypes or {}).items() + } + + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + generator = torch.Generator(device="cpu") + generator.manual_seed(seed) + n = x.shape[0] + last_loss = 0.0 + + model.train() + for _ in range(local_epochs): + order = torch.randperm(n, generator=generator) + for start in range(0, n, batch_size): + batch_idx = order[start : start + batch_size] + xb = x[batch_idx] + yb = y[batch_idx] + optimizer.zero_grad() + log_prob = model(xb) + loss = torch.nn.functional.nll_loss(log_prob, yb) + + if proto_tensors: + embeddings = model.embedding(xb) + align = torch.zeros((), device=device) + count = 0 + for class_id, proto in proto_tensors.items(): + mask = yb == class_id + if mask.any(): + diff = embeddings[mask] - proto.unsqueeze(0) + align = align + (diff ** 2).sum(dim=1).sum() + count += int(mask.sum().item()) + if count: + loss = loss + lam * align / count + + if mu: + penalty = torch.zeros((), device=device) + for param, target in zip(model.parameters(), global_tensors): + penalty = penalty + ((param - target) ** 2).sum() + loss = loss + (mu / 2.0) * penalty + + loss.backward() + optimizer.step() + last_loss = float(loss.detach().cpu().item()) + + model.eval() + with torch.no_grad(): + embeddings = model.embedding(x).cpu().numpy() + prototypes, supports = compute_local_prototypes( + embeddings, np.asarray(labels)[indices], num_classes + ) + return ClientUpdate( + client_id=client_id, + parameters=get_parameters(model), + prototypes=prototypes, + supports=supports, + n_samples=int(n), + loss=last_loss, + ) diff --git a/src/protean_research/federated/round.py b/src/protean_research/federated/round.py new file mode 100644 index 0000000..5705ed3 --- /dev/null +++ b/src/protean_research/federated/round.py @@ -0,0 +1,158 @@ +"""Federated round orchestration for the baseline PROTEAN pipeline. + +Per round: broadcast the global model and global prototypes, run local updates, +average model parameters (1/M), aggregate class prototypes over ``S_j``, then +evaluate the updated global prototypes on the validation split. The test split is +never touched here. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Mapping, Optional, Sequence + +from ..evaluation.metrics import classification_metrics, rare_class_metrics +from ..models.protean_cnn import ( + ModelConfig, + build_model, + extract_embeddings, + get_parameters, + set_parameters, +) +from ..models.prototypes import nearest_prototype_predict +from ..utils.seeding import derive_seed, seed_everything +from . import client as client_mod +from . import server as server_mod + + +@dataclass +class TrainingConfig: + rounds: int = 10 + local_epochs: int = 3 + batch_size: int = 256 + learning_rate: float = 1e-3 + mu: float = 0.1 + lam: float = 1.0 + num_clients: int = 10 + embedding_dim: int = 128 + dense_units: int = 128 + snapshot_round: Optional[int] = 0 + device: str = "cpu" + + +def _evaluate(model, features, labels, global_prototypes, num_classes, rare_class_ids=None): + if not global_prototypes: + return {"accuracy": 0.0, "macro_f1": 0.0, "n_samples": 0} + embeddings = extract_embeddings(model, features) + predictions = nearest_prototype_predict(embeddings, global_prototypes) + metrics = classification_metrics(labels, predictions, num_classes) + summary = { + "accuracy": metrics["accuracy"], + "macro_f1": metrics["macro_f1"], + "n_samples": metrics["n_samples"], + } + if rare_class_ids: + rare = rare_class_metrics(labels, predictions, num_classes, rare_class_ids) + summary["rare_class_macro_f1"] = rare["rare_class_macro_f1"] + summary["rare_class_recall"] = rare["rare_class_recall"] + return summary + + +def run_federated( + *, + train_features, + train_labels, + assignments: Sequence, + val_features, + val_labels, + num_classes: int, + input_dim: int, + config: TrainingConfig, + init_seed_parts: Sequence[object], + rare_class_ids: Optional[Sequence[int]] = None, +) -> dict: + """Run the baseline federated loop and return fit artifacts and history.""" + import numpy as np + + seed_everything(*init_seed_parts, "model_init") + model_cfg = ModelConfig( + input_dim=int(input_dim), + num_classes=int(num_classes), + embedding_dim=config.embedding_dim, + dense_units=config.dense_units, + ) + model = build_model(model_cfg) + model.to(config.device) + + client_models = [] + for _ in range(config.num_clients): + cm = build_model(model_cfg) + cm.to(config.device) + client_models.append(cm) + + global_parameters = get_parameters(model) + global_prototypes: dict[int, object] = {} + history: list[dict] = [] + snapshot: dict | None = None + + for round_id in range(config.rounds): + updates = [] + for client_id in range(config.num_clients): + update = client_mod.local_update( + client_id, + client_models[client_id], + global_parameters, + global_prototypes, + train_features, + train_labels, + assignments[client_id], + num_classes=num_classes, + local_epochs=config.local_epochs, + batch_size=config.batch_size, + learning_rate=config.learning_rate, + mu=config.mu, + lam=config.lam, + seed=derive_seed(*init_seed_parts, "client_shuffle", round_id, client_id), + ) + updates.append(update) + + global_parameters = server_mod.aggregate_models([u.parameters for u in updates]) + set_parameters(model, global_parameters) + global_prototypes = server_mod.aggregate_prototypes( + [u.prototypes for u in updates], + [u.supports for u in updates], + num_classes, + ) + + if config.snapshot_round is not None and round_id == config.snapshot_round: + snapshot = { + "round": round_id, + "client_prototypes": [ + {int(c): np.asarray(v).tolist() for c, v in u.prototypes.items()} + for u in updates + ], + "client_supports": [ + {int(c): int(n) for c, n in u.supports.items()} for u in updates + ], + } + + row = {"round": round_id, "n_global_prototypes": len(global_prototypes)} + row.update( + _evaluate( + model, + val_features, + val_labels, + global_prototypes, + num_classes, + rare_class_ids, + ) + ) + history.append(row) + + return { + "model": model, + "parameters": global_parameters, + "global_prototypes": global_prototypes, + "history": history, + "snapshot": snapshot, + } diff --git a/src/protean_research/federated/server.py b/src/protean_research/federated/server.py new file mode 100644 index 0000000..d7ed68b --- /dev/null +++ b/src/protean_research/federated/server.py @@ -0,0 +1,45 @@ +"""Server-side aggregation and global inference (baseline PROTEAN). + +Model parameters are averaged with equal ``1/M`` weights (paper Eq. 2/5.5). +Class prototypes are aggregated by the active mode over the contributor set +``S_j``; missing classes are never fabricated. Inference assigns the class of the +nearest global prototype by squared L2 distance (thesis Eqs. 5.11-5.13). +""" + +from __future__ import annotations + +from typing import Mapping, Sequence + +from ..aggregation import AggregationMode +from ..models.protean_cnn import average_parameters +from ..models.prototypes import collect_global_prototypes + + +def aggregate_models(client_parameter_lists: Sequence[list]) -> list: + """Equal-weight (1/M) federated averaging of client model parameters.""" + return average_parameters(list(client_parameter_lists)) + + +def aggregate_prototypes( + client_prototypes: Sequence[Mapping[int, object]], + client_supports: Sequence[Mapping[int, int]], + num_classes: int, + mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, +): + """Aggregate client prototypes into ``{class_id: Cbar_j}`` (baseline).""" + return collect_global_prototypes( + client_prototypes, client_supports, num_classes, mode=mode + ) + + +def predict_global(model, features, global_prototypes): + """Predict labels via nearest global prototype (squared L2).""" + from ..models.protean_cnn import extract_embeddings + from ..models.prototypes import nearest_prototype_predict + + if not global_prototypes: + raise ValueError( + "no global prototypes available: cannot perform prototype inference" + ) + embeddings = extract_embeddings(model, features) + return nearest_prototype_predict(embeddings, global_prototypes) diff --git a/src/protean_research/models/__init__.py b/src/protean_research/models/__init__.py new file mode 100644 index 0000000..db4b74a --- /dev/null +++ b/src/protean_research/models/__init__.py @@ -0,0 +1,27 @@ +"""Baseline PROTEAN model and prototype utilities.""" + +from .protean_cnn import ( + ModelConfig, + average_parameters, + build_model, + extract_embeddings, + get_parameters, + set_parameters, +) +from .prototypes import ( + collect_global_prototypes, + compute_local_prototypes, + nearest_prototype_predict, +) + +__all__ = [ + "ModelConfig", + "build_model", + "get_parameters", + "set_parameters", + "average_parameters", + "extract_embeddings", + "compute_local_prototypes", + "collect_global_prototypes", + "nearest_prototype_predict", +] diff --git a/src/protean_research/models/protean_cnn.py b/src/protean_research/models/protean_cnn.py new file mode 100644 index 0000000..cacd99d --- /dev/null +++ b/src/protean_research/models/protean_cnn.py @@ -0,0 +1,116 @@ +"""The PROTEAN detection CNN (baseline architecture, frozen). + +Two Conv1D blocks (64 then 128 filters, ReLU, max-pool, dropout 0.2/0.5), a +flatten, a dense embedding layer (dimension 128) and a log-softmax +classification head. Tabular flow features are treated as a single-channel 1D +input, as the baseline config describes. The architecture is not part of this +study's contribution and must not be altered. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ModelConfig: + input_dim: int + num_classes: int + embedding_dim: int = 128 + conv1_filters: int = 64 + conv2_filters: int = 128 + kernel_size: int = 3 + dropout1: float = 0.2 + dropout2: float = 0.5 + dense_units: int = 128 + + +def build_model(config: ModelConfig): + """Instantiate the CNN from a :class:`ModelConfig`.""" + import torch + import torch.nn as nn + + class ProteanCNN(nn.Module): + def __init__(self, cfg: ModelConfig) -> None: + super().__init__() + self.cfg = cfg + self.conv1 = nn.Conv1d(1, cfg.conv1_filters, kernel_size=cfg.kernel_size, padding=cfg.kernel_size // 2) + self.conv2 = nn.Conv1d(cfg.conv1_filters, cfg.conv2_filters, kernel_size=cfg.kernel_size, padding=cfg.kernel_size // 2) + self.pool = nn.MaxPool1d(2) + self.drop1 = nn.Dropout(cfg.dropout1) + self.drop2 = nn.Dropout(cfg.dropout2) + self.flatten = nn.Flatten() + with torch.no_grad(): + dummy = torch.zeros(1, 1, cfg.input_dim) + out = self._conv_blocks(dummy) + flat_dim = int(out.shape[1]) + self.fc = nn.Linear(flat_dim, cfg.dense_units) + self.head = nn.Linear(cfg.dense_units, cfg.num_classes) + + def _conv_blocks(self, x): + import torch.nn.functional as F + + h = F.relu(self.conv1(x)) + h = self.pool(h) + h = self.drop1(h) + h = F.relu(self.conv2(h)) + h = self.pool(h) + h = self.drop2(h) + return self.flatten(h) + + def embedding(self, features): + """Return phi(x), the dense embedding used for prototypes.""" + x = features.unsqueeze(1) + h = self._conv_blocks(x) + return self.fc(h) + + def forward(self, features): + import torch.nn.functional as F + + return F.log_softmax(self.head(self.embedding(features)), dim=-1) + + return ProteanCNN(config) + + +def get_parameters(model) -> list: + """Return a deep copy of the model parameters as detached tensors.""" + return [p.detach().clone() for p in model.parameters()] + + +def set_parameters(model, parameters) -> None: + """Load a parameter list (same shapes/order as ``model.parameters()``).""" + import torch + + with torch.no_grad(): + for param, new in zip(model.parameters(), parameters): + param.copy_(torch.as_tensor(new, dtype=param.dtype, device=param.device)) + + +def average_parameters(param_lists: list[list]) -> list: + """Equal-weight (1/M) average of client parameter lists (paper Eq. 2/5.5).""" + import torch + + if not param_lists: + raise ValueError("param_lists must not be empty") + count = len(param_lists) + averaged = [] + for tensors in zip(*param_lists): + stacked = torch.stack([t.detach().float() for t in tensors], dim=0) + averaged.append(stacked.mean(dim=0)) + return averaged + + +def extract_embeddings(model, features, batch_size: int = 1024): + """Return ``phi(x)`` as a NumPy array, in eval mode, without gradients.""" + import numpy as np + import torch + + model.eval() + outputs: list[np.ndarray] = [] + with torch.no_grad(): + for start in range(0, len(features), batch_size): + batch = torch.as_tensor(features[start : start + batch_size], dtype=torch.float32) + outputs.append(model.embedding(batch).cpu().numpy()) + if not outputs: + return np.empty((0, model.cfg.dense_units), dtype=np.float32) + return np.concatenate(outputs, axis=0).astype(np.float32) diff --git a/src/protean_research/models/prototypes.py b/src/protean_research/models/prototypes.py new file mode 100644 index 0000000..736df98 --- /dev/null +++ b/src/protean_research/models/prototypes.py @@ -0,0 +1,78 @@ +"""Local prototype construction and global prototype collection. + +Baseline semantics (paper Eq. 1 / thesis Eq. 5.4): the local prototype ``C_ij`` +is the mean embedding over the client's class-``j`` training samples. Missing +classes (``N_ij = 0``) produce no prototype; missing classes are never +fabricated. Global prototypes are built by the server from the contributor set +``S_j`` via the aggregation module. +""" + +from __future__ import annotations + +from typing import Mapping, Sequence + +from ..aggregation import AggregationMode, aggregate_class_prototypes + + +def compute_local_prototypes(embeddings, labels, num_classes: int): + """Return ``(prototypes, supports)`` for the classes present on a client. + + ``prototypes[c]`` is an ``np.ndarray`` mean embedding; ``supports[c]`` is the + sample count. Absent classes are omitted (no zero vectors are emitted). + """ + import numpy as np + + embeddings = np.asarray(embeddings) + labels = np.asarray(labels) + prototypes: dict[int, "np.ndarray"] = {} + supports: dict[int, int] = {} + for class_id in range(num_classes): + mask = labels == class_id + n = int(mask.sum()) + if n == 0: + continue + prototypes[class_id] = embeddings[mask].mean(axis=0).astype(np.float32) + supports[class_id] = n + return prototypes, supports + + +def collect_global_prototypes( + client_prototypes: Sequence[Mapping[int, object]], + client_supports: Sequence[Mapping[int, int]], + num_classes: int, + mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, +): + """Server-side aggregation of per-client prototypes into global prototypes. + + Clients that did not observe a class simply do not appear in + ``client_prototypes`` for that class, so ``S_j`` is built implicitly and + missing classes are never inserted as zero vectors. + """ + contributions: dict[int, list] = {c: [] for c in range(num_classes)} + for prototypes, supports in zip(client_prototypes, client_supports): + for class_id, proto in prototypes.items(): + contributions[int(class_id)].append((proto, supports.get(int(class_id), 0))) + return aggregate_class_prototypes(contributions, mode) + + +def nearest_prototype_predict(embeddings, global_prototypes: Mapping[int, object]): + """Predict by minimum squared L2 distance to global prototypes (Eq. 5.11-5.13). + + ``global_prototypes`` maps class id -> prototype. Samples whose embedding has + no prototype to compare against (empty prototype set) raise ``ValueError``. + """ + import numpy as np + + if not global_prototypes: + raise ValueError("no global prototypes available for inference") + class_ids = sorted(global_prototypes) + matrix = np.stack([np.asarray(global_prototypes[c], dtype=np.float64) for c in class_ids]) + embeddings = np.asarray(embeddings, dtype=np.float64) + # Squared L2 via the expanded norm identity. + dists = ( + (embeddings ** 2).sum(axis=1, keepdims=True) + - 2.0 * embeddings @ matrix.T + + (matrix ** 2).sum(axis=1)[None, :] + ) + chosen = dists.argmin(axis=1) + return np.asarray([class_ids[i] for i in chosen], dtype=np.int64) diff --git a/src/protean_research/utils/integrity.py b/src/protean_research/utils/integrity.py new file mode 100644 index 0000000..5ee46d2 --- /dev/null +++ b/src/protean_research/utils/integrity.py @@ -0,0 +1,39 @@ +"""Integrity helpers: file and content hashing for reproducible manifests.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: str | Path, chunk_size: int = 1 << 20) -> str: + """Stream-hash a file without loading it fully into memory.""" + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def hash_json(obj: Any) -> str: + """Stable SHA-256 of a JSON-serialisable object (sorted keys).""" + payload = json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) + return sha256_bytes(payload.encode("utf-8")) + + +def hash_index_arrays(arrays: Iterable[object]) -> str: + """Stable hash of a sequence of integer index arrays. + + Arrays are normalised to sorted Python int lists so the hash does not depend + on NumPy dtype or memory layout. + """ + import numpy as np + + normalised = [sorted(int(v) for v in np.asarray(a).ravel()) for a in arrays] + return hash_json(normalised) diff --git a/src/protean_research/utils/seeding.py b/src/protean_research/utils/seeding.py new file mode 100644 index 0000000..1dfa2fb --- /dev/null +++ b/src/protean_research/utils/seeding.py @@ -0,0 +1,51 @@ +"""Deterministic seeding helpers. + +All randomness in the project must be reproducible and isolated by purpose, so +that two aggregation variants sharing ``(dataset, alpha, seed)`` also share the +split, partition, model initialisation and batch order. + +Use :func:`derive_seed` to turn semantic identifiers into a stable integer seed, +never ``hash()`` (which is salted per process). +""" + +from __future__ import annotations + +import hashlib +import random + + +def derive_seed(*parts: object, mod: int = 2**31) -> int: + """Return a stable non-negative integer seed derived from *parts*. + + The result depends only on the string form of *parts*, so it is stable + across processes, machines and Python versions. + """ + payload = "|".join(str(p) for p in parts).encode("utf-8") + digest = hashlib.sha256(payload).hexdigest() + return int(digest[:16], 16) % mod + + +def numpy_rng(*parts: object): + """Return a seeded :class:`numpy.random.Generator` for *parts*.""" + import numpy as np + + return np.random.default_rng(derive_seed(*parts)) + + +def seed_everything(*parts: object) -> int: + """Seed Python, NumPy and PyTorch RNGs deterministically from *parts*. + + Returns the derived integer seed for recording in run metadata. + """ + import numpy as np + + seed = derive_seed(*parts) + random.seed(seed) + np.random.seed(seed) + try: + import torch + + torch.manual_seed(seed) + except ImportError: # pragma: no cover - torch is a research dependency + pass + return seed diff --git a/tests/test_baseline_aggregation.py b/tests/test_baseline_aggregation.py new file mode 100644 index 0000000..86fc5b7 --- /dev/null +++ b/tests/test_baseline_aggregation.py @@ -0,0 +1,76 @@ +"""Tests for the reproduced original-PROTEAN prototype aggregation. + +These tests cover only the baseline aggregation present in this checkpoint. The +participant rule (``S_j`` with equal weights) is a documented reproduction +assumption; these tests check its arithmetic and its explicit handling of +missing classes, not its equivalence to the original method. +""" + +from __future__ import annotations + +import math + +import pytest + +from protean_research.aggregation import ( + AggregationMode, + aggregate_class_prototypes, + aggregate_prototypes, +) + +PROTOTYPES = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] + + +def mean(vectors): + dim = len(vectors[0]) + return [sum(v[d] for v in vectors) / len(vectors) for d in range(dim)] + + +def test_original_protean_is_equal_mean_over_contributors(): + out = aggregate_prototypes(PROTOTYPES, AggregationMode.ORIGINAL_PROTEAN) + assert out == pytest.approx(mean(PROTOTYPES)) + + +def test_original_protean_is_default_mode(): + assert aggregate_prototypes(PROTOTYPES) == pytest.approx(mean(PROTOTYPES)) + + +def test_single_contributor_returns_itself(): + out = aggregate_prototypes([[1.0, -2.0, 3.0]]) + assert out == pytest.approx([1.0, -2.0, 3.0]) + + +def test_output_is_finite_for_valid_inputs(): + out = aggregate_prototypes(PROTOTYPES) + assert all(math.isfinite(v) for v in out) + + +def test_aggregate_class_prototypes_builds_s_j_and_drops_missing(): + contributions = { + 0: [([1.0, 1.0], 5), (None, 0), ([3.0, 3.0], 5)], + 1: [([2.0, 2.0], 0)], # no positive support -> class omitted entirely + } + result = aggregate_class_prototypes(contributions) + assert set(result) == {0} + assert result[0] == pytest.approx([2.0, 2.0]) + + +def test_missing_class_is_not_fabricated(): + # A client with N_ij == 0 contributes no prototype and must not enter S_j. + result = aggregate_class_prototypes({0: [(None, 0), ([1.0], 0)]}) + assert result == {} + + +def test_proposed_mode_is_not_available(): + # The support-weighted mode belongs to the later beta/controls phase. + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, "support_weighted") + + +def test_validation_errors(): + with pytest.raises(ValueError): + aggregate_prototypes([[1.0, 2.0], [1.0]]) # mismatched dimensions + with pytest.raises(ValueError): + aggregate_prototypes([]) # no prototypes + with pytest.raises(ValueError): + aggregate_prototypes([[1.0, float("nan")]]) # non-finite values diff --git a/tests/test_data_pipeline.py b/tests/test_data_pipeline.py new file mode 100644 index 0000000..16713df --- /dev/null +++ b/tests/test_data_pipeline.py @@ -0,0 +1,156 @@ +"""Tests for data configuration, loading, splitting and preprocessing.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from protean_research.data import loading, splits +from protean_research.data.preprocessing import Preprocessor, fit_preprocessor, transform + + +@pytest.fixture() +def toy_csv(tmp_path): + frame = pd.DataFrame( + { + "id": list(range(12)), + "num": ["1", "-", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + "cat": ["a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b"], + "label": ["Normal"] * 4 + + ["Attack"] * 4 + + ["Lateral _movement"] * 2 + + ["Attack"] * 2, + } + ) + # Duplicate one row to exercise the duplicate-drop policy. + frame = pd.concat([frame, frame.iloc[[0]]], ignore_index=True) + path = tmp_path / "toy.csv" + frame.to_csv(path, index=False) + return path + + +@pytest.fixture() +def toy_cfg(toy_csv): + return { + "datasets": { + "toy": { + "name": "Toy", + "raw_path": str(toy_csv), + "label_column": "label", + "benign_label": "Normal", + "drop_labels": [], + "missing_tokens": ["-"], + "drop_duplicate_rows": True, + "categorical_columns": ["cat"], + "non_feature_columns": ["id"], + } + } + } + + +def test_spec_resolution_and_feature_columns(toy_cfg): + spec = loading.load_spec("toy", toy_cfg) + assert spec.label_column == "label" + assert spec.non_feature_set == {"id", "label"} + cleaned = loading.clean_frame(loading.read_raw(spec), spec) + # 13 written rows - 1 duplicate = 12. + assert cleaned.n_rows_raw == 13 + assert cleaned.n_duplicates_dropped == 1 + assert len(cleaned.labels) == 12 + assert set(cleaned.feature_columns) == {"num", "cat"} + # Non-feature and alias columns are excluded from features. + assert "id" not in cleaned.feature_columns + assert "label" not in cleaned.feature_columns + + +def test_label_mapping_deterministic_and_canonical(toy_cfg): + spec = loading.load_spec("toy", toy_cfg) + cleaned = loading.clean_frame(loading.read_raw(spec), spec) + assert cleaned.class_names == ["Attack", "Lateral_movement", "Normal"] + assert cleaned.class_names[cleaned.benign_index] == "Normal" + assert loading.canonical_label("Lateral _movement") == "Lateral_movement" + # Re-running produces the identical encoding. + again = loading.clean_frame(loading.read_raw(spec), spec) + assert np.array_equal(cleaned.labels, again.labels) + + +def test_unknown_dataset_raises(): + with pytest.raises(KeyError): + loading.load_spec("does-not-exist", {"datasets": {}}) + + +def test_real_dataset_config_resolves_without_reading(): + # The tracked dataset config must resolve both research datasets and their + # external raw paths without touching the raw files. + for key in ("xiiotid", "5gnidd"): + spec = loading.load_spec(key) + assert spec.key == key + assert spec.label_column + assert spec.benign_label + assert spec.raw_path.is_absolute() + assert spec.non_feature_columns + spec = loading.load_spec("5gnidd") + assert "Unnamed: 0" in spec.non_feature_columns + assert spec.missing_tokens == ("",) + + +def test_split_is_deterministic_and_disjoint(): + labels = np.repeat([0, 1, 2, 3], 25) + a = splits.stratified_split(labels, "toy", seed=0) + b = splits.stratified_split(labels, "toy", seed=0) + assert np.array_equal(a.train, b.train) + assert np.array_equal(a.validation, b.validation) + assert np.array_equal(a.test, b.test) + splits.check_disjoint(a, n_rows=len(labels)) + # Every class appears in validation and test. + for split in (a.validation, a.test): + assert set(np.unique(labels[split])) == {0, 1, 2, 3} + + +def test_different_seed_changes_split(): + labels = np.repeat([0, 1, 2, 3], 25) + a = splits.stratified_split(labels, "toy", seed=0) + b = splits.stratified_split(labels, "toy", seed=1) + assert not np.array_equal(a.train, b.train) + + +def test_preprocessing_is_deterministic(toy_cfg): + spec = loading.load_spec("toy", toy_cfg) + cleaned = loading.clean_frame(loading.read_raw(spec), spec) + split = splits.stratified_split(cleaned.labels, "toy", seed=0) + pre1 = fit_preprocessor(cleaned, split.train) + pre2 = fit_preprocessor(cleaned, split.train) + assert pre1.to_dict() == pre2.to_dict() + x1 = transform(pre1, cleaned) + x2 = transform(pre1, cleaned) + assert np.array_equal(x1, x2) + assert x1.dtype == np.float32 + assert np.isfinite(x1).all() + + +def test_preprocessing_fits_on_train_only(toy_cfg): + spec = loading.load_spec("toy", toy_cfg) + cleaned = loading.clean_frame(loading.read_raw(spec), spec) + train_idx = np.arange(0, 8) + pre = fit_preprocessor(cleaned, train_idx) + train_only = cleaned.features.iloc[train_idx] + values = pd.to_numeric( + train_only["num"].replace({"-": np.nan}), errors="coerce" + ) + expected_median = float(values.median()) + expected_mean = float(values.fillna(expected_median).mean()) + assert pre.medians["num"] == pytest.approx(expected_median) + assert pre.means["num"] == pytest.approx(expected_mean) + + +def test_preprocessor_roundtrip(toy_cfg, tmp_path): + spec = loading.load_spec("toy", toy_cfg) + cleaned = loading.clean_frame(loading.read_raw(spec), spec) + split = splits.stratified_split(cleaned.labels, "toy", seed=0) + pre = fit_preprocessor(cleaned, split.train) + path = tmp_path / "pre.json" + pre.save(path) + loaded = Preprocessor.load(path) + assert loaded.to_dict() == pre.to_dict() + assert np.array_equal(transform(loaded, cleaned), transform(pre, cleaned)) diff --git a/tests/test_federated_smoke.py b/tests/test_federated_smoke.py new file mode 100644 index 0000000..35d6344 --- /dev/null +++ b/tests/test_federated_smoke.py @@ -0,0 +1,50 @@ +"""A tiny end-to-end federated smoke test on synthetic, separable data. + +Guards the baseline loop (local update -> model averaging -> prototype +aggregation -> redistribution -> validation metrics) without needing the raw +datasets. This is a pipeline test, not a research result. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") + +from protean_research.federated import TrainingConfig, run_federated # noqa: E402 + + +def test_federated_smoke_runs_and_produces_prototypes(): + rng = np.random.default_rng(0) + n_per_class = 20 + num_classes = 3 + dim = 6 + features = rng.normal(size=(n_per_class * num_classes, dim)).astype(np.float32) + labels = np.repeat(np.arange(num_classes), n_per_class).astype(np.int64) + features = features + labels[:, None] * 2.0 # make classes separable + assignments = [np.arange(0, 20), np.arange(20, 40), np.arange(40, 60)] + + config = TrainingConfig( + rounds=1, + local_epochs=1, + batch_size=16, + num_clients=3, + embedding_dim=8, + dense_units=8, + ) + result = run_federated( + train_features=features, + train_labels=labels, + assignments=assignments, + val_features=features, + val_labels=labels, + num_classes=num_classes, + input_dim=dim, + config=config, + init_seed_parts=("toy", "seed=0"), + ) + assert set(result["global_prototypes"]) == {0, 1, 2} + assert len(result["parameters"]) > 0 + assert np.isfinite(result["history"][-1]["accuracy"]) + assert result["snapshot"] is not None diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..df5ff0c --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,44 @@ +"""Tests for the evaluation metrics on small known fixtures.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from protean_research.evaluation.metrics import ( + classification_metrics, + confusion_matrix, + rare_class_metrics, +) + + +def test_confusion_matrix_counts(): + matrix = confusion_matrix([0, 0, 1, 1], [0, 1, 1, 1], num_classes=2) + assert matrix.tolist() == [[1, 1], [0, 2]] + + +def test_classification_metrics_known_fixture(): + y_true = [0, 0, 1, 1, 2, 2] + y_pred = [0, 0, 1, 2, 2, 1] + metrics = classification_metrics(y_true, y_pred, num_classes=3) + assert metrics["accuracy"] == pytest.approx(4 / 6) + assert metrics["macro_f1"] == pytest.approx((1.0 + 0.5 + 0.5) / 3) + per_class = {row["class_id"]: row for row in metrics["per_class"]} + assert per_class[0]["f1"] == pytest.approx(1.0) + assert per_class[1]["precision"] == pytest.approx(0.5) + assert per_class[2]["recall"] == pytest.approx(0.5) + + +def test_rare_class_metrics_restricts_to_given_classes(): + y_true = [0, 0, 1, 1, 2, 2] + y_pred = [0, 0, 1, 2, 2, 1] + rare = rare_class_metrics(y_true, y_pred, num_classes=3, rare_class_ids=[1, 2]) + assert rare["rare_class_macro_f1"] == pytest.approx(0.5) + assert rare["rare_class_recall"] == pytest.approx(0.5) + assert rare["rare_support"] == 4 + + +def test_rare_class_metrics_skips_absent_classes(): + rare = rare_class_metrics([0, 0], [0, 0], num_classes=3, rare_class_ids=[2]) + assert rare["rare_class_macro_f1"] == 0.0 + assert rare["rare_support"] == 0 diff --git a/tests/test_partition.py b/tests/test_partition.py new file mode 100644 index 0000000..955d3df --- /dev/null +++ b/tests/test_partition.py @@ -0,0 +1,126 @@ +"""Tests for deterministic Dirichlet partitioning and manifest reload.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from protean_research.data import manifest as manifest_mod +from protean_research.data import partition as partition_mod +from protean_research.data import rare_unseen, splits + + +def make_labels(num_classes=5, per_class=40): + return np.repeat(np.arange(num_classes), per_class).astype(np.int64) + + +def test_partition_same_seed_identical(): + labels = make_labels() + idx = partition_mod.class_index_map(labels, 5) + a = partition_mod.dirichlet_partition(idx, 4, 0.3, rng_seed=123) + b = partition_mod.dirichlet_partition(idx, 4, 0.3, rng_seed=123) + for x, y in zip(a, b): + assert np.array_equal(x, y) + + +def test_partition_different_alpha_differs(): + labels = make_labels() + idx = partition_mod.class_index_map(labels, 5) + a = partition_mod.dirichlet_partition(idx, 4, 0.1, rng_seed=7) + b = partition_mod.dirichlet_partition(idx, 4, 5.0, rng_seed=7) + assert partition_mod.partition_hash(a) != partition_mod.partition_hash(b) + + +def test_every_sample_assigned_exactly_once(): + labels = make_labels() + idx = partition_mod.class_index_map(labels, 5) + assignments = partition_mod.dirichlet_partition(idx, 4, 0.4, rng_seed=99) + union = np.sort(np.concatenate(assignments)) + assert np.array_equal(union, np.arange(len(labels))) + assert len(union) == len(set(union.tolist())) + + +def test_support_matrix_matches_assignments(): + labels = make_labels() + idx = partition_mod.class_index_map(labels, 5) + assignments = partition_mod.dirichlet_partition(idx, 4, 0.4, rng_seed=99) + support = partition_mod.support_matrix(assignments, labels, 5) + assert support.shape == (4, 5) + assert int(support.sum()) == len(labels) + # Column sums equal per-class counts. + assert np.array_equal(support.sum(axis=0), np.bincount(labels, minlength=5)) + + +def test_manifest_save_reload_identity(tmp_path): + labels = make_labels() + split = splits.stratified_split(labels, "toy", seed=0) + labels_train = labels[split.train] + idx = partition_mod.class_index_map(labels_train, 5) + rng_seed = partition_mod.partition_rng_seed("toy", 0.5, 0) + assignments = partition_mod.dirichlet_partition(idx, 4, 0.5, rng_seed) + manifest = manifest_mod.build_manifest( + dataset_key="toy", + dataset_name="Toy", + alpha=0.5, + seed=0, + num_clients=4, + class_names=[f"c{i}" for i in range(5)], + benign_class="c0", + labels=labels, + split=split, + assignments=assignments, + rng_seed=rng_seed, + train_fraction=0.7, + validation_fraction=0.1, + test_fraction=0.2, + ) + path = tmp_path / "toy__alpha0.5__seed0.json" + manifest_mod.save_manifest(path, manifest) + loaded = manifest_mod.load_manifest(path) + split_reload, assignments_reload, support_reload = manifest_mod.reproduce_train_assignments( + loaded, labels + ) + assert np.array_equal(split_reload.train, split.train) + assert partition_mod.partition_hash(assignments_reload) == manifest["partition_hash"] + assert np.array_equal(support_reload, np.asarray(manifest["support"])) + + +def test_manifest_detects_corruption(tmp_path): + labels = make_labels() + split = splits.stratified_split(labels, "toy", seed=0) + labels_train = labels[split.train] + idx = partition_mod.class_index_map(labels_train, 5) + rng_seed = partition_mod.partition_rng_seed("toy", 0.5, 0) + assignments = partition_mod.dirichlet_partition(idx, 4, 0.5, rng_seed) + manifest = manifest_mod.build_manifest( + dataset_key="toy", dataset_name="Toy", alpha=0.5, seed=0, num_clients=4, + class_names=[f"c{i}" for i in range(5)], benign_class="c0", labels=labels, + split=split, assignments=assignments, rng_seed=rng_seed, + train_fraction=0.7, validation_fraction=0.1, test_fraction=0.2, + ) + manifest["rng_seed"] = int(manifest["rng_seed"]) + 1 + with pytest.raises(ValueError): + manifest_mod.reproduce_train_assignments(manifest, labels) + + +def test_rare_unseen_semantics(): + # Client 0: benign + classes 1,2,3 with supports 10, 5, 2 -> rare = {3,2}. + support = np.array( + [ + [50, 10, 5, 2, 0], + [50, 1, 0, 0, 0], + [50, 0, 0, 0, 0], + ] + ) + rows = rare_unseen.rare_unseen_rows(support, ["b", "c1", "c2", "c3", "c4"], 0, "toy", 0.5, 0) + client0 = [r for r in rows if r["client_id"] == 0] + rare = {r["class_id"] for r in client0 if r["rare_flag"]} + unseen = {r["class_id"] for r in client0 if r["unseen_flag"]} + assert rare == {2, 3} + assert unseen == {4} + # Benign is never rare or unseen. + assert not any(r["rare_flag"] or r["unseen_flag"] for r in rows if r["class_id"] == 0) + # Client with a single attack class uses that one. + assert rare_unseen.rare_class_ids_for_client([50, 1, 0, 0, 0], 0) == [1] + # Client with no attack class contributes nothing to the rare aggregate. + assert rare_unseen.rare_class_ids_for_client([50, 0, 0, 0, 0], 0) == [] diff --git a/tests/test_prototype_aggregation.py b/tests/test_prototype_aggregation.py deleted file mode 100644 index a9bc1e1..0000000 --- a/tests/test_prototype_aggregation.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Unit tests for the server-side prototype aggregation math. - -Covers the dependency-free subset of the testing strategy in -``docs/research/IMPLEMENTATION_PLAN.md`` (items 1-7). Items requiring the -future data/federated modules (partition determinism, manifest reload, -baseline invariance, validation/test separation) are deferred. -""" - -from __future__ import annotations - -import math - -import pytest - -from protean_research.aggregation import ( - AggregationMode, - aggregate_prototypes, - aggregation_weights, - beta_weights, -) - -BETAS = [0.0, 0.25, 0.5, 0.75, 1.0] - - -def mean(vectors): - dim = len(vectors[0]) - return [sum(v[d] for v in vectors) / len(vectors) for d in range(dim)] - - -def test_beta_zero_equals_arithmetic_mean() -> None: - prototypes = [[0.0, 2.0, -1.0], [2.0, 0.0, 1.0], [4.0, 4.0, 4.0]] - supports = [1.0, 3.0, 10.0] - out = aggregate_prototypes( - prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=0.0, supports=supports - ) - assert out == pytest.approx(mean(prototypes)) - - -def test_beta_one_gives_sample_count_weighting() -> None: - prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] - supports = [1.0, 3.0, 10.0] - out = aggregate_prototypes( - prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=1.0, supports=supports - ) - total = sum(supports) - expected = [ - sum(s * v[d] for s, v in zip(supports, prototypes)) / total - for d in range(2) - ] - assert out == pytest.approx(expected) - - -def test_single_contributor_invariant_across_betas() -> None: - prototypes = [[1.0, -2.0, 3.0]] - supports = [7.0] - for beta in BETAS: - out = aggregate_prototypes( - prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=supports - ) - assert out == pytest.approx(prototypes[0]) - - -def test_equal_supports_invariant_across_betas() -> None: - prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] - supports = [5.0, 5.0, 5.0] - baseline = aggregate_prototypes( - prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=0.0, supports=supports - ) - for beta in BETAS: - out = aggregate_prototypes( - prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=supports - ) - assert out == pytest.approx(baseline) - - -def test_missing_class_not_in_s_j() -> None: - # A client with N_ij == 0 must never be passed into S_j. - with pytest.raises(ValueError): - beta_weights([1.0, 0.0], beta=0.5) - with pytest.raises(ValueError): - beta_weights([1.0, -3.0], beta=0.5) - with pytest.raises(ValueError): - aggregate_prototypes( - [[1.0], [2.0]], AggregationMode.SUPPORT_WEIGHTED, beta=0.5, supports=[1.0, 0.0] - ) - - -def test_normalized_weights_sum_to_one() -> None: - prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] - for beta in BETAS: - w = aggregation_weights( - prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=[1.0, 3.0, 10.0] - ) - assert sum(w) == pytest.approx(1.0) - w_eq = aggregation_weights(prototypes, AggregationMode.EQUAL_AVAILABLE) - assert sum(w_eq) == pytest.approx(1.0) - assert w_eq == pytest.approx([1 / 3, 1 / 3, 1 / 3]) - - -def test_valid_inputs_never_nan_or_inf() -> None: - prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] - for beta in BETAS: - out = aggregate_prototypes( - prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=[1.0, 3.0, 10.0] - ) - assert all(math.isfinite(v) for v in out) - - -def test_original_protean_is_not_implemented() -> None: - with pytest.raises(NotImplementedError): - aggregate_prototypes([[1.0, 2.0]], AggregationMode.ORIGINAL_PROTEAN) - - -def test_validation_errors() -> None: - with pytest.raises(ValueError): - beta_weights([1.0], beta=1.5) - with pytest.raises(ValueError): - beta_weights([1.0], beta=None) # type: ignore[arg-type] - with pytest.raises(ValueError): - aggregate_prototypes([[1.0, 2.0], [1.0]], AggregationMode.SUPPORT_WEIGHTED, beta=0.5, supports=[1.0, 2.0]) - with pytest.raises(ValueError): - aggregate_prototypes( - [[1.0, 2.0]], AggregationMode.SUPPORT_WEIGHTED, beta=0.5, supports=[1.0, 2.0] - ) - with pytest.raises(ValueError): - aggregate_prototypes([[1.0, float("nan")]], AggregationMode.EQUAL_AVAILABLE) diff --git a/tests/test_prototypes.py b/tests/test_prototypes.py new file mode 100644 index 0000000..177313d --- /dev/null +++ b/tests/test_prototypes.py @@ -0,0 +1,58 @@ +"""Tests for local prototype construction and global prototype collection.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from protean_research.aggregation import AggregationMode +from protean_research.models.prototypes import ( + collect_global_prototypes, + compute_local_prototypes, + nearest_prototype_predict, +) + + +def test_local_prototype_is_mean_embedding(): + embeddings = np.array( + [[0.0, 0.0], [2.0, 0.0], [10.0, 10.0], [0.0, 4.0]], dtype=np.float32 + ) + labels = np.array([0, 0, 1, 1]) + prototypes, supports = compute_local_prototypes(embeddings, labels, num_classes=3) + assert supports == {0: 2, 1: 2} + assert prototypes[0] == pytest.approx([1.0, 0.0]) + assert prototypes[1] == pytest.approx([5.0, 7.0]) + + +def test_missing_class_has_no_prototype(): + embeddings = np.array([[1.0, 1.0]], dtype=np.float32) + labels = np.array([2]) + prototypes, supports = compute_local_prototypes(embeddings, labels, num_classes=4) + assert set(prototypes) == {2} + assert 0 not in prototypes + assert 1 not in prototypes + + +def test_collect_global_original_is_equal_mean_over_contributors(): + client_prototypes = [{0: np.array([0.0, 2.0])}, {0: np.array([4.0, 4.0])}] + client_supports = [{0: 1}, {0: 100}] + result = collect_global_prototypes( + client_prototypes, client_supports, num_classes=1, mode=AggregationMode.ORIGINAL_PROTEAN + ) + assert result[0] == pytest.approx([2.0, 3.0]) + + +def test_collect_global_omits_missing_class(): + client_prototypes = [{0: np.array([1.0, 1.0])}, {}] + client_supports = [{0: 3}, {}] + result = collect_global_prototypes( + client_prototypes, client_supports, num_classes=2, mode=AggregationMode.ORIGINAL_PROTEAN + ) + assert set(result) == {0} + + +def test_nearest_prototype_predict(): + prototypes = {0: np.array([0.0, 0.0]), 1: np.array([10.0, 10.0])} + embeddings = np.array([[0.1, 0.1], [9.0, 9.0]]) + predictions = nearest_prototype_predict(embeddings, prototypes) + assert predictions.tolist() == [0, 1]