From 2cc6a9859e454bb982f5b925464dd330ff91eddb Mon Sep 17 00:00:00 2001 From: willtanoe Date: Thu, 24 Sep 2026 17:08:56 +0000 Subject: [PATCH] chore: checkpoint PROTEAN research planning and architecture --- .gitignore | 28 ++ README.md | 62 ++++ artifacts/analysis/.gitkeep | 0 artifacts/partitions/.gitkeep | 0 configs/datasets/datasets.yaml | 67 ++++ configs/experiments/experiment_protocol.yaml | 60 +++ configs/models/protean_cnn.yaml | 44 +++ .../PROTEAN_BASELINE_INVESTIGATION.md | 164 +++++++++ docs/handoff/SESSION_HANDOFF.md | 346 ++++++++++++++++++ docs/research/DATASET_INSPECTION.md | 192 ++++++++++ docs/research/EXPERIMENT_PROTOCOL.md | 177 +++++++++ docs/research/IMPLEMENTATION_PLAN.md | 217 +++++++++++ docs/research/RESEARCH_SPEC_INTERPRETATION.md | 219 +++++++++++ planningawal/PROTEAN_Revisi_Lurus.md | 118 ++++++ pyproject.toml | 33 ++ scripts/.gitkeep | 0 src/protean_research/__init__.py | 10 + src/protean_research/aggregation/__init__.py | 20 + .../aggregation/prototype_aggregation.py | 163 +++++++++ src/protean_research/utils/__init__.py | 1 + src/protean_research/utils/config.py | 91 +++++ tests/test_prototype_aggregation.py | 126 +++++++ 22 files changed, 2138 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 artifacts/analysis/.gitkeep create mode 100644 artifacts/partitions/.gitkeep create mode 100644 configs/datasets/datasets.yaml create mode 100644 configs/experiments/experiment_protocol.yaml create mode 100644 configs/models/protean_cnn.yaml create mode 100644 docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md create mode 100644 docs/handoff/SESSION_HANDOFF.md create mode 100644 docs/research/DATASET_INSPECTION.md create mode 100644 docs/research/EXPERIMENT_PROTOCOL.md create mode 100644 docs/research/IMPLEMENTATION_PLAN.md create mode 100644 docs/research/RESEARCH_SPEC_INTERPRETATION.md create mode 100755 planningawal/PROTEAN_Revisi_Lurus.md create mode 100644 pyproject.toml create mode 100644 scripts/.gitkeep create mode 100644 src/protean_research/__init__.py create mode 100644 src/protean_research/aggregation/__init__.py create mode 100644 src/protean_research/aggregation/prototype_aggregation.py create mode 100644 src/protean_research/utils/__init__.py create mode 100644 src/protean_research/utils/config.py create mode 100644 tests/test_prototype_aggregation.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24f3aae --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ + +# Test / tooling caches +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Generated research artifacts that are regenerable and may be large. +# NOTE: artifacts/partitions/ and artifacts/analysis/ are intentionally NOT ignored: +# partition manifests are deterministic, content-hashed, and required for paired +# comparisons; analysis outputs are small. They are intended to be version controlled. +artifacts/processed/ +artifacts/checkpoints/ +results/ +logs/ +*.log + +# Local env +.env +.env.* + +# OS +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..0762a8a --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +# perdalam-protean + +Research code for **server-side prototype aggregation in PROTEAN** using class-specific +sample support, for rare attack detection under non-IID federated intrusion detection. + +The study modifies only how the server combines local class prototypes: + +``` +Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta C_ij ) / ( sum_{i in S_j} N_ij^beta ) +``` + +with `S_j = { i | N_ij > 0 and C_ij exists }` and candidate +`beta in {0.0, 0.25, 0.5, 0.75, 1.0}`. Architecture, model aggregation, loss terms, and +inference follow the PROTEAN baseline (Chennoufi et al., 2025). + +## Immutable source + +- `planningawal/PROTEAN_Revisi_Lurus.md` is the authoritative research specification and is + **read-only**. Do not edit, add, rename, move, or delete anything under `planningawal/`. + +## External raw datasets (read-only, never copied into the repo) + +- `~/datasets/X-IIoTID/X-IIoTID-dataset.csv` +- `~/datasets/5G-NIDD/5G-NIDD.csv` + +Paths are configured in `configs/datasets/datasets.yaml` (with environment overrides) and are +never hardcoded in source. Generated artifacts go under `artifacts/`. + +## Layout + +``` +docs/research/ spec interpretation, dataset inspection, implementation plan, protocol +docs/baseline/ PROTEAN baseline investigation +docs/handoff/ session handoff (start here for a new session) +configs/ datasets, experiments, model +src/protean_research/ aggregation (implemented), data/federated/models/evaluation (planned) +tests/ unit tests +artifacts/ results/ generated outputs (not source) +``` + +## Status + +This is a planning/bootstrap state. No full training has been run and no experimental sweep +has been launched. The original PROTEAN baseline has **no public official code**, so it must +be reproduced from the paper. + +Read `docs/handoff/SESSION_HANDOFF.md` first, then +`planningawal/PROTEAN_Revisi_Lurus.md`. + +## Quickstart + +```bash +# inside the repo, with a virtualenv that has pyyaml + pytest +python -m pytest -q +``` + +The aggregation module is dependency-free and testable without pandas/torch: + +```python +from protean_research.aggregation import aggregate_prototypes +aggregate_prototypes([[0.0, 2.0], [2.0, 0.0]], "support_weighted", beta=0.5, supports=[1, 3]) +``` diff --git a/artifacts/analysis/.gitkeep b/artifacts/analysis/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/artifacts/partitions/.gitkeep b/artifacts/partitions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/configs/datasets/datasets.yaml b/configs/datasets/datasets.yaml new file mode 100644 index 0000000..eb82db7 --- /dev/null +++ b/configs/datasets/datasets.yaml @@ -0,0 +1,67 @@ +# Centralized dataset configuration for perdalam-protean. +# +# RAW DATA CONTRACT +# The raw datasets live OUTSIDE this repository and are read-only source data. +# They must never be moved, renamed, overwritten, or copied wholesale into the repo. +# +# PATH RESOLUTION PRECEDENCE (implemented in src/protean_research/utils/config.py) +# 1. per-dataset environment variable listed in `env_override` +# 2. `raw_path` below (`~` is expanded) +# 3. values are resolved at load time; nothing is hardcoded in training logic +# +# GENERATED DATA +# All preprocessed caches, partitions, manifests, checkpoints and analysis +# outputs must be written under `generated_root` (default: artifacts/), never +# back into the raw dataset directory. + +generated_root: artifacts + +datasets: + xiiotid: + name: X-IIoTID + raw_path: ~/datasets/X-IIoTID/X-IIoTID-dataset.csv + env_override: PROTEAN_XIIOTID_RAW_PATH + # Paper uses the 10 "general" attack classes and removes Crypto-ransomware. + label_column: class2 + label_column_aliases: + fine_grained: class1 # 19 specific attacks + binary: class3 # Normal / Attack + benign_label: Normal + drop_labels: + - crypto-ransomware + # '-' is the dataset's documented missing placeholder, not an empty string. + missing_tokens: ["-", "?"] + # Columns that are identifiers / timestamps / network endpoints and must NOT + # be used as model features without an explicit, documented decision. + non_feature_columns: + - Date + - Timestamp + - Scr_IP + - Scr_port + - Des_IP + - Des_port + notes: > + Row-level missing placeholders ('-') and a small number of malformed tokens + (e.g. '?', 'aza') exist. Exact duplicate rows are present. See + docs/research/DATASET_INSPECTION.md. + + 5gnidd: + name: 5G-NIDD + raw_path: ~/datasets/5G-NIDD/5G-NIDD.csv + env_override: PROTEAN_5GNIDD_RAW_PATH + # Multi-class attack label (9 values incl. Benign): the class used by PROTEAN. + label_column: Attack Type + label_column_aliases: + binary: Label # Malicious / Benign + tool: Attack Tool + benign_label: Benign + drop_labels: [] + missing_tokens: [""] + # First column is an unnamed row index. Seq / Offset are flow-ordering fields. + non_feature_columns: + - "" + - Seq + - Offset + notes: > + Several columns are missing for large row fractions (e.g. dTos, DstWin, + sVid, dVid). See docs/research/DATASET_INSPECTION.md. diff --git a/configs/experiments/experiment_protocol.yaml b/configs/experiments/experiment_protocol.yaml new file mode 100644 index 0000000..d9a1365 --- /dev/null +++ b/configs/experiments/experiment_protocol.yaml @@ -0,0 +1,60 @@ +# Frozen experiment protocol for the beta-weighted prototype aggregation study. +# Values below are the required research settings. Seed set is provisional until +# the verified baseline convention is reproduced (see docs/baseline/). + +datasets: + - xiiotid + - 5gnidd + +non_iid: + partition: dirichlet + alphas: [0.75, 0.50, 0.25] # required by the research specification + num_clients: 10 # matches PROTEAN paper cross-silo setting + +aggregation: + # beta candidates under test (one beta is applied to all classes and clients) + betas: [0.00, 0.25, 0.50, 0.75, 1.00] + # Modes: + # original_protean : verified baseline semantics (NOT YET VERIFIED) + # support_weighted : w_ij = N_ij ** beta, normalized over S_j (proposed) + # equal_available : equal weights over S_j (methodological control, + # introduced only if the baseline participant rule differs) + primary_mode: support_weighted + baseline_mode: original_protean + control_mode: equal_available + +training: + rounds: 10 + local_epochs: 3 + # mu is stated in the paper; lambda is NOT stated in the paper text (UNRESOLVED). + mu: 0.1 + lambda: null + batch_size: null # to be fixed after baseline reproduction + learning_rate: null # to be fixed after baseline reproduction + +evaluation: + # The paper uses 80/20 train/test and has no validation split. + # This study ADDS a validation split used only for beta selection. + test_fraction: 0.20 + validation_fraction: null # finalize after baseline reproduction + selection_metric: rare_class_macro_f1 + rare_class_rule: two_smallest_positive_attack_support + report_metrics: + - rare_class_macro_f1 + - rare_class_recall + - overall_macro_f1 + - accuracy + - per_client_metrics + +seeds: + # Paper averaged over three random seeds. Exact seed values are UNRESOLVED; + # they must be fixed before any beta comparison and shared across all modes. + provisional: [0, 1, 2] + status: provisional + +artifacts: + partition_manifest_dir: artifacts/partitions + processed_dir: artifacts/processed + checkpoint_dir: artifacts/checkpoints + analysis_dir: artifacts/analysis + results_dir: results diff --git a/configs/models/protean_cnn.yaml b/configs/models/protean_cnn.yaml new file mode 100644 index 0000000..a63b426 --- /dev/null +++ b/configs/models/protean_cnn.yaml @@ -0,0 +1,44 @@ +# PROTEAN detection model as described in Chennoufi et al. (2025), Section 4.1. +# The architecture is FROZEN to the baseline and is not part of this study's +# contribution. The only proposed change is server-side prototype aggregation. + +model: + name: protean_cnn + framework: pytorch + input: flow_features # tabular -> treated as 1D / single-channel input + embedding_dim: 128 # output of the dense layer == prototype dimension d + layers: + - type: conv1d + filters: 64 + activation: relu + pool: max + dropout: 0.2 + - type: conv1d + filters: 128 + activation: relu + pool: max + dropout: 0.5 + - type: flatten + - type: dense + units: 128 + - type: classification_head + output: log_softmax + # Head concatenates/uses the dense embedding; EOF prototypes are extracted + # from the embedding part phi(x) (first 8 of 10 layers per the paper text). + prototype_source: embedding + +loss: + # Eq. (3): classification loss + lambda * prototype alignment + mu/2 * model alignment + terms: + - classification: cross_entropy + - prototype_alignment: l2_squared_distance_to_global_prototype + weight: lambda + - model_alignment: l2_squared_distance_to_previous_global_model + weight: mu + mu: 0.1 + lambda: null # UNRESOLVED: not stated in the paper text + +inference: + rule: nearest_global_prototype_l2 + trailing: "assign label of the global prototype with the smallest L2 distance + to the instance embedding (paper Section 3.2)" diff --git a/docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md b/docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md new file mode 100644 index 0000000..97989c6 --- /dev/null +++ b/docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md @@ -0,0 +1,164 @@ +# PROTEAN Baseline Investigation + +Status: investigation record +Scope: locate authoritative PROTEAN resources and establish baseline aggregation semantics +before designing the beta-weighted modification. +Investigated: 2026-09-24 + +Evidence tags: + +- `CONFIRMED FROM PAPER` — stated in the published paper and/or the author's PhD thesis. +- `CONFIRMED FROM CODE` — verified in official implementation code. +- `REASONABLE INFERENCE` — not written verbatim but strongly implied. +- `UNKNOWN` — no authoritative evidence. + +> There is **no** `CONFIRMED FROM CODE` entry in this document, because no official +> implementation was found. See Section 1. + +--- + +## 1. Authoritative sources found + +| Resource | Identifier / URL | Notes | +|---|---|---| +| Paper (arXiv v1) | https://arxiv.org/abs/2507.05524 (arXiv:2507.05524v1, 7 Jul 2025) | Full methodology and experiments | +| Paper (HTML) | https://arxiv.org/html/2507.05524v1 | Read in full for this investigation | +| Published version | ESORICS 2025, pp. 103–125, DOI `10.1007/978-3-032-07884-1_6` | Springer | +| HAL deposit | `hal-05379203` / `u-paris.hal.science/TELECOM-SUDPARIS/hal-05379203v1` | Open PDF, CC BY 4.0 | +| Author PhD thesis | `tel-05490927` / `theses.hal.science/tel-05490927v1` | 195 pages, 2025; Section 5 restates PROTEAN with extra detail | +| Author GitHub | https://github.com/sarachn (Sara Chennoufi) | 2 repositories, **neither is PROTEAN** | + +Authors: Sara Chennoufi (SAMOVAR, Télécom SudParis), Yufei Han (PIRAT, INRIA Rennes), +Gregory Blanc (SAMOVAR), Emiliano De Cristofaro (UC Riverside), Christophe Kiennert (SAMOVAR). + +## 2. Code availability status + +**Conclusion: no official PROTEAN code was found. Reproduction from the paper/thesis is +required.** + +Evidence collected: + +1. Author GitHub account `sarachn` exposes exactly two repositories: + - `sarachn/FATE-LLM` — a **fork** of `FederatedAI/FATE-LLM` (Apache-2.0), unrelated. + - `sarachn/SecLLMHolmes` (GPL-3.0), unrelated to PROTEAN. + Neither contains PROTEAN. +2. GitHub repository search for `PROTEAN federated prototype` returned **0** results. + Repository searches for `PROTEAN intrusion detection` / `PROTEAN federated learning` + returned no PROTEAN artifact. +3. GitHub code search requires authentication and was not used; the public + `grep.app` code search for `PROTEAN prototype` returned no results. +4. The GitHub user `Chennoufi` is unrelated (single repository `septembre`). +5. Neither the paper HTML nor the thesis text contains a "code available at", GitHub, + Zenodo, or supplementary-artifact statement. The HAL record only hosts the PDF. +6. No official supplementary artifact was linked from arXiv, Springer, or HAL. + +Consequence: baseline semantics must be reconstructed from the paper and thesis. Any +choice not stated there is `UNKNOWN` and must be frozen as an explicit project decision +and reported as such. + +## 3. What the paper/thesis state (verbatim-level facts) + +Model: CNN in PyTorch — two convolutional layers (64 and 128 filters) with ReLU, each +followed by max-pooling and dropout (0.2 and 0.5); flatten; dense layer of 128 units; final +classification layer with log-softmax. Embedding = "first eight of ten layers" (illustrative). +`mu = 0.1`. No numeric value is given for `lambda`. 10 clients, 80/20 train/test, 10 +federated rounds, 3 local epochs. Dirichlet `alpha in {0.75, 0.5, 0.25}`. Metrics: macro +accuracy, accuracy, F1, precision. `CONFIRMED FROM PAPER`. + +Local prototype (Eq. 1 / thesis Eq. 5.4): +`C_ij = (1/N_ij) * sum_{x_ij in class j} phi(x_ij)`. `CONFIRMED FROM PAPER`. + +Server aggregation (Eq. 2 / thesis Eq. 5.5): +`omega = (1/M) sum_i omega_i`, `Cbar_j = (1/M) sum_i C_ij`. `CONFIRMED FROM PAPER`. + +Prototype alignment (Eq. 3 / thesis Eq. 5.8–5.9): `L_R = ||C_ij - Cbar_j||^2`, with the +sum over `j = 1..K` in the joint objective. `CONFIRMED FROM PAPER`. + +Model alignment: `(mu/2) * ||omega_i - omega^{t-1}||^2`, `mu = 0.1`. `CONFIRMED FROM PAPER`. + +Inference (thesis Eqs. 5.11–5.13): embedding `z = phi(x; omega^T)`, squared L2 distance +`d_j = ||z - Cbar_j^T||^2`, prediction `argmin_j d_j`. `CONFIRMED FROM PAPER`. + +Prototype sharing is limited to locally observed classes: the thesis (Section 5.3.6) states +"prototypes are shared only for locally observed classes" and that "participants disclose to +the server which attack classes they have encountered". `CONFIRMED FROM PAPER`. + +## 4. Baseline aggregation semantics — question by question + +| # | Question | Answer | Classification | +|---|---|---|---| +| 1 | How is `C_ij` computed? | Mean embedding over the client's class-`j` samples (Eq. 1 / 5.4) | CONFIRMED FROM PAPER | +| 2 | What happens when `N_ij = 0`? | Not stated for aggregation. The thesis says only locally observed classes are shared, implying no prototype is sent | REASONABLE INFERENCE for omission; UNKNOWN for server handling | +| 3 | Is a prototype omitted? | Likely yes (see #2), but the server-side consequence is undefined | REASONABLE INFERENCE | +| 4 | Is a placeholder used? | Not described | UNKNOWN | +| 5 | Is an old prototype retained? | Not described | UNKNOWN | +| 6 | Is a zero vector used? | Not described | UNKNOWN | +| 7 | Which clients enter prototype aggregation? | Eq. 2/5.5 sums over all `M`; thesis says prototypes exist only for observed classes | UNKNOWN (paper self-inconsistent) | +| 8 | What is the denominator? | Written as `M`; whether it is `M` or the number of contributors is undefined | UNKNOWN | +| 9 | Does the implementation match Equation (2)? | No implementation exists to check | UNKNOWN | +| 10 | What does "Weighted averaging" refer to? | Only a Figure 1 caption. The thesis separately describes FedAvg as "a weighted average of the local models". Likely refers to model aggregation weighting, not prototype aggregation | REASONABLE INFERENCE | +| 11 | Is class frequency already used? | Not in Eq. 2/5.5 (equal `1/M`). The thesis intro describes FedAvg as weighted by samples, creating an internal contradiction | UNKNOWN | +| 12 | How are global prototypes distributed? | Shared back each round; `LocalUpdate` receives `Cbar^{t-1}` (Algorithm 1/2) | CONFIRMED FROM PAPER | +| 13 | How are they used in local loss? | Prototype alignment term in Eq. 3/5.6 | CONFIRMED FROM PAPER | +| 14 | How is inference performed? | Nearest global prototype by squared L2 (thesis Eqs. 5.11–5.13) | CONFIRMED FROM PAPER | + +## 5. Additional unresolved baseline ambiguities + +- **Round-0 initialisation of global prototypes.** Algorithms 1/2 say "initialize the global + prototype set for all classes" but never state how (random, zeros, first-round local + prototypes). `UNKNOWN`. +- **Prototype alignment for unseen classes.** Eq. 3 sums `L_R` over `j = 1..K`, but a client + with `N_ij = 0` has no local `C_ij`. Whether that class is skipped, uses the global + prototype as its own local prototype, or contributes zero is unspecified. `UNKNOWN`. +- **Per-batch vs end-of-round prototypes.** Algorithm 1 computes prototypes "for each batch" + and returns `{C_ij}`; Eq. 1 defines `C_ij` as the mean over the whole class on the client. + Whether clients average prototypes across batches or recompute once at round end is + unspecified. `UNKNOWN`. +- **Model aggregation weighting.** `1/M` (Eq. 5.5) vs FedAvg sample weighting (thesis + intro). `UNKNOWN`. +- **`lambda`.** No numeric value anywhere in the paper or thesis. `UNKNOWN`. +- **Random seeds.** "Averaged over three runs with different random seeds"; the seed values + are not given. `UNKNOWN`. +- **Feature preprocessing.** Normalisation, categorical encoding, and missing-value handling + are not described. `UNKNOWN`. + +## 6. Consequence for `beta = 0` equivalence + +The specification's caution is confirmed: `beta = 0` cannot be assumed equal to original +PROTEAN. Even if all clients share every class (so the participant-set question is moot), +the original uses equal `1/M` weights and the proposed control uses equal weights over +`S_j`; these coincide only when `S_j` contains all `M` clients. Under non-IID Dirichlet +partitions some clients lack some classes, so the two participant rules differ unless the +baseline denominator is `|S_j|`. + +Therefore the project must (a) reproduce PROTEAN with an explicitly documented +missing-class rule, and (b) report `equal_available` as a separate control only if needed, +exactly as the specification requires. + +## 7. Reproduction plan (from the paper/thesis) + +Because no code exists, the baseline must be re-implemented and its behaviour frozen and +reported. The minimal reproduction contract: + +1. Data pipeline: 80/20 train/test per dataset, Dirichlet non-IID partition of the training + set over 10 clients, `alpha in {0.75, 0.5, 0.25}`. +2. Model: the CNN above; embedding dimension 128. +3. Local training: cross-entropy + prototype alignment + model alignment, `mu = 0.1`, + `lambda` to be fixed and documented, 3 local epochs, 10 rounds. +4. Server: model aggregation and prototype aggregation per a documented missing-class rule. +5. Inference: nearest global prototype by squared L2. +6. Report macro accuracy, accuracy, F1, precision; add the specification's rare/unseen + metrics and the validation-based beta selection. + +Every deviation from the paper (especially `lambda`, seeds, preprocessing, missing-class +rule) must be listed in `docs/research/EXPERIMENT_PROTOCOL.md` and the handoff. + +## 8. Reference table of primary equations + +| Quantity | Paper | Thesis | +|---|---|---| +| Local prototype | Eq. (1) | Eq. (5.4) | +| Server aggregation | Eq. (2) | Eq. (5.5) | +| Joint objective | Eq. (3) | Eq. (5.6) | +| Prototype alignment | Eq. (3) term | Eq. (5.8)–(5.9) | +| Inference | Section 3.2 | Eqs. (5.11)–(5.13) | diff --git a/docs/handoff/SESSION_HANDOFF.md b/docs/handoff/SESSION_HANDOFF.md new file mode 100644 index 0000000..7c7a24d --- /dev/null +++ b/docs/handoff/SESSION_HANDOFF.md @@ -0,0 +1,346 @@ +# Session Handoff + +This is the canonical continuation point for the next agent session. It is written to be +readable without any prior chat history, terminal scrollback, or git history. + +Session date: 2026-09-24 +Session type: research study, baseline investigation, architecture planning, dataset +inspection, minimal bootstrap. **No training was run and no experimental sweep was launched.** + +--- + +## 1. Project objective + +Build the code and experiment infrastructure for a thesis study that modifies **only the +server-side prototype aggregation** of PROTEAN (Chennoufi et al., 2025), a federated +intrusion-detection framework. The proposed mechanism weights each contributing client's +class prototype by its class sample support: + +``` +Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) / ( sum_{i in S_j} N_ij^beta ) +S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } +beta in {0.0, 0.25, 0.5, 0.75, 1.0} +``` + +Goal: characterise when support weighting helps or hurts rare attack detection under +non-IID data, selecting beta on a validation set. `beta = 0` is a control, **not** assumed +identical to the original baseline. + +## 2. Authoritative source + +- `planningawal/PROTEAN_Revisi_Lurus.md` — the authoritative research specification. +- The entire `planningawal/` directory is **immutable / read-only**. Do not edit, create, + rename, move, delete, reformat, overwrite, or generate reports inside it. It must not + appear in any "files modified" list. +- Verified unchanged this session: SHA-256 + `9ba1d7beb94354db0c4b588199fb1e2823aec8a23be1625ec09a756507fbcb89` (12,050 bytes). + +## 3. Dataset locations (read-only source data) + +- `~/datasets/X-IIoTID/X-IIoTID-dataset.csv` +- `~/datasets/5G-NIDD/5G-NIDD.csv` + +Do not move, rename, overwrite, modify, or copy the full raw files into the repository. Use +configurable external paths (`configs/datasets/datasets.yaml`, with env overrides). + +Recorded SHA-256: + +- X-IIoTID: `7b9290057ee42e784da3c0d84b781815502c9205c74175c96374e71a5ffd98a0` +- 5G-NIDD: `fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59` + +## 4. Current repository tree + +``` +perdalam-protean/ +├── README.md +├── pyproject.toml +├── .gitignore +├── planningawal/ # IMMUTABLE (read-only) +│ └── PROTEAN_Revisi_Lurus.md +├── configs/ +│ ├── datasets/datasets.yaml +│ ├── experiments/experiment_protocol.yaml +│ └── models/protean_cnn.yaml +├── docs/ +│ ├── research/ +│ │ ├── RESEARCH_SPEC_INTERPRETATION.md +│ │ ├── DATASET_INSPECTION.md +│ │ ├── IMPLEMENTATION_PLAN.md +│ │ └── EXPERIMENT_PROTOCOL.md +│ ├── baseline/ +│ │ └── PROTEAN_BASELINE_INVESTIGATION.md +│ └── handoff/ +│ └── SESSION_HANDOFF.md +├── src/protean_research/ +│ ├── __init__.py +│ ├── aggregation/ +│ │ ├── __init__.py +│ │ └── prototype_aggregation.py +│ └── utils/ +│ ├── __init__.py +│ └── config.py +├── tests/ +│ └── test_prototype_aggregation.py +├── scripts/ # .gitkeep only (tracked) +├── artifacts/ +│ ├── partitions/ # tracked: deterministic manifests (.gitkeep now) +│ ├── analysis/ # tracked: small analysis outputs (.gitkeep now) +│ ├── processed/ # ignored (regenerable) +│ └── checkpoints/ # ignored (regenerable) +└── results/ # ignored (regenerable) +``` + +`.gitignore` deliberately keeps `artifacts/partitions/` and `artifacts/analysis/` +version-controlled (partition manifests are deterministic, content-hashed, and required for +paired comparisons), while ignoring `artifacts/processed/`, `artifacts/checkpoints/`, +`results/`, `logs/`, and `*.log`. `scripts/` is preserved with a `.gitkeep` placeholder. +`planningawal/` is never ignored. + +Note: the repository was **not** empty at session start. A partial scaffold already existed +(`configs/*`, `src/protean_research/aggregation/*`, `src/protean_research/utils/config.py`, +`__init__.py` files, and empty `docs/`, `scripts/`, `tests/`, `artifacts/`, `results/` +directories). `docs/` was empty and there was no handoff. That scaffold was kept and verified; +it is not the product of this session. + +## 5. Work completed this session + +1. Read the full specification and produced a tagged interpretation + (`docs/research/RESEARCH_SPEC_INTERPRETATION.md`). +2. Investigated authoritative PROTEAN sources (paper, ESORICS/Springer, HAL, author PhD + thesis) and searched for official code. +3. Established baseline aggregation semantics question-by-question with evidence tags + (`docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). +4. Inspected both raw datasets in full (read-only) and recorded shapes, label distributions, + missingness, duplicates, identifiers, and mismatches + (`docs/research/DATASET_INSPECTION.md`). +5. Verified the pre-existing config/path-resolution scaffold and aggregation module by running + them. +6. Wrote the implementation architecture (`docs/research/IMPLEMENTATION_PLAN.md`) and the + experiment protocol (`docs/research/EXPERIMENT_PROTOCOL.md`). +7. Added a minimal bootstrap: `pyproject.toml`, `README.md`, and dependency-free unit tests + for the aggregation math (`tests/test_prototype_aggregation.py`, 9 tests, all passing). +8. Recorded dataset and source hashes for integrity verification. +9. Wrote this handoff. + +## 6. Evidence and findings + +### 6.1 Official PROTEAN source status + +- **No official code exists publicly (as of 2026-09-24).** +- Author GitHub `sarachn` (Sara Chennoufi) has only `FATE-LLM` (fork) and `SecLLMHolmes` — + neither is PROTEAN. +- GitHub repository search `PROTEAN federated prototype` → 0 results; `grep.app` → none. +- The unrelated GitHub user `Chennoufi` has only a `septembre` repo — do not chase it. +- No code/artifact/supplementary link in the arXiv HTML, HAL record, or thesis text. +- **Consequence: the baseline must be reproduced from the paper/thesis.** + +Primary sources: + +- arXiv:2507.05524v1 (7 Jul 2025), ESORICS 2025 pp. 103–125, + DOI `10.1007/978-3-032-07884-1_6`. +- HAL `hal-05379203`. +- Author PhD thesis `tel-05490927` (195 pages, 2025) — restates PROTEAN with extra detail. + +### 6.2 Baseline semantics discovered + +Confirmed from the paper/thesis: + +- Local prototype (Eq. 1 / 5.4): mean embedding over the client's class-`j` samples. +- Server aggregation (Eq. 2 / 5.5): `omega = (1/M) sum_i omega_i`, + `Cbar_j = (1/M) sum_i C_ij` (equal, no `N_ij`). +- Local loss (Eq. 3 / 5.6): cross-entropy + `lambda * ||C_ij - Cbar_j||^2` + + `(mu/2) * ||omega_i - omega^{t-1}||^2`, with `mu = 0.1`. +- Inference: nearest global prototype by squared L2 (thesis Eqs. 5.11–5.13). +- Model: 2 Conv1D (64, 128 filters) + ReLU + maxpool + dropout (0.2, 0.5), flatten, + dense 128, log-softmax head. +- Setup: 10 clients, 80/20 train/test, 10 rounds, 3 local epochs, + `alpha in {0.75, 0.5, 0.25}`. +- Thesis (Sec. 5.3.6): "prototypes are shared only for locally observed classes". + +### 6.3 Missing-class findings (critical) + +- Equation (2)/5.5 sums over all `M` participants, but the paper never states how a missing + class prototype is handled. The thesis implies absent classes are not shared, but the + server denominator (`M` vs number of contributors) is undefined. +- Therefore `beta = 0` over `S_j` is **not** automatically equal to the original baseline; + equality requires the same participant rule. This is the central baseline ambiguity. +- `original_protean` is deliberately **not implemented** in + `src/protean_research/aggregation/prototype_aggregation.py`; it raises + `NotImplementedError` with an explanatory message. +- Additional unknowns: round-0 global-prototype initialisation; how the alignment loss sums + over `j = 1..K` when a client has `N_ij = 0`; per-batch vs end-of-round prototypes; + whether model aggregation is equal `1/M` or FedAvg sample-weighted (the thesis intro says + FedAvg is a weighted average, contradicting Eq. 5.5); numeric value of `lambda`; seed values; + feature preprocessing. + +### 6.4 Dataset findings + +- X-IIoTID: 820,834 × 68; labels `class1` (19), `class2` (10 general), `class3` (binary). + `class2` benign = `Normal`. After dropping `crypto-ransomware`: 9 classes. `'-'` + placeholders in 42 columns; 6,222 duplicate rows; label typo `Lateral _movement`. + Identifiers: `Date`, `Timestamp`, `Scr_IP`, `Scr_port`, `Des_IP`, `Des_port`. +- 5G-NIDD: 1,215,890 × 52; `Attack Type` (9 values, benign = `Benign`), `Label` binary, + `Attack Tool` (6). 8 attack classes + Benign = 9. Heavy structural missingness (e.g. `dVid` + 99.83%, `dTos` 77.56%); 1 duplicate row. Identifiers: empty-header index, `Seq`, `Offset`. +- Both map to PROTEAN's 9-class setup, but PROTEAN's exact preprocessing is unspecified and + must be frozen here. + +## 7. Decisions made + +1. **Treat `planningawal/` as immutable** and keep all outputs elsewhere. +2. **Do not implement `original_protean` until its semantics are verified.** Encode only the + unambiguous modes (`support_weighted`, `equal_available`). Rationale: avoid silently + assuming baseline behaviour. +3. **Keep `equal_available` as a dormant control**, introduced only if baseline investigation + shows the participant rule differs from `S_j` (per the specification). +4. **Centralise dataset paths** in `configs/datasets/datasets.yaml` with env overrides; never + hardcode in source. +5. **Separate raw and generated data**; generated outputs under `artifacts/`. +6. **Use `class2` for X-IIoTID and `Attack Type` for 5G-NIDD** as the multi-class labels, drop + `crypto-ransomware`, treat `Normal`/`Benign` as benign. Matches the paper. +7. **Keep the pre-existing scaffold** (configs + aggregation + utils) and extend it minimally + rather than restructuring; no evidence justified a different layout. +8. **Add dependency-free unit tests** for aggregation invariants now; defer data/federated + tests until those modules exist. +9. **Do not initialize git during the planning session** (not requested at the time). A + follow-up checkpoint task then initialized Git and added remote `origin`; see Section 11. +10. **Mark seeds, `lambda`, validation fraction, Dirichlet allocation, and preprocessing as + PROVISIONAL**; they must be frozen before any sweep. + +## 8. Unresolved questions + +1. Baseline missing-class rule and prototype-aggregation denominator (`M` vs `|S_j|`). +2. Round-0 initialisation of global prototypes for all `K` classes. +3. `lambda` value (paper gives `mu = 0.1` only). +4. Whether model-parameter aggregation is equal or FedAvg sample-weighted. +5. How the prototype alignment loss treats `N_ij = 0` classes. +6. Per-batch vs end-of-round local prototype computation. +7. Validation split fraction and procedure (the paper has no validation set). +8. Exact Dirichlet allocation algorithm (affects realised `N_ij`). +9. Exact seed values and whether metrics are averaged per client then per seed. +10. Feature preprocessing: encoding, scaling, `'-'`/NaN handling, duplicate-row policy. +11. ~~Whether to initialize git to protect the immutable source.~~ Resolved: Git was + initialized in the follow-up checkpoint task (see Section 11). +12. Compute budget: no timing measured; a profiling smoke test is required before any sweep. + +## 9. Files created this session + +- `docs/research/RESEARCH_SPEC_INTERPRETATION.md` +- `docs/research/DATASET_INSPECTION.md` +- `docs/research/IMPLEMENTATION_PLAN.md` +- `docs/research/EXPERIMENT_PROTOCOL.md` +- `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md` +- `docs/handoff/SESSION_HANDOFF.md` +- `README.md` +- `pyproject.toml` +- `.gitignore` +- `tests/test_prototype_aggregation.py` + +## 10. Files modified this session + +- None. No existing file was edited. `planningawal/` was not touched (and must not be). + +Pre-existing files (present at session start, verified but not modified): +`configs/datasets/datasets.yaml`, `configs/experiments/experiment_protocol.yaml`, +`configs/models/protean_cnn.yaml`, `src/protean_research/__init__.py`, +`src/protean_research/aggregation/__init__.py`, +`src/protean_research/aggregation/prototype_aggregation.py`, +`src/protean_research/utils/__init__.py`, `src/protean_research/utils/config.py`. + +## 11. Git status + +The planning session ran with **no Git repository**. A follow-up checkpoint task then +initialized Git: + +- Branch: `main` +- Remote `origin`: `git@github.com:willtanoe/prototype-agregation.git` +- Initial checkpoint commit: `chore: checkpoint PROTEAN research planning and architecture` + (exact hash reported in the checkpoint task's terminal summary; deliberately not embedded + here to avoid a recursive "hash inside the commit" loop). +- `.gitkeep` placeholders were added for `scripts/`, `artifacts/partitions/`, and + `artifacts/analysis/`. `artifacts/processed/`, `artifacts/checkpoints/`, `results/`, + `logs/`, and `*.log` are ignored. +- `planningawal/PROTEAN_Revisi_Lurus.md` is tracked exactly as-is and was not modified. +- Raw datasets are outside the repository and are not tracked. + +Because this is the initial commit, `planningawal/` was verified with SHA-256 hashing and +directory listing rather than `git diff`. + +## 12. Verification performed + +- Read the full specification (118 lines). +- Read the full paper HTML and extracted the author PhD thesis text (195 pages). +- GitHub API searches (author account, repositories), DuckDuckGo, `grep.app`. +- Full pandas load of both datasets; recorded shape, dtypes, label counts, missingness, + duplicates, identifier columns. +- SHA-256 of both datasets and of `planningawal/PROTEAN_Revisi_Lurus.md`. +- Ran the pre-existing aggregation module and config resolver (modes, weights, path + resolution, `original_protean` raises `NotImplementedError`). +- Ran `python -m pytest -q` in a temporary venv: **9 passed**. +- Confirmed no training or sweep was launched. +- Final check: `planningawal/` contains only `PROTEAN_Revisi_Lurus.md` with the same hash; + raw datasets retain the same hashes and mtimes. + +Checkpoint task (follow-up): + +- `git rev-parse --is-inside-work-tree` before init: not a repository. +- Searched the whole tree for `*.csv`, `*.parquet`, `*.feather`, `*.pkl`, `*.pickle`, model + checkpoints: none present; largest file is the 12 KB spec. +- Scanned tracked text for secret patterns (`API_KEY`, `SECRET`, `TOKEN`, `PASSWORD`, + `PRIVATE KEY`, `BEGIN OPENSSH`, `BEGIN RSA`, `sk-`, `ghp_`, `github_pat_`): none found. +- `find planningawal -maxdepth 2 -type f` → only `PROTEAN_Revisi_Lurus.md`. +- Re-ran `pytest`: 9 passed. + +## 13. Exact next task + +**Implement the deterministic data foundation, and freeze the remaining protocol choices +before coding.** + +Concretely, the next session should: + +1. In `docs/research/EXPERIMENT_PROTOCOL.md`, replace the PROVISIONAL entries with explicit + frozen decisions and rationale for: preprocessing recipe, duplicate-row policy, + `'-'`/NaN handling, the Dirichlet allocation algorithm, the train/validation/test split + procedure and fraction, the seed set, `lambda`, batch size, and learning rate. These are + project decisions because the paper does not specify them; label them as reproduction + deviations. +2. Create `src/protean_research/data/` with `loading.py`, `preprocessing.py`, `splits.py`, + `partition.py`, `manifest.py`, `characterization.py`, `rare_unseen.py` implementing the + flow in `docs/research/IMPLEMENTATION_PLAN.md` Section 3. +3. Persist one partition manifest per `(dataset, alpha, seed)` under `artifacts/partitions/` + with a content hash, and derive rare/unseen metadata from the training split. +4. Add tests for determinism of partition generation (strategy item 8) and identical manifest + reload (item 9), plus the non-IID characterization fields. + +Do **not** start model training or a beta sweep. The baseline missing-class rule is still +unresolved, so `original_protean` remains unimplemented. + +## 14. Recommended next-session prompt + +``` +You are continuing the perdalam-protean research repository. + +FIRST, read these two files completely: + 1. docs/handoff/SESSION_HANDOFF.md + 2. planningawal/PROTEAN_Revisi_Lurus.md + +Then read, as needed: + - docs/research/IMPLEMENTATION_PLAN.md + - docs/research/EXPERIMENT_PROTOCOL.md + - docs/research/DATASET_INSPECTION.md + - docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md + +Critical rules: + - planningawal/ is immutable. Do not edit/create/move/rename/delete anything inside it. + - ~/datasets/X-IIoTID/X-IIoTID-dataset.csv and ~/datasets/5G-NIDD/5G-NIDD.csv are + read-only external source data. Do not modify or copy them into the repo. + - Do NOT run full training or an experimental sweep. + +Your task is Section 13 of the handoff: first freeze the PROVISIONAL protocol decisions in +docs/research/EXPERIMENT_PROTOCOL.md, then implement the deterministic data foundation in +src/protean_research/data/ with partition-manifest persistence, rare/unseen metadata, and +tests for partition determinism and manifest reload. + +Start by reading the handoff, then continue from the documented state. Update +docs/handoff/SESSION_HANDOFF.md at the end of your session. +``` diff --git a/docs/research/DATASET_INSPECTION.md b/docs/research/DATASET_INSPECTION.md new file mode 100644 index 0000000..0745aaa --- /dev/null +++ b/docs/research/DATASET_INSPECTION.md @@ -0,0 +1,192 @@ +# Dataset Inspection + +Status: completed inspection (read-only) +Inspected: 2026-09-24 +Method: full load with pandas 3.0.6 / Python 3.12 in an isolated temporary virtualenv. + +The raw files are external source data. They were only read; paths are configured in +`configs/datasets/datasets.yaml`. No files were moved, renamed, overwritten, or copied +into the repository. + +## Integrity (recorded before and used for final verification) + +| Dataset | Path | Size (bytes) | SHA-256 | +|---|---|---|---| +| X-IIoTID | `~/datasets/X-IIoTID/X-IIoTID-dataset.csv` | 355,308,902 (338.8 MiB) | `7b9290057ee42e784da3c0d84b781815502c9205c74175c96374e71a5ffd98a0` | +| 5G-NIDD | `~/datasets/5G-NIDD/5G-NIDD.csv` | 275,265,610 (262.5 MiB) | `fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59` | + +Source filesystem modification times: X-IIoTID `2021-10-07`, 5G-NIDD `2023-02-18` +(i.e. unchanged by this session). + +--- + +## 1. X-IIoTID + +- Shape: **820,834 rows × 68 columns**. +- In-memory (deep) usage when loaded: ~671 MB. +- Encoding: UTF-8. +- dtypes: 52 string columns, 9 int64, 7 bool. +- Many numeric-looking columns are stored as strings because they contain the missing + placeholder `'-'` (e.g. `Duration`, `Scr_bytes`, `Scr_pkts`, `Avg_*`, `Std_*`). + +### Label columns + +`class1` (fine-grained, 19 values): + +| class1 | count | +|---|---| +| Normal | 421,417 | +| RDOS | 141,261 | +| Scanning_vulnerability | 52,852 | +| Generic_scanning | 50,277 | +| BruteForce | 47,241 | +| MQTT_cloud_broker_subscription | 23,524 | +| Discovering_resources | 23,148 | +| Exfiltration | 22,134 | +| insider_malcious | 17,447 | +| Modbus_register_reading | 5,953 | +| False_data_injection | 5,094 | +| C&C | 2,863 | +| Dictionary | 2,572 | +| TCP Relay | 2,119 | +| fuzzing | 1,313 | +| Reverse_shell | 1,016 | +| crypto-ransomware | 458 | +| MitM | 117 | +| Fake_notification | 28 | + +`class2` (general classes, 10 values) — the label used by PROTEAN: + +| class2 | count | +|---|---| +| Normal | 421,417 | +| RDOS | 141,261 | +| Reconnaissance | 127,590 | +| Weaponization | 67,260 | +| Lateral _movement | 31,596 | +| Exfiltration | 22,134 | +| Tampering | 5,122 | +| C&C | 2,863 | +| Exploitation | 1,133 | +| crypto-ransomware | 458 | + +`class3` (binary, 2 values): `Normal` 421,417; `Attack` 399,417. + +- Benign label: **`Normal`**. +- Attack labels (general, after removing `crypto-ransomware` as the paper does): + `RDOS`, `Reconnaissance`, `Weaponization`, `Lateral _movement`, `Exfiltration`, + `Tampering`, `C&C`, `Exploitation` → **8 attack classes + Normal = 9 classes**, matching + the PROTEAN `K` for this dataset. + +### Missing data / anomalies + +- True empty cells: `Date` 331 (0.04%), `Timestamp` 297 (0.04%). All other columns have + zero empty cells. +- Documented missing placeholder `'-'` appears in 42 columns. Highest counts: + `Scr_bytes`, `Des_bytes`, `total_bytes`, `paket_rate`, `byte_rate`, `Scr_bytes_ratio`, + `Des_bytes_ratio` (≈223,871 each), `Scr_pkts`/`Des_pkts`/`Scr_ip_bytes`/`Des_ip_bytes`/ + `total_packet`/`*_ratio` (≈152,743), `Duration` (75,761), and the `Avg_*`/`Std_*` system + columns (≈479–526). +- Exact duplicate rows: **6,222**. +- Label anomaly: the value is `Lateral _movement` (extra space before the underscore), + whereas the paper writes "Lateral Movement". Label normalisation is required. +- Fine-grained typos exist (`insider_malcious`). Not used by the general-class study but + relevant if `class1` is ever used. + +### Non-feature / identifier fields + +`Date`, `Timestamp` (temporal), `Scr_IP`, `Scr_port`, `Des_IP`, `Des_port` (network +endpoints). These must not be model features without an explicit documented decision. +`Protocol`/`Service` are categorical but plausibly valid flow features. + +--- + +## 2. 5G-NIDD + +- Shape: **1,215,890 rows × 52 columns**. +- In-memory (deep) usage when loaded: ~551 MB. +- Encoding: UTF-8. +- dtypes: 32 float64, 12 int64, 8 string columns. +- Column 0 has an empty header (an unnamed row index). + +### Label columns + +`Label` (binary): `Malicious` 738,153; `Benign` 477,737. + +`Attack Type` (9 values) — the multi-class label used by PROTEAN: + +| Attack Type | count | +|---|---| +| Benign | 477,737 | +| UDPFlood | 457,340 | +| HTTPFlood | 140,812 | +| SlowrateDoS | 73,124 | +| TCPConnectScan | 20,052 | +| SYNScan | 20,043 | +| UDPScan | 15,906 | +| SYNFlood | 9,721 | +| ICMPFlood | 1,155 | + +`Attack Tool` (6 values): `Benign` 477,737; `Hping3` 468,216; `Goldeneye` 140,812; +`Torshammer` 59,870; `Nmap` 56,001; `Slowloris` 13,254. + +- Benign label: **`Benign`**. +- Attack classes: 8 (all `*Flood`/`*Scan`/`*DoS`), giving **9 classes including Benign**, + matching PROTEAN's `K` for this dataset. + +### Missing data / anomalies + +Large, structurally correlated missingness (likely attack-type dependent): + +| Column | missing | % | +|---|---|---| +| dVid | 1,213,881 | 99.83% | +| sVid | 1,101,319 | 90.58% | +| DstWin | 1,038,812 | 85.44% | +| DstTCPBase | 985,843 | 81.08% | +| SrcWin | 973,470 | 80.06% | +| dTos, dHops, dDSb, dTtl | 943,067 | 77.56% | +| DstGap, SrcGap, SrcTCPBase | 937,219 | 77.08% | +| sHops, sTtl, sTos, sDSb | 214 | 0.02% | + +- Exact duplicate rows: **1**. +- No `'-'` placeholder tokens; missingness is real NaN. + +### Non-feature / identifier fields + +Empty-header index column (`""`), `Seq` (sequence number), `Offset` (flow ordering). +`Label`, `Attack Type`, `Attack Tool` are targets and must not be features. +`Proto`, `sDSb`, `dDSb`, `Cause`, `State` are categorical features. + +--- + +## 3. Mapping to PROTEAN assumptions + +| PROTEAN assumption | X-IIoTID | 5G-NIDD | Match | +|---|---|---|---| +| 9 attack classes (incl. benign) | Normal + 8 attacks after dropping `crypto-ransomware` | Benign + 8 attacks | Yes | +| Multi-class label | `class2` | `Attack Type` | Yes | +| Benign present | `Normal` | `Benign` | Yes | +| Dirichlet over classes | feasible | feasible | Yes | +| 80/20 train/test | feasible | feasible | Yes | +| Time-agnostic Dirichlet | `Date`/`Timestamp` exist but unused | `Seq`/`Offset` exist but unused | Paper is time-agnostic; no temporal split | + +### Mismatches / items to freeze + +1. `Lateral _movement` typo must be normalised before class-index mapping. +2. PROTEAN's exact feature preprocessing is unspecified (encoding, scaling, missing + handling). This must be frozen here and reported as a reproduction deviation. +3. X-IIoTID numeric columns carry `'-'`; coercion strategy must be explicit and identical + across aggregation variants. +4. Exact duplicate rows exist (X-IIoTID 6,222; 5G-NIDD 1). Dropping them is a decision that + must be frozen and applied identically to all variants. +5. 5G-NIDD's high structural missingness means imputation/dropping choices strongly affect + features; these choices must be recorded. +6. Identifier/temporal fields should be excluded from features by default. + +## 4. Reproduction of this inspection + +The inspection script is not stored in the repository (to keep the bootstrap minimal), but +was a straight `pandas.read_csv(..., low_memory=False)` plus `value_counts`, `isna`, and +`duplicated` per dataset. Re-running it must reproduce the counts above for the recorded +hashes. A future `scripts/inspect_datasets.py` may formalise it (see implementation plan). diff --git a/docs/research/EXPERIMENT_PROTOCOL.md b/docs/research/EXPERIMENT_PROTOCOL.md new file mode 100644 index 0000000..1f35dfe --- /dev/null +++ b/docs/research/EXPERIMENT_PROTOCOL.md @@ -0,0 +1,177 @@ +# Experiment Protocol + +Status: frozen design (to be executed in later sessions) +Machine-readable counterpart: `configs/experiments/experiment_protocol.yaml` + +Values marked **PROVISIONAL** require confirmation before any scientific sweep. Values +marked **FROZEN** must remain identical across every aggregation variant. + +--- + +## 1. Datasets + +FROZEN: + +| Key | Dataset | Label column | Benign | Dropped | Classes `K` | +|---|---|---|---|---|---| +| `xiiotid` | X-IIoTID | `class2` | `Normal` | `crypto-ransomware` | 9 | +| `5gnidd` | 5G-NIDD | `Attack Type` | `Benign` | — | 9 | + +Details and mismatches: `docs/research/DATASET_INSPECTION.md`. +`crypto-ransomware` is removed exactly as the paper does (very few valid instances). + +## 2. Non-IID partition (Dirichlet) + +FROZEN: symmetric Dirichlet over classes, `num_clients = 10`, `alpha ∈ {0.75, 0.50, 0.25}`. + +- Partitions are generated from the **training split only**. +- Class proportions are drawn per client from `Dirichlet(alpha)` and used to allocate + training samples per class to clients. +- The exact Dirichlet implementation (per-class client allocation vs per-client class + sampling) must be fixed and documented, because it changes realised `N_ij`. `PROVISIONAL`. +- Success criterion for reproducing the paper's setting: at low alpha, several clients miss + several classes, consistent with the paper's reported counts (e.g. X-IIoTID `alpha=0.25`: + 8/10 clients miss 1–5 classes). + +## 3. Splits + +- Test: 20% of each dataset (paper uses 80/20). FROZEN. +- Validation: a separate split used only for beta selection. The paper has **no** validation + set, so this is an explicit project addition. `PROVISIONAL` fraction and procedure. +- Requirement: train/validation/test come from a single deterministic split with a recorded + seed, and the same split is reused for every beta variant. + +## 4. Seeds + +- The paper averages over three runs with unnamed seeds. `PROVISIONAL` set: `{0, 1, 2}`. +- Once frozen, the seed set is identical across all aggregation variants. +- Each seed deterministically controls: split, Dirichlet partition, model initialisation, + batch order/shuffling. +- A beta run MUST NOT derive a new partition; it loads the manifest for + `(dataset, alpha, seed)`. + +## 5. Beta candidates and modes + +FROZEN: `beta ∈ {0.00, 0.25, 0.50, 0.75, 1.00}`, one beta for all classes and clients. + +Modes: + +- `original_protean` — verified baseline semantics. Not yet encodable (see baseline + investigation). This is the baseline to reproduce. +- `support_weighted` — `w_ij = N_ij^beta`, normalised over `S_j` (proposed mechanism). +- `equal_available` — equal weights over `S_j`; a methodological control introduced only if + the baseline participant rule differs from `S_j`. Not to be presented as a novelty. + +`S_j = { i | N_ij > 0 and a valid local prototype C_ij exists }`. + +## 6. Partition persistence + +Each `(dataset, alpha, seed)` produces one persisted manifest containing at least: + +- dataset name, alpha, seed, num_clients, K; +- per-client sample indices or a reproducible reference plus a content hash; +- per-client `N_ij` matrix `[client][class]`; +- class label mapping; +- split indices for train/validation/test; +- preprocessing/config hash and code version. + +Manifests live under `artifacts/partitions/`. Reloading a manifest must reproduce the same +partition (tested). Training reads manifests; it does not regenerate them. + +## 7. Rare and unseen classes + +Derived from the **training** split of the manifest, before beta selection. + +- Rare classes on client `i`: the two **attack** classes with the smallest positive support + `N_ij > 0`. If exactly one attack class is available, use one. If none, the client is + excluded from the rare-class aggregate. +- Unseen classes on client `i`: attack classes with `N_ij = 0`. Evaluated separately from + rare classes; never merged. +- The benign class is not a rare class. + +Persisted metadata (minimum one row per `(dataset, alpha, seed, client, class)`): +`dataset, alpha, seed, client_id, class, N_ij, rare_flag, unseen_flag`. + +## 8. Validation-based beta selection + +Procedure, strictly ordered: + +1. **Candidate evaluation (validation only).** For each beta candidate, train with the + paired manifest and compute validation metrics. Primary criterion: rare-class macro-F1, + computed per client that has rare classes, then averaged over those clients. +2. **Beta selection.** Choose the beta with the best mean validation rare-class macro-F1 + (ties broken by a pre-declared rule, e.g. smaller beta). The result is + "best among tested candidates under this procedure", never a global optimum. +3. **Final test evaluation.** Evaluate the selected beta on the test split. Report overall + macro-F1, rare-class recall, rare-class macro-F1, and per-client results, plus the test + results of all candidates for transparency. + +Test data is never used in steps 1–2. The selection code must not have access to test +outputs; test evaluation is a separate step keyed on the selected beta. + +## 9. Required metrics + +Per dataset/alpha/seed/beta and per client, record: + +- rare-class macro-F1 (primary selection metric); +- rare-class recall; +- overall macro-F1; +- accuracy (for comparison with the paper); +- per-class precision/recall/F1 and confusion counts; +- per-client metrics. + +Also record the validation/test rare-class support counts, so rare-class metrics are never +read without their sample sizes. Results are reported as mean and variation across seeds. + +## 10. Non-IID characterization + +For every manifest, record at least: `N_ij`, class total, number of clients containing class +`j`, `min N_ij`, `max N_ij`, `mean N_ij`, `std N_ij`, coefficient of variation, and +`max/min` ratio where defined. Alpha alone is insufficient; different seeds realise +different distributions. + +## 11. Direct prototype analysis + +Separate the direct mathematical aggregation effect from the end-to-end training effect. + +- During a run, persist at least one round's fixed `C_ij` and `N_ij` snapshot. +- Reusing those snapshots, compute `Cbar_j(beta)` for several beta without retraining. +- Report `|| Cbar_j(beta) - Cbar_j(0) ||`; optionally the normalised aggregation weights, + distances to contributing prototypes, pairwise global-prototype displacement, and support + imbalance. +- This analysis complements, and never replaces, end-to-end evaluation. + +## 12. Paired comparison guarantee + +For each `(dataset, alpha, seed)`, every beta variant and every mode reuses: + +- the same partition manifest (content-hash verified); +- the same train/validation/test split; +- the same class mapping; +- the same model initialisation policy; +- the same hyperparameters; +- the same client membership. + +A beta run that regenerates a fresh Dirichlet partition is a protocol violation. + +## 13. Frozen vs provisional summary + +| Item | Value | Status | +|---|---|---| +| Datasets | X-IIoTID, 5G-NIDD | FROZEN | +| Clients | 10 | FROZEN | +| Alpha | 0.75, 0.50, 0.25 | FROZEN | +| Beta candidates | 0, 0.25, 0.50, 0.75, 1.00 | FROZEN | +| Test fraction | 0.20 | FROZEN | +| Rounds / local epochs | 10 / 3 | FROZEN | +| `mu` | 0.1 | FROZEN | +| `lambda` | unknown | PROVISIONAL | +| Batch size / learning rate | unknown | PROVISIONAL | +| Validation fraction / procedure | unknown | PROVISIONAL | +| Seeds | 0, 1, 2 | PROVISIONAL | +| Dirichlet allocation algorithm | unknown | PROVISIONAL | +| Preprocessing recipe | unknown | PROVISIONAL | +| Missing-class baseline rule | unknown | PROVISIONAL / blocked on baseline | + +The provisional items must be resolved and recorded here before the first sweep. Until then +only smoke tests are permitted. diff --git a/docs/research/IMPLEMENTATION_PLAN.md b/docs/research/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..1f535a7 --- /dev/null +++ b/docs/research/IMPLEMENTATION_PLAN.md @@ -0,0 +1,217 @@ +# Implementation Plan + +Status: design document (no full training implemented in this session) +Scope: architecture for the server-side prototype-aggregation study on top of a reproduced +PROTEAN baseline. + +Governing constraint: only server-side prototype aggregation changes. Architecture, model +aggregation, loss terms, and inference follow the verified baseline +(`docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). + +--- + +## 1. Repository layout + +``` +perdalam-protean/ +├── planningawal/ # IMMUTABLE source material (read-only) +│ └── PROTEAN_Revisi_Lurus.md +├── docs/ +│ ├── research/ # spec interpretation, datasets, plan, protocol +│ ├── baseline/ # PROTEAN baseline investigation +│ └── handoff/ # session handoff (canonical continuation point) +├── configs/ +│ ├── datasets/datasets.yaml +│ ├── experiments/experiment_protocol.yaml +│ └── models/protean_cnn.yaml +├── src/protean_research/ +│ ├── data/ # (planned) loading, cleaning, partitions, manifests +│ ├── federated/ # (planned) client update, round orchestration +│ ├── aggregation/ # (exists) pure prototype aggregation + modes +│ ├── models/ # (planned) CNN embedding/classifier +│ ├── evaluation/ # (planned) metrics, rare/unseen, per-client +│ └── utils/ # (exists) config/path resolution; (planned) seeding +├── scripts/ # (planned) thin CLI entry points +├── tests/ # (exists: none yet; planned aggregation tests) +├── artifacts/ # generated: partitions, processed, checkpoints, analysis +└── results/ # experiment result tables/plots +``` + +Existing code at the start of this session: `configs/*`, `src/protean_research/__init__.py`, +`src/protean_research/aggregation/*`, `src/protean_research/utils/config.py`. Everything +marked "(planned)" is future work. + +## 2. Configuration + +Centralised YAML, loaded through `src/protean_research/utils/config.py`. + +- `configs/datasets/datasets.yaml` — raw paths (external, `~`-expanded), env overrides + (`PROTEAN_XIIOTID_RAW_PATH`, `PROTEAN_5GNIDD_RAW_PATH`), label columns, benign label, + drop labels, missing tokens, non-feature columns, and `generated_root`. +- `configs/experiments/experiment_protocol.yaml` — datasets, alphas, betas, modes, training + and evaluation settings, provisional seeds, artifact directories. +- `configs/models/protean_cnn.yaml` — frozen architecture and loss/inference contract. + +Rules: + +- No raw dataset path is hardcoded in source. Resolution precedence: env override, then + `raw_path`, then `~` expansion. Generated artifacts always resolve under `generated_root`. +- Raw data and generated data are strictly separated. +- `lambda` is `null` and seeds are `provisional`; both must be frozen before experiments. + +## 3. Data flow + +``` +raw CSV (external) + -> load + clean (label normalise, '-'/NaN handling, drop duplicates policy) + -> encode + scale features (frozen recipe) + -> train/validation/test split (frozen) + -> Dirichlet partition of TRAIN over 10 clients --(persist manifest)--> + per-client class support N_ij + -> per (dataset, alpha, seed): reuse the SAME manifest for every beta variant + -> federated training (original / equal_available / support_weighted) + -> validation metrics -> beta selection + -> test metrics for selected beta (+ all candidates reported) +``` + +Partition manifests are the unit of pairing. A beta run must load a manifest, never +regenerate a partition. + +## 4. Modules + +### `data/` (planned) + +- `loading.py`: read CSV with configured dtypes/encoding; apply `missing_tokens`; drop + non-feature columns; normalise labels (including the `Lateral _movement` typo). +- `preprocessing.py`: deterministic encoding/scaling; persisted so test data uses train + statistics. +- `splits.py`: train/validation/test split with a fixed seed policy. +- `partition.py`: Dirichlet non-IID partition of the training split over `num_clients`. +- `manifest.py`: save/load partition manifests; verify content hash on reload. +- `characterization.py`: compute `N_ij`, class totals, contributor counts, and inequality + statistics (min/max/mean/std/CV/max-min ratio). +- `rare_unseen.py`: derive rare (two smallest positive attack support) and unseen + (`N_ij = 0` attack classes) metadata per `(dataset, alpha, seed, client, class)`. + +### `models/` (planned) + +- `protean_cnn.py`: two Conv1D layers (64, 128 filters, ReLU, max-pool, dropout 0.2/0.5), + flatten, dense 128 (embedding `phi`), classification head (log-softmax). +- `prototypes.py`: compute `C_ij` from embeddings; require `N_ij > 0`. + +### `federated/` (planned) + +- `client.py`: local update minimising Eq. (3)/(5.6): cross-entropy + prototype alignment + + model alignment. Returns local model and observed-class prototypes. +- `server.py`: model aggregation + prototype aggregation dispatch by mode; builds `S_j`. +- `round.py`: one communication round; records per-round prototype snapshots for direct + analysis. + +### `aggregation/` (exists) + +Pure, dependency-free functions implementing only the unambiguous modes: + +- `beta_weights(supports, beta)` → `N_ij ** beta` (unnormalised). +- `aggregation_weights(prototypes, mode, beta, supports)` → normalised weights summing to 1. +- `aggregate_prototypes(...)` → `Cbar_j` for one class. +- `AggregationMode`: `original_protean` (raises `NotImplementedError` until verified), + `support_weighted`, `equal_available`. + +Planned addition: a server-facing helper that accepts `{client_id: (C_ij, N_ij)}` for class +`j`, constructs `S_j` (drop clients with `N_ij = 0` or missing prototype), and delegates to +`aggregate_prototypes`. This keeps all weighting logic in one tested place. + +### `evaluation/` (planned) + +- `metrics.py`: accuracy, macro-F1, per-class precision/recall/F1. +- `rare_unseen.py`: rare-class macro-F1 and recall per client; unseen-class reporting. +- `per_client.py`: per-client result aggregation. + +### `utils/` (partly exists) + +- `config.py` (exists): YAML loading and path resolution. +- `seeding.py` (planned): deterministic seeding for Python/NumPy/PyTorch. +- `integrity.py` (planned): hash helpers for manifests and source verification. + +### `analysis/` (planned) + +- `direct_prototype.py`: given a saved round snapshot of fixed `C_ij, N_ij`, compute + `Cbar_j(beta)` for several beta without retraining and report + `||Cbar_j(beta) - Cbar_j(0)||`, weights, distances, displacement, imbalance. + +## 5. Aggregation API concept + +``` +aggregate_prototypes(prototypes, mode, beta=None, supports=None) -> list[float] +``` + +Contract: + +- `prototypes[i]` is `C_ij`; `supports[i]` is `N_ij > 0`; order is client order. +- `support_weighted`: weights `N_ij ** beta`, normalised over `S_j`. +- `equal_available`: equal weights over `S_j` (control; not for `beta > 0` claims). +- `original_protean`: only after the baseline missing-class rule is verified and encoded. +- Validates finiteness, strictly positive supports, equal dimensions; normalised weights sum + to 1; never emits NaN/Inf for valid inputs. + +## 6. Experiment orchestration + +- A run is keyed by `(dataset, alpha, seed)`. +- For each key: generate or load one partition manifest; derive rare/unseen metadata once. +- For each beta in the candidate set: train using the same manifest and the same model + initialisation policy; evaluate on validation; store per-round metrics. +- Select the best beta by validation rare-class macro-F1. +- Evaluate the selected beta (and report all candidates) on the test split. +- Every run records the manifest hash, config hash, and code version. +- Orchestration must make test leakage structurally hard: selection code only receives + validation outputs; test outputs are produced by a separate step keyed on the selected + beta. + +## 7. Testing strategy + +Required tests (from the specification), to be implemented as the code is built: + +1. `beta = 0` equals the arithmetic mean over the same `S_j`. +2. `beta = 1` equals sample-count weighting (equals pooling of included embeddings). +3. One contributing client gives the same output for all beta. +4. Equal supports give the same output for all beta. +5. A missing class does not enter `S_j`. +6. Normalised weights sum to one. +7. Valid inputs never produce NaN/Inf. +8. Partition generation is deterministic for a fixed seed. +9. Saved partition manifests reload identically. +10. Baseline behaviour is unchanged by adding the proposed mode. +11. Validation/test separation is maintained. + +This session adds the dependency-free subset (1–7) in `tests/test_prototype_aggregation.py`. +Items 8–11 require the future data/federated modules and are deferred. + +## 8. Compute feasibility + +`SOURCE FACT` Paper environment: Linux Ubuntu 22.04.4, 11th Gen Intel i7-11800H (16 threads), +31 GiB RAM. No GPU is reported. + +Local environment (measured this session): 8 CPU threads, ~29 GiB RAM, ~113 GiB free disk. +No CUDA device was probed. + +Assessment: + +- **Framework**: PyTorch (paper). No GPU is strictly required by the paper's setup. +- **Memory**: full CSVs load at ~671 MB (X-IIoTID) and ~551 MB (5G-NIDD) in pandas. Training + on a partitioned subset plus a CNN with 128-dim embeddings is comfortably within 29 GiB. +- **Clients**: 10 clients, simulated sequentially (standard for cross-silo CPU runs). +- **Storage**: partitions and manifests are small; per-round prototype snapshots are + `K x d` floats per client. Checkpoints of the small CNN are small. The main storage + pressure is the processed feature cache, which should be stored once under + `artifacts/processed/`. +- **Likely bottleneck**: local training over 10 clients × 3 epochs × 10 rounds per + `(dataset, alpha, seed, beta)`. With 5 betas × 3 alphas × 3 seeds × 2 datasets this is + 90 training runs; wall-clock is unknown and must not be estimated without measurement. +- **Checkpoint strategy**: save per-round model and prototype snapshots so direct prototype + analysis and interrupted runs do not require retraining. +- **One-GPU execution**: feasible in principle (CPU-only was enough for the paper); a GPU + would only speed up local training. + +`UNRESOLVED` No timing benchmark was run (this session forbids launching training). Define a +future profiling smoke test: 1 dataset, 1 alpha, 1 seed, 1 beta, 1 round, 1 epoch, and record +wall-clock and peak memory before any sweep. diff --git a/docs/research/RESEARCH_SPEC_INTERPRETATION.md b/docs/research/RESEARCH_SPEC_INTERPRETATION.md new file mode 100644 index 0000000..41123f7 --- /dev/null +++ b/docs/research/RESEARCH_SPEC_INTERPRETATION.md @@ -0,0 +1,219 @@ +# Research Specification Interpretation + +Status: interpretation document +Authoritative source: `planningawal/PROTEAN_Revisi_Lurus.md` (immutable, read-only) +Companion documents: `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`, +`docs/research/EXPERIMENT_PROTOCOL.md`, `docs/research/IMPLEMENTATION_PLAN.md` + +This document translates the local research specification into implementation-relevant +statements. Every statement is labelled with one of: + +- `SOURCE FACT` — written explicitly in `planningawal/PROTEAN_Revisi_Lurus.md` or in the + published PROTEAN paper/thesis. +- `INFERENCE` — a defensible reading that is not written verbatim. +- `IMPLEMENTATION DECISION` — a choice made in this project that the specification leaves open. +- `UNRESOLVED` — cannot be settled without more evidence. + +The terminology of the specification is preserved. This project studies **server-side +prototype aggregation** only. + +--- + +## 1. Research focus + +`SOURCE FACT` The study optimises how the server combines class prototypes in PROTEAN for +rare attack detection under non-IID federated data. The focus is the server-side prototype +aggregation step, because the global prototype is used both as a training alignment target +and as the inference classifier (nearest prototype). + +`SOURCE FACT` The novelty is **not** "using N_ij" or "adding a beta". FedProto already +weights prototypes by sample counts (Tan et al., 2022). Novelty must come from mechanism +differences, behavioural findings, and empirical evidence. + +`SOURCE FACT` The reference aggregation for discussion is Equation (2) of the paper: +equal contribution of local prototypes (`1/M * sum_i C_ij`), even though Figure 1 uses the +phrase "Weighted averaging". + +## 2. Original PROTEAN mechanism (as stated in the spec) + +- `SOURCE FACT` Local prototype (Eq. 1): for class `j` with `N_ij > 0`, + `C_ij = (1 / N_ij) * sum_{x in class j} phi(x)`. +- `SOURCE FACT` Clients send local model parameters and local class prototypes to the server. +- `SOURCE FACT` Server aggregation (Eq. 2): `omega = (1/M) sum_i omega_i`, + `Cbar_j = (1/M) sum_i C_ij`. +- `SOURCE FACT` Server sends back global model and global prototypes. +- `SOURCE FACT` Local objective (Eq. 3): classification loss + prototype alignment + (`||C_ij - Cbar_j||^2`) + model parameter alignment (`mu/2 * ||omega_i - omega||^2`). +- `SOURCE FACT` Inference: assign the label of the global prototype with the smallest L2 + distance to the sample embedding. + +`SOURCE FACT` The spec explicitly warns that "Weighted averaging" in Figure 1 is **not** +present in Equation (2), and adopts Equation (2) as the formal reference. + +## 3. Proposed formulation (this study) + +`SOURCE FACT` For `S_j = { i | N_ij > 0 and a valid local prototype C_ij exists }`: + +``` +Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) / ( sum_{i in S_j} N_ij^beta ) +``` + +`SOURCE FACT` Candidate values: `beta in {0.0, 0.25, 0.5, 0.75, 1.0}`. One beta is used for +all classes and clients in an experiment. + +`SOURCE FACT` Interpretations: + +- `beta = 0` gives equal weights inside the proposed `S_j`. It is a **control**, not + automatically the original baseline. +- `beta = 1` weights by class sample count; because `C_ij` is a mean embedding, the result + equals the mean of all local embeddings of class `j` included that round. +- `0 < beta < 1` gives a moderate support influence. + +`SOURCE FACT` `N_ij` is aggregation metadata only; per-sample embeddings are never sent. + +`SOURCE FACT` `beta = 0` may only be called "identical to baseline PROTEAN" if the +aggregation participant set and the missing-class handling are also identical. + +## 4. Missing-class ambiguity in the specification + +`SOURCE FACT` Equation (2) sums over `M` participants but does not state how a class +prototype that is unavailable on a client is handled. + +`SOURCE FACT` `S_j` (clients that actually contributed a valid prototype for class `j`) is +defined **for the proposed formulation**. Its conformance to the original implementation +must still be verified. + +`SOURCE FACT` If the baseline's missing-class rule differs from `S_j`, an +`equal_available` control (equal weights over `S_j`) may be used to separate the effect of +the participant rule from the effect of the weights. The spec says such a control is only +introduced if needed. + +`IMPLEMENTATION DECISION` `equal_available` is kept as a dormant, documented control mode. +It must not be reported as a result unless baseline investigation demonstrates its +necessity (see `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). + +## 5. Rare class and unseen class definitions + +`SOURCE FACT` Rare classes on client `i`: + +- the two **attack** classes with the smallest **positive** training support (`N_ij > 0`) + on that client; +- if exactly one attack class is available, that one is used; +- if no attack class is available (`N_ij = 0` for all attack classes), the client does not + enter the rare-class aggregate. + +`SOURCE FACT` Classes with `N_ij = 0` are **unseen** classes and are evaluated separately +from rare classes. Rare and unseen evaluation must not be merged. + +`SOURCE FACT` Rare-class scoring is computed per client that has rare classes, then averaged +across clients and across random seeds. + +`INFERENCE` The benign class (Normal / Benign) is never a "rare class" because the +definition restricts to attack classes. This should be kept explicit in code. + +`UNRESOLVED` Whether "attack class" excludes the benign label by name/position, and how +the class label mapping is fixed across clients and rounds, must be frozen before +experiments. See `docs/research/EXPERIMENT_PROTOCOL.md`. + +## 6. Validation, selection, test + +`SOURCE FACT` Beta is selected on a **validation** set using macro-F1 of rare classes as the +primary criterion. + +`SOURCE FACT` The test set is used only after beta selection is complete. The spec explicitly +forbids using test data for selection. + +`SOURCE FACT` The selected value is "the best beta among the tested candidates under the +specified validation procedure", **not** a claim of global optimality. + +`SOURCE FACT` After selection, overall macro-F1, rare-class recall, and per-client results +are still reported, to expose the consequences of the choice. + +`UNRESOLVED` The example PROTEAN paper uses an 80/20 train/test split with **no** +validation set. The specification adds a validation split. The exact split procedure and +fraction are left to this project. An `IMPLEMENTATION DECISION` must be recorded once fixed +(see EXPERIMENT_PROTOCOL.md). + +## 7. Datasets and non-IID settings + +`SOURCE FACT` Datasets: X-IIoTID and 5G-NIDD. + +`SOURCE FACT` Initial partition uses a Dirichlet distribution with `alpha in {0.75, 0.5, 0.25}`. + +`SOURCE FACT` Training, validation, and test data are separated. Rare classes are determined +from **training** data before beta selection. + +`SOURCE FACT` For each partition, record the total samples per class, the number of clients +that contain the class, and the inequality of `N_ij` across contributing clients. Alpha alone +is insufficient because different seeds give different realised distributions. + +## 8. Metrics + +`SOURCE FACT` Required outputs include: + +- macro-F1 of rare classes (primary selection criterion); +- rare-class recall; +- overall macro-F1; +- per-client results. + +`SOURCE FACT` Composition of the validation and test sets must be recorded so that rare-class +metrics are not interpreted without the number of evaluation samples. + +`IMPLEMENTATION DECISION` Per-class confusion counts for each evaluated split are persisted, +so precision/recall/F1 can be recomputed without re-running training. + +## 9. Direct prototype analysis + +`SOURCE FACT` Separate the **direct mathematical aggregation effect** from the +**end-to-end training effect**. + +`SOURCE FACT` The direct effect is inspected by computing candidate global prototypes from +the **same set of local prototypes at the same round**, across beta values, without +retraining. + +`SOURCE FACT` Report at least `|| Cbar_j(beta) - Cbar_j(0) ||`, and optionally normalized +aggregation weights, distance to contributing prototypes, pairwise global-prototype +displacement, and support imbalance. + +`SOURCE FACT` Direct analysis does not replace end-to-end evaluation. + +## 10. Research invariants (must hold in all experiments) + +1. Only server-side prototype aggregation changes. Architecture, model aggregation, loss + terms (`lambda`, `mu`), and the inference rule follow the verified baseline. +2. For a given `(dataset, alpha, seed)`, all aggregation variants reuse the exact same + partition, splits, client membership, class mapping, model initialisation policy, and + hyperparameters. +3. Beta comparisons are paired: a beta run must not regenerate a fresh Dirichlet partition. +4. Validation data selects beta; test data never does. +5. Rare classes are two smallest positive-support **attack** classes; unseen classes are + separate. +6. `beta = 0` inside `S_j` is reported as an equal-weighting control; it is only equated + with the original PROTEAN when the participant rule is proven identical. + +## 11. Methodological constraints and non-goals + +`SOURCE FACT` Do not introduce, without need to reproduce the frozen baseline: client +rejection, poisoning defence, adaptive losses, prototype clustering, distance-metric +experiments, or any optimisation beyond beta weighting. + +`SOURCE FACT` Learning rate, communication rounds, local epochs, batch size, architecture, +and loss parameters are held equal across comparisons. Partitions and seeds are paired. + +## 12. Unresolved questions carried forward + +1. Original PROTEAN missing-class and denominator semantics (see baseline investigation). +2. Original global-prototype initialisation for all `K` classes at round 0. +3. Value of `lambda` (the paper states `mu = 0.1` but not `lambda`). +4. Whether model-parameter aggregation is equal (`1/M`) or FedAvg-style weighted by `N_i`. +5. Exact validation split procedure/fraction (paper has none). +6. Exact random seed set and reported units (paper averages over three runs, seeds unnamed). +7. Feature preprocessing (encoding, scaling, missing-value handling) is not specified by + the paper and must be frozen here. +8. Class-label normalisation for the `Lateral _movement` typo and for any class with zero + valid samples. +9. How the prototype alignment loss term handles `N_ij = 0` classes (sum over `j = 1..K` + with no local prototype is undefined as written). + +These are tracked with evidence in `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md` and +carried into `docs/handoff/SESSION_HANDOFF.md`. diff --git a/planningawal/PROTEAN_Revisi_Lurus.md b/planningawal/PROTEAN_Revisi_Lurus.md new file mode 100755 index 0000000..f95bf80 --- /dev/null +++ b/planningawal/PROTEAN_Revisi_Lurus.md @@ -0,0 +1,118 @@ +# Optimasi Agregasi Prototype pada PROTEAN melalui Pembobotan Berbasis Dukungan Sampel Kelas untuk Deteksi Serangan Langka pada Data Non-IID + +Optimizing Prototype Aggregation in PROTEAN through Class-Sample-Support Weighting for Rare Attack Detection under Non-IID Data Distributions + +Nama: Muhamad Wildan Rizky +NIM: 20102520001 + +## 1. Dasar Pemilihan Fokus Penelitian + +PROTEAN dipilih karena dirancang untuk federated intrusion detection pada data non-IID, termasuk ketika jenis serangan tertentu jarang ditemukan atau tidak tersedia pada sebagian klien. Metode ini menggabungkan pertukaran class prototype dengan model aggregation untuk mendukung pembelajaran antarklien (Chennoufi et al., 2025). + +Penelitian ini berfokus pada prototype aggregation di server. Bagian ini dipilih karena global prototype berperan dalam dua proses: sebagai acuan prototype alignment selama training dan sebagai pembanding embedding saat inferensi. Dengan demikian, cara server menggabungkan local prototype dapat memengaruhi representasi kelas sekaligus keputusan deteksi. + +Persamaan (2) PROTEAN memberi kontribusi yang sama kepada local prototype, sedangkan jumlah sampel yang membentuknya dapat berbeda antarklien. Perbedaan ini menjadi dasar untuk menguji pembobotan berdasarkan dukungan sampel kelas. Manfaat pembobotan tersebut masih perlu dibuktikan melalui eksperimen. + +## 2. Mekanisme PROTEAN + +### Pembentukan local prototype + +Klien melatih model dengan data lokal. Untuk kelas j yang memiliki Nᵢⱼ > 0 sampel pada klien i, local prototype Cᵢⱼ merupakan rata-rata embedding sampel kelas tersebut, sebagaimana dituliskan pada Persamaan (1) PROTEAN. Dengan zᵢⱼₖ sebagai embedding sampel ke-k, rumusnya dapat ditulis sebagai: + +Cᵢⱼ = (1 / Nᵢⱼ) Σₖ zᵢⱼₖ + +### Agregasi di server + +Klien mengirim parameter model dan local prototype ke server. Persamaan (2) pada paper menuliskan agregasi keduanya sebagai rata-rata dari M peserta. Untuk global prototype kelas j, rumusnya adalah: + +C̄ⱼ = (1 / M) Σᵢ₌₁ᴹ Cᵢⱼ + +Gambar 1 menggunakan istilah “Weighted averaging”, tetapi Persamaan (2) tidak memuat bobot berdasarkan Nᵢⱼ. Karena itu, rujukan formal untuk pembahasan agregasi dalam penelitian ini adalah persamaan tersebut. + +### Pembelajaran lokal berikutnya + +Server mengirim model global dan global prototype kembali ke klien. Objective pada Persamaan (3) menggabungkan classification loss, prototype alignment, dan model parameter alignment. Ketiganya digunakan untuk melatih model lokal pada communication round berikutnya. + +### Inferensi + +Pada Bagian 3.2, paper menjelaskan bahwa sampel baru diberi label berdasarkan global prototype dengan jarak L² terkecil terhadap embedding sampel tersebut. Karena itu, perubahan posisi global prototype dapat langsung memengaruhi hasil klasifikasi. + +## 3. Baseline dan Penanganan Kelas yang Tidak Tersedia + +Baseline utama adalah reproduksi PROTEAN. Sebelum eksperimen, implementasi agregasi perlu diperiksa, terutama ketika suatu klien tidak memiliki sampel kelas j. Persamaan (2) memakai M peserta, tetapi rumus itu sendiri belum menjelaskan perlakuan terhadap prototype kelas yang tidak tersedia. + +Agregasi hanya dari klien yang memiliki kelas j tidak langsung dianggap identik dengan baseline PROTEAN. Jika aturan tersebut diperlukan dalam implementasi penelitian, perbedaannya harus dicatat dan diuji secara terpisah. Dengan demikian, pengaruh perubahan peserta agregasi tidak tercampur dengan pengaruh perubahan bobot. + +Dalam formulasi usulan, Sⱼ menyatakan kumpulan klien yang menyumbang local prototype kelas j dari sampel kelas yang benar-benar tersedia pada round tersebut. Setiap anggota Sⱼ memiliki Nᵢⱼ > 0. Definisi ini berlaku untuk usulan penelitian; kesesuaiannya dengan implementasi asli tetap harus diverifikasi. + +## 4. Masalah yang Diteliti + +Local prototype merangkum embedding dari sejumlah sampel kelas. Ketika Nᵢⱼ berbeda antarklien, equal weighting memberi kontribusi yang sama kepada ringkasan yang dibentuk dari jumlah observasi berbeda. Pembobotan berdasarkan Nᵢⱼ dapat menggeser global prototype, tetapi pergeseran tersebut belum tentu meningkatkan deteksi. + +Nᵢⱼ digunakan sebagai ukuran dukungan sampel, bukan ground-truth quality score. Prototype dari sedikit sampel belum tentu buruk, dan prototype dari banyak sampel belum tentu lebih mewakili variasi serangan yang ingin dideteksi. + +Hipotesis penelitian adalah bahwa pengaruh pembobotan bergantung pada ketimpangan dukungan sampel dan perbedaan local prototype antarklien. Eksperimen perlu menunjukkan kapan pembobotan membantu deteksi kelas langka, kapan pengaruhnya kecil, dan kapan pembobotan justru merugikan klien tertentu. + +## 5. Mekanisme Agregasi yang Diusulkan + +Usulan penelitian mengatur kekuatan pembobotan dukungan sampel melalui parameter β. Bobot klien i untuk kelas j ditentukan oleh wᵢⱼ = Nᵢⱼᵝ, dengan 0 ≤ β ≤ 1. Global prototype dihitung sebagai: + +C̄ⱼ(β) = [Σᵢ∈Sⱼ Nᵢⱼᵝ Cᵢⱼ] / [Σᵢ∈Sⱼ Nᵢⱼᵝ] + +Pada β = 0, semua klien dalam Sⱼ memiliki bobot sama. Nilai ini menjadi kontrol equal weighting untuk formulasi usulan. Nilai β = 0 hanya dapat disebut identik dengan baseline PROTEAN jika peserta agregasi dan penanganan kelas yang tidak tersedia juga sama. + +Pada β = 1, kontribusi sebanding dengan jumlah sampel kelas. Karena local prototype merupakan rata-rata embedding, hasil agregasinya setara dengan rata-rata seluruh embedding lokal kelas j yang disertakan pada round tersebut. Nilai 0 < β < 1 memberi pengaruh dukungan sampel yang lebih moderat. + +Rumus dengan β merupakan usulan penelitian. Informasi Nᵢⱼ diperlukan sebagai metadata agregasi; embedding tiap sampel tidak perlu dikirim ke server. + +## 6. Kondisi yang Memengaruhi Hasil Pembobotan + +Equal weighting memberi pengaruh yang sama kepada setiap klien penyumbang prototype. Sebaliknya, semakin besar β, semakin besar pengaruh klien dengan Nᵢⱼ yang tinggi. Pilihan ini dapat memperkuat representasi dari banyak observasi, tetapi juga dapat mengurangi pengaruh variasi serangan yang hanya ditemukan pada klien dengan sedikit sampel. + +Jika hanya satu klien yang menyumbang prototype kelas j, hasil agregasi tetap Cᵢⱼ untuk semua nilai β. Untuk local prototype yang sama, pembobotan juga tidak mengubah hasil ketika jumlah sampel seluruh klien dalam Sⱼ sama. Artinya, perubahan β tidak selalu menghasilkan global prototype yang berbeda. + +Batas tersebut berlaku pada satu langkah agregasi dengan local prototype yang tetap. Dalam training berulang, perubahan pada kelas lain masih mungkin memengaruhi model dan embedding. Oleh karena itu, efek langsung pembobotan dan perubahan kinerja setelah training perlu dianalisis secara terpisah. + +## 7. Parameter dan Sasaran Optimasi + +Parameter utama yang dicari adalah β. Pengujian awal menggunakan β ∈ {0; 0,25; 0,5; 0,75; 1}. Satu nilai β digunakan untuk seluruh kelas dan klien dalam satu eksperimen. Untuk setiap kondisi non-IID, β dipilih berdasarkan macro-F1 rare class pada validation set. Skor dihitung per klien yang memiliki rare class, lalu dirata-ratakan antarklien dan antar-random seed. + +Optimasi dalam rancangan ini dilakukan melalui pencarian pada kandidat β tersebut. Nilai yang dipilih adalah yang terbaik di antara kandidat yang diuji, bukan klaim optimum global. Test set digunakan setelah pemilihan β selesai. Macro-F1 keseluruhan, recall rare class, dan hasil per klien tetap dilaporkan untuk menilai konsekuensi pilihan tersebut. + +Learning rate, jumlah communication round, local epoch, batch size, arsitektur model, serta parameter loss dibuat sama dalam setiap perbandingan. Partisi data dan random seed juga dipasangkan agar perubahan hasil dapat ditelusuri ke mekanisme agregasi. + +## 8. Pertanyaan Penelitian + +RQ1. Bagaimana hubungan antara ketimpangan dukungan sampel kelas, perbedaan local prototype, dan kinerja deteksi kelas langka pada PROTEAN? + +RQ2. Bagaimana perubahan β memengaruhi global prototype dan kinerja deteksi kelas langka pada kondisi non-IID yang berbeda? + +RQ3. Apakah nilai β yang dipilih melalui validasi berubah pada tingkat ketimpangan yang berbeda, dan apa dampaknya terhadap kinerja keseluruhan serta hasil per klien? + +## 9. Rancangan Evaluasi + +Evaluasi menggunakan X-IIoTID dan 5G-NIDD. Mengikuti kondisi eksperimen PROTEAN, partisi awal menggunakan Dirichlet dengan α sebesar 0,75; 0,5; dan 0,25. Data training, validasi, dan pengujian dipisahkan. Kelas langka ditentukan dari data training sebelum pemilihan β. + +Paper PROTEAN mengevaluasi dua kelas serangan dengan sampel training paling sedikit pada setiap klien dan membahas kelas yang tidak terlihat saat training dalam evaluasi zero-shot. Dalam penelitian ini, rare class adalah dua kelas serangan dengan jumlah sampel training paling sedikit di antara kelas yang tersedia pada klien tersebut (Nᵢⱼ > 0). Jika hanya satu kelas serangan yang tersedia, kelas itu yang dievaluasi. Kelas serangan dengan Nᵢⱼ = 0 dianalisis sebagai unseen class. Klien tanpa sampel serangan pada training tidak masuk perhitungan rare class. Pemisahan ini merupakan definisi operasional penelitian dan perlu dicatat saat membandingkan hasil dengan paper. + +Baseline PROTEAN dibandingkan dengan formulasi usulan pada seluruh kandidat β. Jika penanganan kelas yang tidak tersedia pada baseline berbeda dari definisi Sⱼ, equal weighting pada Sⱼ digunakan sebagai kontrol tambahan. Dengan kontrol ini, perbedaan akibat peserta agregasi dapat dibedakan dari perbedaan akibat bobot. + +Untuk setiap partisi, catat total sampel per kelas, jumlah klien yang memiliki kelas tersebut, dan ketimpangan Nᵢⱼ di antara klien penyumbang prototype. Nilai α saja tidak cukup untuk menjelaskan kondisi data karena random seed yang berbeda dapat menghasilkan distribusi yang berbeda. + +Efek langsung pembobotan diperiksa dengan menghitung kandidat global prototype dari kumpulan local prototype yang sama pada round yang sama. Evaluasi training lengkap kemudian mengukur F1 dan recall rare class, macro-F1 keseluruhan, serta hasil per klien. Evaluasi diulang pada beberapa random seed dan dilaporkan sebagai rata-rata beserta variasinya. Komposisi validation set dan test set perlu dicatat agar metrik kelas langka tidak ditafsirkan tanpa melihat jumlah sampel evaluasinya. + +## 10. Kontribusi yang Ditargetkan + +Penelitian menargetkan analisis tentang kondisi ketika pembobotan dukungan sampel membantu atau merugikan prototype aggregation pada PROTEAN. Hasil yang dicari mencakup pengaruh β terhadap representasi global, kinerja kelas langka, dan klien dengan dukungan sampel kecil, serta pemilihan β berdasarkan validasi pada kondisi non-IID yang berbeda. + +Pembobotan prototype yang melibatkan jumlah sampel kelas sudah dibahas dalam FedProto (Tan et al., 2022). Karena itu, kebaruan tidak dapat ditetapkan hanya dari penggunaan Nᵢⱼ atau penambahan β. Kontribusi perlu ditunjukkan melalui perbedaan mekanisme, temuan tentang perilakunya, dan bukti eksperimen dibandingkan penelitian terkait. + +## 11. Batas Fokus Penelitian + +Modifikasi difokuskan pada prototype aggregation di server. Arsitektur embedding, model aggregation, fungsi loss beserta λ dan μ, serta aturan inferensi mengikuti baseline yang telah diverifikasi. Eksperimen mempertahankan pengaturan tersebut agar dampak pembobotan dapat diinterpretasikan dengan jelas. + +## Daftar Pustaka + +Chennoufi, S., Han, Y., Blanc, G., De Cristofaro, E., & Kiennert, C. (2025). PROTEAN: Federated Intrusion Detection in Non-IID Environments through Prototype-Based Knowledge Sharing. Proceedings of the 30th European Symposium on Research in Computer Security (ESORICS 2025). arXiv:2507.05524v1. + +Tan, Y., Long, G., Liu, L., Zhou, T., Lu, Q., Jiang, J., & Zhang, C. (2022). FedProto: Federated Prototype Learning across Heterogeneous Clients. Proceedings of the AAAI Conference on Artificial Intelligence, 36(8), 8432–8440. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..53dd0eb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "protean-research" +version = "0.0.1" +description = "Server-side prototype aggregation study on the PROTEAN federated IDS baseline" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "Proprietary" } +dependencies = [ + "pyyaml>=6.0", +] + +[project.optional-dependencies] +# Research/runtime stack. Not installed in the planning session. +research = [ + "numpy>=1.24", + "pandas>=2.0", + "torch>=2.0", +] +dev = [ + "pytest>=7.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/protean_research"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +addopts = "-q" diff --git a/scripts/.gitkeep b/scripts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/protean_research/__init__.py b/src/protean_research/__init__.py new file mode 100644 index 0000000..cf1f9d8 --- /dev/null +++ b/src/protean_research/__init__.py @@ -0,0 +1,10 @@ +"""PROTEAN research package. + +Server-side prototype aggregation study on top of the PROTEAN federated +intrusion-detection baseline (Chennoufi et al., 2025, arXiv:2507.05524). + +The authoritative research specification is +``planningawal/PROTEAN_Revisi_Lurus.md`` (immutable, read-only). +""" + +__version__ = "0.0.1" diff --git a/src/protean_research/aggregation/__init__.py b/src/protean_research/aggregation/__init__.py new file mode 100644 index 0000000..0fd4296 --- /dev/null +++ b/src/protean_research/aggregation/__init__.py @@ -0,0 +1,20 @@ +"""Server-side prototype aggregation. + +This package is the single locus of the proposed modification. Local model +training, the embedding architecture, the loss terms and the inference rule are +intentionally left untouched (see docs/research/IMPLEMENTATION_PLAN.md). +""" + +from .prototype_aggregation import ( + AggregationMode, + aggregate_prototypes, + aggregation_weights, + beta_weights, +) + +__all__ = [ + "AggregationMode", + "aggregate_prototypes", + "aggregation_weights", + "beta_weights", +] diff --git a/src/protean_research/aggregation/prototype_aggregation.py b/src/protean_research/aggregation/prototype_aggregation.py new file mode 100644 index 0000000..4b7fc37 --- /dev/null +++ b/src/protean_research/aggregation/prototype_aggregation.py @@ -0,0 +1,163 @@ +"""Pure server-side prototype aggregation. + +Formulation under study +----------------------- +For a class ``j`` with contributing-client set ``S_j = {i | N_ij > 0 and C_ij exists}``:: + + C_bar_j(beta) = sum_{i in S_j} N_ij**beta * C_ij + -------------------------------- + sum_{i in S_j} N_ij**beta + +with candidate ``beta in {0.0, 0.25, 0.5, 0.75, 1.0}``. + +The functions in this module are pure and dependency-free so they can be unit +tested in a bare interpreter. They implement only the two *unambiguous* modes: + +``equal_available`` + equal weight for every contributing client in ``S_j`` (``beta == 0`` control). + +``support_weighted`` + ``w_ij = N_ij ** beta`` normalized over ``S_j`` (the proposed mechanism). + +``original_protean`` is deliberately NOT implemented here. The published +Equation (2) writes ``(1/M) * sum_{i=1}^{M} C_ij`` but the paper does not define +what happens to ``C_ij`` when ``N_ij == 0`` (omitted / zero vector / stale value / +different denominator). Encoding any choice here would silently assume baseline +semantics that must first be verified from official code. See +``docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md``. +""" + +from __future__ import annotations + +import math +from enum import Enum +from typing import Optional, Sequence + +Vector = Sequence[float] + + +class AggregationMode(str, Enum): + """Supported prototype-aggregation modes.""" + + ORIGINAL_PROTEAN = "original_protean" + SUPPORT_WEIGHTED = "support_weighted" + EQUAL_AVAILABLE = "equal_available" + + +def _validate_beta(beta: Optional[float]) -> float: + if beta is None: + raise ValueError("beta is required") + beta = float(beta) + if math.isnan(beta) or math.isinf(beta): + raise ValueError(f"beta must be finite, got {beta!r}") + if not 0.0 <= beta <= 1.0: + raise ValueError(f"beta must be in the closed interval [0, 1], got {beta!r}") + return beta + + +def _validate_supports(supports: Optional[Sequence[float]]) -> list[float]: + if supports is None: + raise ValueError("supports (N_ij) are required") + result = [float(n) for n in supports] + if not result: + raise ValueError("supports must not be empty") + for n in result: + if math.isnan(n) or math.isinf(n) or n <= 0.0: + raise ValueError( + "supports must be finite and strictly positive: S_j contains " + f"only clients with N_ij > 0, got {n!r}" + ) + return result + + +def _validate_vectors(prototypes: Sequence[Vector]) -> int: + if not prototypes: + raise ValueError("at least one prototype is required") + if len(prototypes) == 1 and len(prototypes[0]) == 0: + raise ValueError("prototype vectors must be non-empty") + dim = len(prototypes[0]) + if dim == 0: + raise ValueError("prototype vectors must be non-empty") + for vec in prototypes: + if len(vec) != dim: + raise ValueError( + f"all prototypes must share the same dimension, got {len(vec)} vs {dim}" + ) + for value in vec: + if not math.isfinite(float(value)): + raise ValueError(f"prototype values must be finite, got {value!r}") + return dim + + +def beta_weights(supports: Sequence[float], beta: float) -> list[float]: + """Return the unnormalized weights ``N_ij ** beta``.""" + beta = _validate_beta(beta) + return [n ** beta for n in _validate_supports(supports)] + + +def aggregation_weights( + prototypes: Sequence[Vector], + mode: AggregationMode | str, + beta: Optional[float] = None, + supports: Optional[Sequence[float]] = None, +) -> list[float]: + """Return normalized aggregation weights that sum to 1. + + ``prototypes`` is used for length/dimension validation and to mirror the + required shape of the weight vector. Weights are returned in the same client + order as ``prototypes``. + """ + _validate_vectors(prototypes) + mode = AggregationMode(mode) + + if mode is AggregationMode.ORIGINAL_PROTEAN: + raise NotImplementedError( + "original_protean aggregation semantics are not yet verified. " + "Missing-class behavior (N_ij == 0) is undefined in the paper and no " + "official implementation was found. Verify the baseline first; see " + "docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md." + ) + + if mode is AggregationMode.SUPPORT_WEIGHTED: + weights = beta_weights(_validate_supports(supports), _validate_beta(beta)) + else: # EQUAL_AVAILABLE + weights = [1.0] * len(prototypes) + + total = sum(weights) + if not math.isfinite(total) or total <= 0.0: + raise ValueError(f"aggregation weights must sum to a positive value, got {total!r}") + return [w / total for w in weights] + + +def aggregate_prototypes( + prototypes: Sequence[Vector], + mode: AggregationMode | str, + beta: Optional[float] = None, + supports: Optional[Sequence[float]] = None, +) -> list[float]: + """Aggregate ``C_ij`` for one class into ``C_bar_j``. + + Parameters + ---------- + prototypes: + Local prototypes ``C_ij`` for the contributing clients in ``S_j``. + mode: + One of :class:`AggregationMode`. + beta: + Weighting exponent, required for ``support_weighted`` (and meaningful + for ``equal_available`` as a documented ``beta == 0`` equivalent). + supports: + Sample counts ``N_ij`` (> 0), required for ``support_weighted``. + """ + dim = _validate_vectors(prototypes) + if supports is not None and len(supports) != len(prototypes): + raise ValueError( + f"supports length ({len(supports)}) must match prototypes " + f"({len(prototypes)})" + ) + weights = aggregation_weights(prototypes, mode, beta=beta, supports=supports) + result = [0.0] * dim + for weight, vec in zip(weights, prototypes): + for d in range(dim): + result[d] += weight * float(vec[d]) + return result diff --git a/src/protean_research/utils/__init__.py b/src/protean_research/utils/__init__.py new file mode 100644 index 0000000..6ff8f52 --- /dev/null +++ b/src/protean_research/utils/__init__.py @@ -0,0 +1 @@ +"""Utility helpers (configuration, path resolution, source integrity).""" diff --git a/src/protean_research/utils/config.py b/src/protean_research/utils/config.py new file mode 100644 index 0000000..83f19f2 --- /dev/null +++ b/src/protean_research/utils/config.py @@ -0,0 +1,91 @@ +"""Centralized configuration loading and path resolution. + +Design rules +------------ +* Dataset paths are never hardcoded in training logic. +* Raw datasets live outside the repository and are read-only. +* Resolution precedence for a raw dataset path: + 1. environment variable named in ``env_override`` + 2. ``raw_path`` in ``configs/datasets/datasets.yaml`` (``~`` expanded) +* Generated artifacts must resolve under the configured ``generated_root``. + +PyYAML is required to load the YAML configs. It is intentionally imported lazily +so that dependency-free helpers (e.g. pure aggregation maths) can be tested in a +bare Python interpreter. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Mapping + +REPO_ROOT = Path(__file__).resolve().parents[3] + +DEFAULT_DATASETS_CONFIG = REPO_ROOT / "configs" / "datasets" / "datasets.yaml" + + +def repo_root() -> Path: + """Absolute path of the repository root.""" + return REPO_ROOT + + +def _require_yaml(): + try: + import yaml # type: ignore + except ImportError as exc: # pragma: no cover - environment dependent + raise ImportError( + "PyYAML is required to load configuration files. " + "Install project dependencies (see pyproject.toml), e.g. with " + "`uv sync` or `pip install pyyaml`." + ) from exc + return yaml + + +def load_yaml(path: str | os.PathLike[str]) -> Any: + """Load a YAML document from *path*.""" + path = Path(path) + with path.open("r", encoding="utf-8") as handle: + return _require_yaml().safe_load(handle) + + +def expand_path(value: str | os.PathLike[str], base: str | os.PathLike[str] | None = None) -> Path: + """Expand ``~`` and resolve *value* relative to *base* (default: repo root).""" + path = Path(os.path.expanduser(str(value))) + if not path.is_absolute(): + path = Path(base or REPO_ROOT) / path + return path.resolve() + + +def load_datasets_config( + path: str | os.PathLike[str] | None = None, +) -> Mapping[str, Any]: + """Load ``datasets.yaml`` (or an explicit override).""" + return load_yaml(path or DEFAULT_DATASETS_CONFIG) + + +def resolve_raw_path( + dataset_cfg: Mapping[str, Any], +) -> Path: + """Resolve the raw CSV path for one dataset entry. + + Precedence: environment override, then ``raw_path``. The path is expanded + and resolved but *not* checked for existence here so callers can give their + own diagnostics. + """ + env_name = dataset_cfg.get("env_override") + if env_name and os.environ.get(env_name): + return expand_path(os.environ[env_name]) + if not dataset_cfg.get("raw_path"): + raise KeyError("dataset config is missing 'raw_path'") + return expand_path(dataset_cfg["raw_path"]) + + +def generated_root( + datasets_cfg: Mapping[str, Any] | None = None, +) -> Path: + """Root directory for all generated artifacts inside the repository.""" + cfg = datasets_cfg or load_datasets_config() + root = expand_path(cfg.get("generated_root", "artifacts")) + root.mkdir(parents=True, exist_ok=True) + return root diff --git a/tests/test_prototype_aggregation.py b/tests/test_prototype_aggregation.py new file mode 100644 index 0000000..a9bc1e1 --- /dev/null +++ b/tests/test_prototype_aggregation.py @@ -0,0 +1,126 @@ +"""Unit tests for the server-side prototype aggregation math. + +Covers the dependency-free subset of the testing strategy in +``docs/research/IMPLEMENTATION_PLAN.md`` (items 1-7). Items requiring the +future data/federated modules (partition determinism, manifest reload, +baseline invariance, validation/test separation) are deferred. +""" + +from __future__ import annotations + +import math + +import pytest + +from protean_research.aggregation import ( + AggregationMode, + aggregate_prototypes, + aggregation_weights, + beta_weights, +) + +BETAS = [0.0, 0.25, 0.5, 0.75, 1.0] + + +def mean(vectors): + dim = len(vectors[0]) + return [sum(v[d] for v in vectors) / len(vectors) for d in range(dim)] + + +def test_beta_zero_equals_arithmetic_mean() -> None: + prototypes = [[0.0, 2.0, -1.0], [2.0, 0.0, 1.0], [4.0, 4.0, 4.0]] + supports = [1.0, 3.0, 10.0] + out = aggregate_prototypes( + prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=0.0, supports=supports + ) + assert out == pytest.approx(mean(prototypes)) + + +def test_beta_one_gives_sample_count_weighting() -> None: + prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] + supports = [1.0, 3.0, 10.0] + out = aggregate_prototypes( + prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=1.0, supports=supports + ) + total = sum(supports) + expected = [ + sum(s * v[d] for s, v in zip(supports, prototypes)) / total + for d in range(2) + ] + assert out == pytest.approx(expected) + + +def test_single_contributor_invariant_across_betas() -> None: + prototypes = [[1.0, -2.0, 3.0]] + supports = [7.0] + for beta in BETAS: + out = aggregate_prototypes( + prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=supports + ) + assert out == pytest.approx(prototypes[0]) + + +def test_equal_supports_invariant_across_betas() -> None: + prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] + supports = [5.0, 5.0, 5.0] + baseline = aggregate_prototypes( + prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=0.0, supports=supports + ) + for beta in BETAS: + out = aggregate_prototypes( + prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=supports + ) + assert out == pytest.approx(baseline) + + +def test_missing_class_not_in_s_j() -> None: + # A client with N_ij == 0 must never be passed into S_j. + with pytest.raises(ValueError): + beta_weights([1.0, 0.0], beta=0.5) + with pytest.raises(ValueError): + beta_weights([1.0, -3.0], beta=0.5) + with pytest.raises(ValueError): + aggregate_prototypes( + [[1.0], [2.0]], AggregationMode.SUPPORT_WEIGHTED, beta=0.5, supports=[1.0, 0.0] + ) + + +def test_normalized_weights_sum_to_one() -> None: + prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] + for beta in BETAS: + w = aggregation_weights( + prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=[1.0, 3.0, 10.0] + ) + assert sum(w) == pytest.approx(1.0) + w_eq = aggregation_weights(prototypes, AggregationMode.EQUAL_AVAILABLE) + assert sum(w_eq) == pytest.approx(1.0) + assert w_eq == pytest.approx([1 / 3, 1 / 3, 1 / 3]) + + +def test_valid_inputs_never_nan_or_inf() -> None: + prototypes = [[0.0, 2.0], [2.0, 0.0], [4.0, 4.0]] + for beta in BETAS: + out = aggregate_prototypes( + prototypes, AggregationMode.SUPPORT_WEIGHTED, beta=beta, supports=[1.0, 3.0, 10.0] + ) + assert all(math.isfinite(v) for v in out) + + +def test_original_protean_is_not_implemented() -> None: + with pytest.raises(NotImplementedError): + aggregate_prototypes([[1.0, 2.0]], AggregationMode.ORIGINAL_PROTEAN) + + +def test_validation_errors() -> None: + with pytest.raises(ValueError): + beta_weights([1.0], beta=1.5) + with pytest.raises(ValueError): + beta_weights([1.0], beta=None) # type: ignore[arg-type] + with pytest.raises(ValueError): + aggregate_prototypes([[1.0, 2.0], [1.0]], AggregationMode.SUPPORT_WEIGHTED, beta=0.5, supports=[1.0, 2.0]) + with pytest.raises(ValueError): + aggregate_prototypes( + [[1.0, 2.0]], AggregationMode.SUPPORT_WEIGHTED, beta=0.5, supports=[1.0, 2.0] + ) + with pytest.raises(ValueError): + aggregate_prototypes([[1.0, float("nan")]], AggregationMode.EQUAL_AVAILABLE)