From ad9aa53ba76f60671ace991fcee85f003781f196 Mon Sep 17 00:00:00 2001 From: willtanoe Date: Thu, 24 Sep 2026 23:37:26 +0000 Subject: [PATCH] feat: implement support-weighted prototype aggregation --- configs/experiments/experiment_protocol.yaml | 12 +- docs/handoff/SESSION_HANDOFF.md | 335 ++++++++---------- .../BETA_AGGREGATION_IMPLEMENTATION.md | 176 +++++++++ docs/research/EXPERIMENT_PROTOCOL.md | 28 +- scripts/analyze_prototype_effect.py | 108 ++++++ scripts/run_baseline.py | 37 +- src/protean_research/aggregation/__init__.py | 14 +- .../aggregation/prototype_aggregation.py | 159 +++++++-- src/protean_research/analysis/__init__.py | 24 ++ .../analysis/direct_prototype.py | 184 ++++++++++ src/protean_research/federated/round.py | 4 + src/protean_research/federated/server.py | 10 +- src/protean_research/models/prototypes.py | 6 +- tests/test_baseline_aggregation.py | 15 +- tests/test_prototype_effect.py | 142 ++++++++ tests/test_support_weighted_aggregation.py | 200 +++++++++++ 16 files changed, 1220 insertions(+), 234 deletions(-) create mode 100644 docs/research/BETA_AGGREGATION_IMPLEMENTATION.md create mode 100644 scripts/analyze_prototype_effect.py create mode 100644 src/protean_research/analysis/__init__.py create mode 100644 src/protean_research/analysis/direct_prototype.py create mode 100644 tests/test_prototype_effect.py create mode 100644 tests/test_support_weighted_aggregation.py diff --git a/configs/experiments/experiment_protocol.yaml b/configs/experiments/experiment_protocol.yaml index a226fa8..3344ea2 100644 --- a/configs/experiments/experiment_protocol.yaml +++ b/configs/experiments/experiment_protocol.yaml @@ -18,10 +18,16 @@ non_iid: num_clients: 10 # matches PROTEAN paper cross-silo setting aggregation: - # Baseline checkpoint: only the reproduced original-PROTEAN aggregation is - # implemented. Support-weighted beta candidates and the equal_available control - # are DEFERRED to the later beta/controls phase (see docs/research/EXPERIMENT_PROTOCOL.md). + # Both the reproduced original-PROTEAN aggregation and the proposed + # class-specific sample-support weighting are implemented. The proposed mode is + # selected with `support_weighted` plus a beta in [0, 1]; the same participant + # set S_j is used by both. See docs/research/BETA_AGGREGATION_IMPLEMENTATION.md. baseline_mode: original_protean + modes: [original_protean, support_weighted] + support_weighting: class_specific_sample_support + beta_candidates: [0.00, 0.25, 0.50, 0.75, 1.00] + beta_range: [0.0, 1.0] + selection_metric: rare_class_macro_f1 training: rounds: 10 diff --git a/docs/handoff/SESSION_HANDOFF.md b/docs/handoff/SESSION_HANDOFF.md index fc99c50..e175a8c 100644 --- a/docs/handoff/SESSION_HANDOFF.md +++ b/docs/handoff/SESSION_HANDOFF.md @@ -3,11 +3,11 @@ This is the canonical continuation point for the next agent session. It is written to be readable without prior chat history, terminal scrollback, or git history. -Last updated: 2026-09-24 (Session 3, baseline validation and reproduction verification). -Session 3 type: validation-only. No beta / support-weighted / `equal_available` code was -added. Two measured full-data baseline conditions were run; partition consistency, per- -class/per-client behaviour, the `S_j` ambiguity, round-0, the loss, inference, -reproducibility and a deterministic end-to-end regression test were all checked. +Last updated: 2026-09-24 (Session 4, support-weighted beta aggregation implementation). +Session 4 type: implementation + validation. The proposed `support_weighted` +(`w_ij = N_ij ** beta`) mode was added **without changing the frozen `original_protean` +baseline**. Only small smoke tests were run; **no beta sweep and no beta selection were +performed**. --- @@ -24,14 +24,15 @@ S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } beta in {0.0, 0.25, 0.5, 0.75, 1.0} ``` -**The repo still contains the baseline only.** The beta-weighted aggregation and the -`equal_available` control are still NOT implemented. Session 3 did not add them. +Both the baseline and the proposed mode are now implemented. The baseline remains the +reference; `beta = 0` is arithmetic-identical to it only because both use the same +documented `S_j` participant rule. ## 2. Authoritative source and immutability - `planningawal/PROTEAN_Revisi_Lurus.md` — authoritative research specification. -- `planningawal/` is immutable. Re-verified unchanged this session: - SHA-256 `9ba1d7beb94354db0c4b588199fb1e2823aec8a23be1625ec09a756507fbcb89`. +- `planningawal/` is immutable. Re-verified unchanged this session: `git status -- + planningawal` and `git diff -- planningawal` are both empty. ## 3. Dataset locations (read-only external source data) @@ -40,228 +41,204 @@ beta in {0.0, 0.25, 0.5, 0.75, 1.0} - `~/datasets/5G-NIDD/5G-NIDD.csv` — SHA-256 `fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59` (re-verified unchanged). -## 4. Session 3 objective and outcome +## 4. Session 4 objective and outcome -Objective: determine whether the frozen `original_protean` baseline is coherent, -reproducible and consistent with the best available PROTEAN evidence, and explicitly -identify unresolved reproduction assumptions. **All acceptance items were met.** +Objective: implement the proposed support-weighted prototype aggregation while preserving +the frozen `original_protean` baseline, add the deferred beta-invariant tests, add direct +prototype-effect instrumentation, run small smoke tests for beta 0 / 0.5 / 1, and update +docs. **All acceptance items were met.** No commit was made. -## 5. Full-data runs completed +## 5. Implementation summary -Both with the frozen protocol (`original_protean`, 10 clients, 10 rounds, 3 local epochs, -batch 256, lr 1e-3, mu 0.1, lambda 1.0), CPU-only (`torch 2.14.0+cpu`, CUDA unavailable). +Added a new aggregation mode and threaded it through the stack; the baseline code path is +untouched. -| condition | wall | peak RSS | accuracy | macro-F1 | weighted-F1 | rare macro-F1 | rare recall | exit | -|---|---|---|---|---|---|---|---|---| -| X-IIoTID α=0.50 seed 0 | 42:07 | 6.21 GiB | 0.8904 | 0.7077 | 0.9176 | 0.6096 | 0.8715 | 0 | -| 5G-NIDD α=0.50 seed 0 | 1:04:52 | 10.37 GiB | 0.7344 | 0.8369 | 0.7088 | 0.8720 | 0.9743 | 0 | +- `src/protean_research/aggregation/prototype_aggregation.py`: `AggregationMode.SUPPORT_ + WEIGHTED`, `BETA_CANDIDATES = (0.0, 0.25, 0.50, 0.75, 1.00)`, strict `validate_beta` + (`[0, 1]`; rejects `None`/non-numeric/non-finite/out-of-range), `normalized_support_ + weights`; `aggregate_prototypes(..., supports=None, beta=None)` and `aggregate_class_ + prototypes(..., beta=None)`. `S_j` is built once and is identical to the baseline: + `None`/`N_ij <= 0` entries dropped, no zero-vector padding. Passing `beta` to + `original_protean` raises. +- `src/protean_research/aggregation/__init__.py`: exports updated. +- `src/protean_research/models/prototypes.py`, `federated/server.py`: forward `beta`. +- `src/protean_research/federated/round.py`: `TrainingConfig.aggregation_mode` (default + `original_protean`) and `.beta` (default `None`); the round loop dispatches to the mode. +- `scripts/run_baseline.py`: new `--aggregation-mode {original_protean,support_weighted}` + and `--beta`. Default behavior is the unchanged baseline. Run name/metadata record mode + and beta (`...__original_protean` unchanged; `...__support_weighted__beta0.5` for beta). +- `src/protean_research/analysis/direct_prototype.py` (new) + + `scripts/analyze_prototype_effect.py` (new): recompute `Cbar_j(beta)` from a fixed round + snapshot of `{C_ij, N_ij}` **without retraining**; emit `||Cbar_j(beta) - Cbar_j(0)||`, + support stats, norms, and normalised client weights. -Both re-used their frozen manifest (`manifest_reload_ok = true`), completed all 10 rounds -with all 10 clients active and no failures, and produced 9/9 global prototypes. Artifact -dirs: `results/xiiotid__alpha0.5__seed0__original_protean/`, -`results/5gnidd__alpha0.5__seed0__original_protean/` (ignored by git). Full detail: -`docs/baseline/BASELINE_VALIDATION.md`. +`equal_available` was **not** added (the baseline already is equal-over-`S_j`, so it equals +`beta = 0`). -Resource sizing for the future grid: ~42 min and ~6 GiB for X-IIoTID, ~65 min and ~10 GiB -for 5G-NIDD, per `(dataset, alpha, seed)` condition. A full grid -`2 datasets × 3 alphas × 3 seeds = 18` conditions is therefore on the order of -15–20 CPU-hours plus beta multiplicity; measure before launching anything large. +## 6. Tests and baseline regression status -## 6. Tests added / test status +``` +.venv/bin/python -m pytest -> 75 passed +``` -- New file `tests/test_baseline_validation.py` (12 tests): round-0 handling (spy confirms - round 0 receives no global prototypes and round 1 does), independent replication of the - documented loss formula (`atol=1e-5`), prototype-alignment skip for classes without a - global prototype, frozen-coefficient guard (`lambda=1.0`, `mu=0.1`, rounds=10, epochs=3, - batch=256, lr=1e-3), equal-weight model averaging, non-contiguous class-id inference, - brute-force squared-L2 cross-check, empty-prototype errors, local prototype class - mapping, and a deterministic end-to-end regression test. -- Total suite: `.venv/bin/python -m pytest -q` -> **46 passed** (34 pre-existing + 12 new). -- Note: `BASELINE_IMPLEMENTATION.md` previously said "39 tests"; the accurate Session-2 - count was 34. Corrected this session. +Session-3 suite was 46 tests. Added `tests/test_support_weighted_aggregation.py` (21) and +`tests/test_prototype_effect.py` (7); one baseline-aggregation test was replaced by two +explicit tests (support_weighted now available; `beta` rejected for `original_protean`). -## 7. `S_j` stress-test findings (central methodological result) +Covered beta invariants: `beta=0` arithmetic mean; `beta=1` support-weighted mean; +hand-computed intermediate-beta fixture; single-contributor invariance; equal-support +invariance; missing-class exclusion; normalised weights sum to 1; invalid/non-numeric +beta rejection; supports strictly positive and aligned. **Baseline regression is green**: +`test_end_to_end_baseline_is_deterministic_and_learns_separable_fixture` and all baseline +arithmetic/validation tests pass unchanged. -- `|S_j| < M` occurs in **53/162 class-condition cells (32.7 %)** across **15/18** - `(dataset, alpha, seed)` conditions (`artifacts/analysis/sj_participant_rule_analysis.csv`). -- Affected classes include high-frequency ones and even the benign class in two conditions; - the effect is strongest at `alpha = 0.25`. -- On the fixed X-IIoTID round-0 snapshot, the hypothetical literal `1/M` reading with - missing prototypes replaced by the zero vector is displaced by **~10–20 % of the - prototype norm** for the three affected classes (Exfiltration 0.724, Exploitation 1.131, - Tampering 0.510). Hypothetical-only, never used for training: - `artifacts/analysis/sj_vs_literal_1m_hypothetical_xiiotid__alpha0.5__seed0__original_protean.csv`, - reproducible via `scripts/validate_sj_participant_rule.py`. -- Conclusion: the ambiguity is **material, not rare**, but **cannot be resolved** without - original code (none exists). The current `S_j` equal-weight rule remains a documented - REPRODUCTION ASSUMPTION, defensible from the thesis sentence that prototypes are shared - only for locally observed classes. It is **not** verified original behaviour. The - `equal_available` control is unnecessary: the baseline already *is* equal-over-`S_j`, so - it is arithmetically identical to the proposed `beta = 0` rule. +## 7. Smoke results (NOT scientific results) -## 8. Round-0, loss and inference validation +Condition: `xiiotid`, `alpha=0.5`, `seed=0`, `--max-rows 3000 --sample-from 60000`, +4 clients, 2 rounds, 1 local epoch (ignored dirs only). -- **Round 0**: no global prototypes exist; the prototype-alignment term is skipped; the - first global prototypes come from round-0 aggregation and are used from round 1. - Provenance INFERRED / PROJECT DECISION. Covered by a focused test. -- **Loss**: `nll_loss(mean) + lambda * mean_matched ||phi(x)-Cbar_y||^2 + (mu/2)*sum||w-w_global||^2`. - Prototype term is per-sample and only for classes with a global prototype; missing - prototypes add no term and do not enter the normaliser. Coefficients `lambda=1.0` - (PROJECT DECISION), `mu=0.1` (PAPER). Independently replicated to `atol=1e-5`. -- **Inference**: nearest global prototype by squared L2 (expanded norm), returns true - class ids, restricts to classes with a global prototype, raises on empty prototype sets. - Cross-checked against brute force. +| mode / beta | test accuracy | test macro-F1 | test rare macro-F1 | test rare recall | +|---|---|---|---|---| +| `original_protean` | 0.5167 | 0.2998 | 0.3140 | 0.4572 | +| `support_weighted` 0.0 | 0.5167 | 0.2998 | 0.3140 | 0.4572 | +| `support_weighted` 0.5 | 0.2333 | 0.1397 | 0.1797 | 0.3193 | +| `support_weighted` 1.0 | 0.0383 | 0.0252 | 0.0324 | 0.1995 | -## 9. Reproducibility findings +Baseline-preservation evidence: the `original_protean` and `support_weighted beta=0` smoke +runs produced **byte-identical** result artifacts (only `metadata.json` differed in +mode/beta/run_name). The round-0 snapshots for `beta = 0, 0.5, 1` were **byte-identical**, +confirming the paired comparison (same partition/split/init/round-0 training). The +beta=0.5/1 metrics are toy-scale and must not be interpreted as method quality. -The same smoke condition was run twice into separate output roots. All result artifacts -(`history.json`, `test_global_metrics.json`, `test_confusion_matrix.json`, -`test_per_class.csv`, `per_client.csv`, `train_support_Nij.csv`, `characterization.csv`, -`rare_unseen.csv`, prototype snapshot) were **byte-identical**; `manifest_content_hash` and -`preprocessing_hash` were identical. Only differences were the run-specific manifest -`created_utc` timestamp and the explicit `manifest_path`. The deterministic regression test -also produced exactly equal parameters/prototypes/history across two in-process runs. -No nondeterminism source was observed on this CPU backend; tolerance clause for future -accelerated backends is `atol=1e-5 / rtol=1e-4`. +Direct-effect analysis was exercised on the `beta=0` smoke snapshot (9 classes, +max displacement 7.65 at beta=1; relative displacement up to 1.51 for the most +support-imbalanced class), written under the ignored smoke directory +`artifacts/processed/smoke/analysis/`. -## 10. Partition consistency (X-IIoTID α=0.50 seed 0) +## 8. Methodological decisions / assumptions carried forward -All PASS: manifest reproduces (split hash + partition hash + `N_ij`); splits disjoint and -cover the cleaned dataset; each train sample assigned exactly once with no cross-client -duplication; `N_ij` matches actual assigned samples; client totals sum to `num_train` -(569,909) and class totals match train label counts. +1. **`S_j` participant/denominator rule** (`S_j = {i | N_ij > 0 and valid C_ij}`, equal + weights in the baseline) is a **REPRODUCTION ASSUMPTION**, not verified original + behaviour. It is material in 53/162 class-condition cells (32.7%), 15/18 conditions + (`docs/baseline/BASELINE_VALIDATION.md` §D). `beta = 0` equality with the baseline is + conditional on this shared rule. Carry this caveat in all reporting. +2. `N_ij` is **class-specific sample support**, not prototype quality. +3. `beta` is one value for all classes/clients; candidates are frozen at + `{0, 0.25, 0.5, 0.75, 1}`. +4. `lambda = 1.0`, optimiser (Adam), lr (1e-3), batch (256) remain PROJECT DECISIONS. + +## 9. Unresolved questions / blockers + +1. `S_j` reproduction ambiguity (above) — unresolved, no official PROTEAN code exists. +2. Beta selection has **not** been run; no validation/test separation result exists yet. +3. Smoke-scale metrics are unstable and not meaningful; a full paired condition is needed. +4. Session-2 manifests store `preprocessing_hash = null` / `config_hash = null`; live + preprocessing hash is recorded in run metadata only. + +## 10. Git status + +No commit or push was made. Working tree has 9 modified tracked files and 5 untracked +paths from this session (listed by `git status`). `planningawal/` and both raw datasets +are unchanged. `results/`, `artifacts/processed/` remain ignored. `artifacts/analysis/` is +tracked (no new files were written there this session; smoke analysis went to +`artifacts/processed/smoke/analysis/`). ## 11. Files created / modified this session Created: -- `docs/baseline/BASELINE_VALIDATION.md` (new authoritative validation record). -- `tests/test_baseline_validation.py` (12 tests). -- `scripts/validate_sj_participant_rule.py` (read-only analysis, no experiment mode). -- `artifacts/analysis/sj_participant_rule_analysis.csv` (all 18 conditions, |S_j| analysis). -- `artifacts/analysis/sj_vs_literal_1m_hypothetical_xiiotid__alpha0.5__seed0__original_protean.csv`. -- `artifacts/analysis/xiiotid__alpha0.5__seed0__original_protean__prototype_snapshot.json`. -- `artifacts/analysis/5gnidd__alpha0.5__seed0__original_protean__prototype_snapshot.json`. +- `src/protean_research/analysis/__init__.py` +- `src/protean_research/analysis/direct_prototype.py` +- `scripts/analyze_prototype_effect.py` +- `tests/test_support_weighted_aggregation.py` +- `tests/test_prototype_effect.py` +- `docs/research/BETA_AGGREGATION_IMPLEMENTATION.md` Modified: -- `docs/baseline/BASELINE_IMPLEMENTATION.md` (validation link, corrected test count, - updated remaining-assumptions §14). -- `docs/handoff/SESSION_HANDOFF.md` (this file). +- `src/protean_research/aggregation/prototype_aggregation.py` +- `src/protean_research/aggregation/__init__.py` +- `src/protean_research/models/prototypes.py` +- `src/protean_research/federated/server.py` +- `src/protean_research/federated/round.py` +- `scripts/run_baseline.py` +- `tests/test_baseline_aggregation.py` +- `configs/experiments/experiment_protocol.yaml` +- `docs/research/EXPERIMENT_PROTOCOL.md` +- `docs/handoff/SESSION_HANDOFF.md` (this file) -Ignored/generated (not tracked): `results/xiiotid__alpha0.5__seed0__original_protean/`, -`results/5gnidd__alpha0.5__seed0__original_protean/`, -`artifacts/processed/*` (preprocessor state). +## 12. Exact next task -`planningawal/` and both raw datasets were not modified. +**Recommended: run ONE paired full `(dataset, alpha, seed)` condition across +`beta in {0, 0.25, 0.5, 0.75, 1}` under the validation-based selection protocol, then +inspect.** Do not launch the full grid. -## 12. Git status +Concretely: -No commit was requested or made. Session 2 baseline commit `0aa8e2b` and the docs commit -`374651a` are on `main` at `origin/main`. Working tree has one modified tracked file -(`docs/baseline/BASELINE_IMPLEMENTATION.md`) and the new untracked files listed above. -`results/`, `artifacts/processed/`, `.venv/`, `__pycache__/` remain ignored. No secrets, -no raw datasets. `AGENTS.md` is tracked (commit `374651a`). +1. Reuse the frozen manifest for one condition (e.g. X-IIoTID `alpha=0.5`, seed 0) and the + frozen training settings (10 clients, 10 rounds, 3 local epochs, batch 256, lr 1e-3, + mu 0.1, lambda 1.0). Measure the single-run cost first (Session 3 measured ~42 min / + 6.2 GiB on CPU for one baseline run; beta adds one run per candidate). +2. Train each beta with `run_baseline.py --aggregation-mode support_weighted --beta ...` + and record validation rare-class macro-F1; select beta on validation only. +3. Evaluate the selected beta on test, and report all candidates transparently. +4. Run `scripts/analyze_prototype_effect.py` on the round-0 snapshot for the direct + aggregation effect, clearly separated from end-to-end results. +5. Report the `S_j` reproduction caveat alongside every between-beta claim. -## 13. Unresolved assumptions / blockers +Do **not** add `equal_available`. Do **not** run the alpha x seed x dataset x beta grid. -1. **`S_j` participant/denominator rule** is a reproduction assumption, quantified but - unresolved (see §7). All future between-method claims must carry this caveat. -2. `lambda = 1.0`, optimiser (Adam), lr (1e-3), batch size (256) are PROJECT DECISIONS. -3. Prototype-alignment form (per-sample) and round-0 initialisation are INFERRED. -4. Model aggregation is equal `1/M` (PAPER); a FedAvg-style weighted wording in the thesis - intro conflicts and is not implemented (documented conflict). -5. Per-client evaluation partitioning is a PROJECT DECISION. -6. Session-2 manifests store `preprocessing_hash = null` / `config_hash = null`; they do - not bind the preprocessing recipe. Split/partition hashes and `N_ij` reproduce exactly, - and run metadata records the live preprocessing hash. Binding the hash into future - manifests would close this provenance gap. -7. Only one full-data condition per dataset has been validated; no reproduction success is - claimed from two runs. - -## 14. Exact next task - -**Recommended: (B) freeze the validated baseline and prepare the paired beta -aggregation implementation — not a full grid.** - -Justification from evidence: - -- Implementation correctness, determinism, partition consistency and the full-data paths - are validated on both datasets (Sessions 2–3). -- Option (A), "resolve remaining baseline reproduction ambiguity", is **not achievable - with available evidence**: no official PROTEAN code exists and the paper is internally - inconsistent. The ambiguity has been quantified; further "resolution" would only be - another documented decision. -- Therefore freeze the baseline definition *including* the documented `S_j` equal-weight - assumption and its quantified materiality, then implement the proposed beta weighting - as a new aggregation mode **without altering the baseline** and run a **single paired - `(dataset, alpha, seed)` condition first** (e.g. X-IIoTID α=0.5 seed 0, reusing the - frozen manifest) across `beta in {0, 0.25, 0.5, 0.75, 1}` before any grid. - -Next session must: - -1. Add `support_weighted` (`w_ij = N_ij^beta`) as a new mode only; keep - `original_protean` byte-identical and add the deferred beta-invariant tests - (`beta=0` == equal mean over `S_j`; `beta=1` == support weighting; single-contributor - and equal-support invariance; normalised weights sum to 1). -2. Do **not** add an executable `equal_available` mode (baseline already is equal-over-`S_j`). -3. Keep validation/test separation: beta selection on validation rare-class macro-F1, - test only after selection. -4. Run one paired condition end to end and inspect; measure before any grid. -5. Report the `S_j` reproduction assumption explicitly alongside any beta result. - -Do **not** launch the full grid until the single paired condition is verified. - -## 15. Important commands +## 13. Important commands ```bash -# Tests (46 expected) -.venv/bin/python -m pytest -q +# Tests (75 expected) +.venv/bin/python -m pytest + +# Smoke: baseline and beta candidates (ignored output dirs) +.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ + --max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1 +.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ + --max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1 \ + --aggregation-mode support_weighted --beta 0.5 + +# Direct prototype-effect analysis (no retraining) +.venv/bin/python scripts/analyze_prototype_effect.py \ + --snapshot artifacts/processed/smoke/analysis/\ +xiiotid__alpha0.5__seed0__support_weighted__beta0__prototype_snapshot.json \ + --output-dir artifacts/processed/smoke/analysis # Full-data baseline for one condition (measure first; ~40-65 min, CPU) .venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ --clients 10 --rounds 10 --local-epochs 3 -.venv/bin/python scripts/run_baseline.py --dataset 5gnidd --alpha 0.5 --seed 0 \ - --clients 10 --rounds 10 --local-epochs 3 - -# Smoke (writes only to ignored dirs) -.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ - --max-rows 3000 --sample-from 60000 --clients 4 --rounds 2 --local-epochs 1 - -# Read-only S_j participant-rule validation (writes analysis artifacts only) -.venv/bin/python scripts/validate_sj_participant_rule.py \ - --snapshot artifacts/analysis/xiiotid__alpha0.5__seed0__original_protean__prototype_snapshot.json - -# Regenerate / verify all partition manifests (full data, ~30 s) -.venv/bin/python scripts/generate_partitions.py ``` -## 16. Recommended next-session prompt +## 14. Recommended next-session prompt ``` -You are continuing the perdalam-protean research repository (Session 4). +You are continuing the perdalam-protean research repository (Session 5). FIRST, read completely: 1. docs/handoff/SESSION_HANDOFF.md - 2. docs/baseline/BASELINE_VALIDATION.md - 3. docs/baseline/BASELINE_IMPLEMENTATION.md + 2. docs/research/BETA_AGGREGATION_IMPLEMENTATION.md + 3. docs/baseline/BASELINE_VALIDATION.md 4. docs/research/EXPERIMENT_PROTOCOL.md Critical rules: - planningawal/ is immutable. Do not edit/create/move/rename/delete anything inside it. - ~/datasets/X-IIoTID/X-IIoTID-dataset.csv and ~/datasets/5G-NIDD/5G-NIDD.csv are read-only external source data. Do not modify or copy them into the repo. - - The baseline is validated but the S_j participant rule is a REPRODUCTION ASSUMPTION - (material in 32.7% of class-condition cells). Carry this caveat in all reporting. + - The S_j participant rule is a REPRODUCTION ASSUMPTION (material in 32.7% of + class-condition cells). Carry this caveat in all reporting. - Do NOT add an executable equal_available mode. - Do NOT launch the full alpha x seed x dataset x beta grid. -Task (Section 14 of the handoff): freeze the validated baseline and add the proposed -support_weighted aggregation (w_ij = N_ij^beta) as a NEW mode without changing -original_protean. Add the deferred beta-invariant tests. Then run ONE paired -(dataset, alpha, seed) condition across beta in {0, 0.25, 0.5, 0.75, 1}, select beta on -validation rare-class macro-F1, and evaluate the selected beta on test. Measure runtime -before scaling. Update docs and docs/handoff/SESSION_HANDOFF.md. +Task: the support_weighted mode (w_ij = N_ij^beta) is implemented and unit/smoke +validated; original_protean is unchanged. Run ONE paired full (dataset, alpha, seed) +condition across beta in {0, 0.25, 0.5, 0.75, 1}, reusing the frozen manifest and frozen +training settings. Select beta on validation rare-class macro-F1 only, then evaluate the +selected beta on test and report all candidates. Add the direct prototype-effect analysis +from the round-0 snapshot. Measure runtime before scaling. Update docs and +docs/handoff/SESSION_HANDOFF.md. Use .venv/bin/python for all Python commands. Do not commit or push unless asked. ``` diff --git a/docs/research/BETA_AGGREGATION_IMPLEMENTATION.md b/docs/research/BETA_AGGREGATION_IMPLEMENTATION.md new file mode 100644 index 0000000..e5834be --- /dev/null +++ b/docs/research/BETA_AGGREGATION_IMPLEMENTATION.md @@ -0,0 +1,176 @@ +# Beta-Weighted Prototype Aggregation — Implementation + +Status: **IMPLEMENTED and unit/smoke-validated** (Session 4, 2026-09-24). The +proposed class-specific sample-support weighting is now available as an +independent aggregation mode. The frozen `original_protean` baseline path is +unchanged. **No scientific beta sweep was run**; only small smoke tests. + +Companion documents: + +- `docs/baseline/BASELINE_IMPLEMENTATION.md` — baseline semantics and provenance. +- `docs/baseline/BASELINE_VALIDATION.md` — Session-3 baseline validation evidence. +- `docs/research/EXPERIMENT_PROTOCOL.md` — frozen protocol. +- `docs/handoff/SESSION_HANDOFF.md` — continuation point. + +--- + +## 1. Scope + +The only change relative to the validated baseline is the **server-side prototype +aggregation**. The data pipeline, partitions, splits, model, local loss, model +averaging (`1/M`) and inference are reused unchanged. The proposed rule is: + +``` +Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) / ( sum_{i in S_j} N_ij^beta ) +S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } +beta ∈ {0.00, 0.25, 0.50, 0.75, 1.00}, 0 <= beta <= 1 +``` + +`N_ij` is **class-specific sample support**, not a quality score. Larger `N_ij` +is not assumed to mean a better prototype (AGENTS.md §9). + +## 2. Code changes + +| File | Change | +|---|---| +| `src/protean_research/aggregation/prototype_aggregation.py` | Added `AggregationMode.SUPPORT_WEIGHTED`, `validate_beta`, `normalized_support_weights`; extended `aggregate_prototypes` and `aggregate_class_prototypes` with optional `supports`/`beta`. Baseline branch kept arithmetic-identical. | +| `src/protean_research/aggregation/__init__.py` | Exports the new mode, helpers and `BETA_CANDIDATES`. | +| `src/protean_research/models/prototypes.py` | `collect_global_prototypes` forwards `beta`. | +| `src/protean_research/federated/server.py` | `aggregate_prototypes` forwards `mode`/`beta`. | +| `src/protean_research/federated/round.py` | `TrainingConfig.aggregation_mode` + `.beta`; the round loop dispatches to the active mode. | +| `src/protean_research/analysis/direct_prototype.py` | New direct prototype-effect instrumentation (fixed `{C_ij, N_ij}`). | +| `scripts/analyze_prototype_effect.py` | New CLI: snapshot → effect CSV + weights CSV, no retraining. | +| `scripts/run_baseline.py` | New `--aggregation-mode`/`--beta`; run name and metadata now record mode + beta. Default remains `original_protean`. | + +### 2.1 Interface + +```python +aggregate_prototypes(prototypes, mode="original_protean", supports=None, beta=None) +aggregate_class_prototypes(contributions, mode="original_protean", beta=None) +normalized_support_weights(supports, beta) # weights summing to 1 +validate_beta(beta) # strict [0, 1] validation +``` + +- `original_protean` computes the equal mean over `S_j`. Passing `beta` with it + raises `ValueError` (no silent mixing). +- `support_weighted` requires strictly positive, aligned `supports` and a + `beta ∈ [0, 1]`; `None`, non-numeric, non-finite and out-of-range betas raise + `ValueError`. +- `S_j` is built once, identically, for both modes: entries with `proto is None` + or `N_ij <= 0` are dropped; a class with no valid contributor is omitted. No + zero-vector padding or prototype fabrication. + +## 3. Beta semantics and invariants + +| Property | Expected | Test | +|---|---|---| +| `beta = 0` | equal (arithmetic) mean over `S_j` | `test_beta0_is_equal_arithmetic_mean_over_s_j`, `test_beta0_matches_original_protean_on_identical_s_j` | +| `beta = 1` | support-weighted mean | `test_beta1_is_support_weighted_mean` | +| intermediate beta | closed-form weighted mean | `test_intermediate_beta_fixture`, `test_intermediate_beta_matches_closed_form` | +| single contributor | beta-invariant | `test_single_contributor_is_invariant_to_beta` | +| equal support | beta-invariant | `test_equal_support_is_invariant_to_beta` | +| missing class | excluded from `S_j`, never fabricated | `test_missing_class_is_excluded_from_s_j`, `test_class_with_only_missing_contributors_is_omitted` | +| weights | normalised, sum to 1, positive | `test_normalized_weights_sum_to_one`, `test_beta0_weights_are_uniform` | +| invalid beta | rejected | `test_invalid_beta_is_rejected`, `test_none_and_non_numeric_beta_are_rejected` | + +`beta = 0` is **not** labelled original PROTEAN. It is arithmetically identical +to the baseline *only because both use the same documented `S_j` participant +rule*; the `S_j` rule itself remains a reproduction assumption (§5). + +## 4. Direct prototype-effect instrumentation + +`analysis/direct_prototype.py` reads a round snapshot of fixed `{C_ij, N_ij}` +(produced by the federated loop at round 0) and recomputes `Cbar_j(beta)` for the +candidate betas **without retraining**. It reuses the production aggregation, so +only the training feedback is removed, not the aggregation arithmetic. + +Outputs (via `scripts/analyze_prototype_effect.py`): + +- `*__prototype_effect.csv`: per `(class, beta)` — `n_contributors`, + `support_min/max/mean`, `prototype_norm`, `reference_norm_beta0`, + `displacement_from_beta0 = ||Cbar_j(beta) - Cbar_j(0)||`, + `relative_displacement`. +- `*__prototype_weights.csv`: per `(class, beta, client)` normalised weights. + +Displacement is a *direct aggregation* measure only; it is **not** interpreted as +improved or degraded detection (AGENTS.md §17). + +## 5. Validation performed + +### 5.1 Test suite + +``` +.venv/bin/python -m pytest -> 75 passed +``` + +Session-3 suite was 46 tests. Added: `tests/test_support_weighted_aggregation.py` +(21), `tests/test_prototype_effect.py` (7), and one extra baseline-aggregation +test (the old "support_weighted not available" guard was replaced by explicit +availability/validation guards). The deterministic end-to-end baseline regression +(`test_end_to_end_baseline_is_deterministic_and_learns_separable_fixture`) and all +baseline arithmetic tests remain green. + +### 5.2 Baseline preservation (integration evidence) + +Smoke condition: `xiiotid`, `alpha=0.5`, `seed=0`, `max-rows=3000`, +`sample-from=60000`, 4 clients, 2 rounds, 1 local epoch. + +- `original_protean` and `support_weighted beta=0` produced **byte-identical** + result artifacts (`history.json`, `test_global_metrics.json`, confusion matrix, + per-class/per-client CSVs, `N_ij`, characterization, rare/unseen, round-0 + snapshot). Only `metadata.json` differed (`aggregation_mode`, `beta`, + `run_name`). +- The three round-0 snapshots (`beta = 0, 0.5, 1`) were **byte-identical**, + confirming that the paired comparison reuses the same partition, split, model + initialisation and round-0 training; beta only changes server aggregation. + +### 5.3 Smoke runs (NOT scientific results) + +| mode / beta | test accuracy | test macro-F1 | test rare macro-F1 | test rare recall | +|---|---|---|---|---| +| `original_protean` | 0.5167 | 0.2998 | 0.3140 | 0.4572 | +| `support_weighted` 0.0 | 0.5167 | 0.2998 | 0.3140 | 0.4572 | +| `support_weighted` 0.5 | 0.2333 | 0.1397 | 0.1797 | 0.3193 | +| `support_weighted` 1.0 | 0.0383 | 0.0252 | 0.0324 | 0.1995 | + +These are 3,000-row toy smoke runs with a random 4-client partition and a +stratified 70/10/20 split; the split's class support at this scale is tiny and the +metrics are unstable. They demonstrate that the beta path runs end to end and +that beta **does** change training feedback (global prototypes enter the local +loss from round 1). They must not be read as evidence about method quality. A +full-scale paired validation/test protocol is required (§7). + +Direct-effect instrumentation was exercised on the `beta=0` smoke snapshot +(9 classes, max `||Cbar_j(1) - Cbar_j(0)|| = 7.65`, relative displacement up to +1.51 for the most support-imbalanced class), written under the ignored smoke +directory `artifacts/processed/smoke/analysis/`. + +## 6. Provenance and assumptions + +- `support_weighted` arithmetic (`N_ij^beta`, normalised over `S_j`) — **SPEC / + PROJECT IMPLEMENTATION**; the research specification defines the formula. +- `S_j = {i | N_ij > 0 and valid C_ij}` for **both** modes — **REPRODUCTION + ASSUMPTION**, not verified original PROTEAN behaviour, carried from the + baseline. It is **material, not rare**: `|S_j| < M` in **53/162 class-condition + cells (32.7%)** across **15/18** `(dataset, alpha, seed)` conditions + (`docs/baseline/BASELINE_VALIDATION.md` §D). It is not source-confirmed. +- `beta = 0` equality with the baseline is **conditional on that shared `S_j` + rule**; it is not an independent claim that `beta = 0` reproduces original + PROTEAN. +- No `equal_available` executable mode was added (the baseline already *is* + equal-over-`S_j`, so it is arithmetically identical to `beta = 0`). +- `N_ij` is treated strictly as sample support, not prototype quality. + +## 7. Not done in this session + +- No validation-based beta selection, no test evaluation of a selected beta. +- No full or partial `(dataset, alpha, seed, beta)` sweep. +- No `equal_available` mode, no client rejection, no new distance/metric. + +## 8. Exact next step + +Run **one** paired full condition (e.g. X-IIoTID `alpha=0.5`, seed 0, reusing the +frozen manifest and frozen training settings) across +`beta ∈ {0, 0.25, 0.5, 0.75, 1}` under the validation/test separation of +`EXPERIMENT_PROTOCOL.md` §8, then inspect per-client/per-class artifacts before +scaling. Measure runtime before any grid. diff --git a/docs/research/EXPERIMENT_PROTOCOL.md b/docs/research/EXPERIMENT_PROTOCOL.md index 830fd6d..fb11a62 100644 --- a/docs/research/EXPERIMENT_PROTOCOL.md +++ b/docs/research/EXPERIMENT_PROTOCOL.md @@ -70,9 +70,9 @@ A beta run MUST NOT derive a new partition; it loads the manifest for `(dataset, ## 5. Aggregation modes -**Baseline checkpoint: only `original_protean` is implemented.** The proposed -support-weighted aggregation (`support_weighted`) and the `equal_available` control are -**DEFERRED** to the later beta/controls phase and are not present in executable code. +**Both `original_protean` and the proposed `support_weighted` are implemented.** The +`equal_available` control is **not** implemented and is unnecessary (the baseline is already +equal-over-`S_j`). See `docs/research/BETA_AGGREGATION_IMPLEMENTATION.md`. - `original_protean` — reproduced baseline. **REPRODUCTION ASSUMPTION (project decision)**: because Eq. (2)/(5.5) is internally inconsistent about missing classes, the baseline @@ -80,12 +80,14 @@ support-weighted aggregation (`support_weighted`) and the `equal_available` cont **equal weights `1/|S_j|`**, following the thesis statement that prototypes are shared only for locally observed classes. This is not verified against code and must be validated in the baseline-verification phase. -- `support_weighted` (deferred) — `w_ij = N_ij^beta`, normalised over `S_j`. -- `equal_available` (deferred) — equal weights over `S_j`; only introduced later if the - baseline participant rule is shown to differ from `S_j`. +- `support_weighted` — `w_ij = N_ij^beta`, normalised over the **same** `S_j` as the + baseline. `beta` must lie in `[0, 1]`; invalid or missing beta is rejected explicitly. +- `equal_available` — deliberately not implemented; arithmetically identical to + `support_weighted` with `beta = 0` given the shared `S_j`. -FROZEN candidates for the later phase: `beta ∈ {0.00, 0.25, 0.50, 0.75, 1.00}`, one beta for -all classes and clients. They are **not** part of the baseline checkpoint. +FROZEN candidates for the beta phase: `beta ∈ {0.00, 0.25, 0.50, 0.75, 1.00}`, one beta for +all classes and clients. Implementation status and smoke validation: +`docs/research/BETA_AGGREGATION_IMPLEMENTATION.md`. ## 6. Partition persistence @@ -157,6 +159,10 @@ For every manifest, record at least: `N_ij`, class total, number of clients cont Separate the direct mathematical aggregation effect from the end-to-end training effect. +**Implemented** in `src/protean_research/analysis/direct_prototype.py` and exposed through +`scripts/analyze_prototype_effect.py`; it reuses the production aggregation on a fixed +snapshot and requires no retraining. + - Persist at least one round's fixed `C_ij` and `N_ij` snapshot. - Reusing those snapshots, compute `Cbar_j(beta)` for several beta without retraining. - Report `|| Cbar_j(beta) - Cbar_j(0) ||`; optionally normalised aggregation weights, @@ -239,4 +245,8 @@ No statistic from validation or test data ever enters preprocessing. | Model aggregation | equal `1/M` | paper Eq. 2/5.5 | | Prototype computation | end-of-round mean embedding | paper Eq. 1 | -Until the baseline is verified, only smoke tests are permitted; no sweep may be launched. +The baseline is validated and frozen (`docs/baseline/BASELINE_VALIDATION.md`), and the +proposed `support_weighted` mode is implemented (`docs/research/BETA_AGGREGATION_IMPLEMENTATION.md`). +Only smoke tests have been run for beta. Before any grid, one paired full +`(dataset, alpha, seed)` condition across the beta candidates must be run and inspected +(validation-based selection, then test), and runtime measured. diff --git a/scripts/analyze_prototype_effect.py b/scripts/analyze_prototype_effect.py new file mode 100644 index 0000000..009839d --- /dev/null +++ b/scripts/analyze_prototype_effect.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +"""Direct prototype-effect analysis from a fixed ``{C_ij, N_ij}`` snapshot. + +Reads a round snapshot written by ``run_baseline`` (or ``run_federated``) and +recomputes ``Cbar_j(beta)`` for the candidate betas **without retraining**, +reporting per-class ``|| Cbar_j(beta) - Cbar_j(0) ||`` and the normalised client +weights. This measures the direct aggregation effect only; it is not an +end-to-end detection result. + +Usage:: + + python scripts/analyze_prototype_effect.py \ + --snapshot artifacts/analysis/xiiotid__alpha0.5__seed0__original_protean__prototype_snapshot.json +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from protean_research.aggregation import BETA_CANDIDATES # noqa: E402 +from protean_research.analysis.direct_prototype import ( # noqa: E402 + client_weight_rows, + prototype_effect_rows, +) + + +def parse_args(argv=None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--snapshot", required=True) + parser.add_argument( + "--betas", + type=float, + nargs="+", + default=list(BETA_CANDIDATES), + help="beta candidates to evaluate (default: frozen candidate set).", + ) + parser.add_argument( + "--class-names", + default=None, + help="optional comma-separated class names indexed by class id.", + ) + parser.add_argument("--output-dir", default="artifacts/analysis") + return parser.parse_args(argv) + + +def write_csv(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8") + return + fieldnames = list(rows[0].keys()) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def main(argv=None) -> int: + args = parse_args(argv) + snapshot_path = Path(args.snapshot).expanduser() + if not snapshot_path.is_absolute(): + snapshot_path = REPO_ROOT / snapshot_path + with snapshot_path.open("r", encoding="utf-8") as handle: + snapshot = json.load(handle) + + class_names = None + if args.class_names: + class_names = { + idx: name.strip() + for idx, name in enumerate(args.class_names.split(",")) + } + + effect_rows = prototype_effect_rows(snapshot, args.betas, class_names) + weight_rows = client_weight_rows(snapshot, args.betas, class_names) + + output_dir = Path(args.output_dir).expanduser() + if not output_dir.is_absolute(): + output_dir = REPO_ROOT / output_dir + stem = snapshot_path.stem + effect_path = output_dir / f"{stem}__prototype_effect.csv" + weight_path = output_dir / f"{stem}__prototype_weights.csv" + write_csv(effect_path, effect_rows) + write_csv(weight_path, weight_rows) + + summary = { + "snapshot": str(snapshot_path), + "round": snapshot.get("round"), + "betas": [float(b) for b in args.betas], + "n_classes": len({row["class_id"] for row in effect_rows}), + "max_displacement": max( + (row["displacement_from_beta0"] for row in effect_rows), default=0.0 + ), + "effect_csv": str(effect_path), + "weights_csv": str(weight_path), + } + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_baseline.py b/scripts/run_baseline.py index 580293b..e9c6165 100644 --- a/scripts/run_baseline.py +++ b/scripts/run_baseline.py @@ -29,6 +29,7 @@ sys.path.insert(0, str(REPO_ROOT / "src")) from protean_research.data import manifest as manifest_mod # noqa: E402 from protean_research.data import partition as partition_mod # noqa: E402 from protean_research.data import rare_unseen # noqa: E402 +from protean_research.aggregation import validate_beta # noqa: E402 from protean_research.data.characterization import characterize_support # noqa: E402 from protean_research.data.pipeline import ( # noqa: E402 create_partition, @@ -61,6 +62,18 @@ def parse_args(argv=None) -> argparse.Namespace: parser.add_argument("--lr", type=float, default=1e-3) parser.add_argument("--mu", type=float, default=0.1) parser.add_argument("--lam", type=float, default=1.0) + parser.add_argument( + "--aggregation-mode", + choices=["original_protean", "support_weighted"], + default="original_protean", + help="server-side prototype aggregation mode (default: frozen baseline).", + ) + parser.add_argument( + "--beta", + type=float, + default=None, + help="support-weighting exponent in [0, 1]; required for support_weighted.", + ) parser.add_argument( "--max-rows", type=int, @@ -97,6 +110,20 @@ def write_csv(path: Path, rows: list[dict]) -> None: def main(argv=None) -> int: args = parse_args(argv) + + if args.aggregation_mode == "support_weighted": + if args.beta is None: + raise SystemExit( + "--beta is required when --aggregation-mode support_weighted" + ) + beta = validate_beta(args.beta) + else: + if args.beta is not None: + raise SystemExit( + "--beta is only valid with --aggregation-mode support_weighted" + ) + beta = None + datasets_cfg = load_datasets_config() protocol = load_yaml(REPO_ROOT / "configs" / "experiments" / "experiment_protocol.yaml") @@ -170,6 +197,8 @@ def main(argv=None) -> int: mu=args.mu, lam=args.lam, num_clients=args.clients, + aggregation_mode=args.aggregation_mode, + beta=beta, ) result = run_federated( train_features=train_features, @@ -224,7 +253,10 @@ def main(argv=None) -> int: }) # ---- structured artifacts -------------------------------------------- - run_name = f"{args.dataset}__alpha{args.alpha:g}__seed{args.seed}__original_protean" + mode_tag = args.aggregation_mode + if args.aggregation_mode == "support_weighted": + mode_tag = f"support_weighted__beta{beta:g}" + run_name = f"{args.dataset}__alpha{args.alpha:g}__seed{args.seed}__{mode_tag}" run_dir = results_root / run_name run_dir.mkdir(parents=True, exist_ok=True) @@ -237,7 +269,8 @@ def main(argv=None) -> int: "num_classes": bundle.num_classes, "class_names": bundle.class_names, "benign_class": bundle.spec.benign_label, - "aggregation_mode": "original_protean", + "aggregation_mode": args.aggregation_mode, + "beta": beta, "training": { "rounds": args.rounds, "local_epochs": args.local_epochs, diff --git a/src/protean_research/aggregation/__init__.py b/src/protean_research/aggregation/__init__.py index 458ad15..41a7bab 100644 --- a/src/protean_research/aggregation/__init__.py +++ b/src/protean_research/aggregation/__init__.py @@ -1,18 +1,26 @@ """Server-side prototype aggregation. -This package is the single locus of the proposed modification. In the baseline -checkpoint it implements only the reproduced original-PROTEAN aggregation. The -proposed support-weighted aggregation is deferred to the beta/controls phase. +This package is the single locus of the proposed modification. It implements +the reproduced original-PROTEAN aggregation (``original_protean``) and the +proposed class-specific sample-support weighting (``support_weighted``) behind an +explicit mode/beta interface. Missing-class handling (``S_j``) stays explicit in +the aggregation functions; no zero-vector padding is used. """ from .prototype_aggregation import ( + BETA_CANDIDATES, AggregationMode, aggregate_class_prototypes, aggregate_prototypes, + normalized_support_weights, + validate_beta, ) __all__ = [ + "BETA_CANDIDATES", "AggregationMode", "aggregate_class_prototypes", "aggregate_prototypes", + "normalized_support_weights", + "validate_beta", ] diff --git a/src/protean_research/aggregation/prototype_aggregation.py b/src/protean_research/aggregation/prototype_aggregation.py index 71d1d08..c298967 100644 --- a/src/protean_research/aggregation/prototype_aggregation.py +++ b/src/protean_research/aggregation/prototype_aggregation.py @@ -1,25 +1,49 @@ -"""Baseline server-side prototype aggregation. +"""Server-side prototype aggregation. -This module implements **only the reproduced original-PROTEAN prototype -aggregation** used by the baseline. The proposed support-weighted aggregation -(``N_ij ** beta``) is deliberately NOT present in this checkpoint; it belongs to -the later beta/controls phase and must not be introduced before the baseline is -verified. +This module implements the reproduced original-PROTEAN prototype aggregation +(``original_protean``) and the proposed class-specific sample-support weighting +(``support_weighted``). Only this server-side step differs between the two modes; +the data pipeline, model, loss, model averaging and inference stay frozen. -Participant rule (reproduction assumption, not verified against code) ---------------------------------------------------------------------- -The published Equation (2) / thesis Eq. (5.5) writes ``C_bar_j = (1/M) * sum_i -C_ij`` over all ``M`` clients, but never states how a class that is missing on a -client (``N_ij = 0``) is handled. The thesis separately states that "prototypes -are shared only for locally observed classes", so this baseline assumes the -server receives prototypes only from the contributor set:: +Supported modes +--------------- +``original_protean`` + Equal weights over the contributor set ``S_j`` (reproduction baseline):: + + Cbar_j = (1 / |S_j|) * sum_{i in S_j} C_ij + +``support_weighted`` + Class-specific sample-support weighting (the proposed method):: + + Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) + / ( sum_{i in S_j} N_ij^beta ) + + with ``0 <= beta <= 1``. + +Participant rule (``S_j``) +-------------------------- +Both modes share the same participant set:: S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } -and averages them with **equal weights ``1/|S_j|``**. This is an explicit -REPRODUCTION ASSUMPTION of this project, not a source-confirmed fact. It must be -validated in the baseline-verification phase; see -``docs/baseline/BASELINE_IMPLEMENTATION.md``. +Missing classes (``N_ij = 0`` or no prototype) never enter ``S_j`` and are never +fabricated as zero vectors. For ``original_protean`` this participant rule is an +explicit REPRODUCTION ASSUMPTION, not a source-confirmed fact (the published +Eq. (2) / thesis Eq. (5.5) writes ``1/M`` over all clients but never defines +missing-class handling). The same documented rule is reused for +``support_weighted`` so that beta candidates are compared on an identical +contributor set. + +Beta semantics +-------------- +- ``beta = 0``: every weight is ``N_ij ** 0 = 1``, so for a fixed ``S_j`` the + aggregate is the equal (arithmetic) mean over ``S_j``. This is *not* asserted to + be the original PROTEAN baseline beyond the shared participant rule. +- ``beta = 1``: weights are proportional to the class-specific support; because + each ``C_ij`` is a local class-embedding mean, the aggregate corresponds to the + combined mean of the included embeddings when all assumptions match. +- A single contributor and an all-equal support vector make the aggregate + independent of beta. """ from __future__ import annotations @@ -30,11 +54,15 @@ from typing import Mapping, Sequence Vector = Sequence[float] +#: Frozen beta candidates from the research specification. +BETA_CANDIDATES = (0.0, 0.25, 0.50, 0.75, 1.00) + class AggregationMode(str, Enum): - """Prototype-aggregation modes implemented in the baseline checkpoint.""" + """Prototype-aggregation modes implemented in this checkpoint.""" ORIGINAL_PROTEAN = "original_protean" + SUPPORT_WEIGHTED = "support_weighted" def _validate_vectors(prototypes: Sequence[Vector]) -> int: @@ -56,9 +84,53 @@ def _validate_vectors(prototypes: Sequence[Vector]) -> int: return dim +def validate_beta(beta) -> float: + """Return ``beta`` as a float in ``[0, 1]`` or raise ``ValueError``. + + Explicit and strict: ``None``, non-numeric, non-finite and out-of-range + values are all rejected rather than silently defaulted. + """ + if beta is None: + raise ValueError("beta is required for support_weighted aggregation") + if isinstance(beta, bool) or not isinstance(beta, (int, float)): + raise ValueError(f"beta must be a real number, got {beta!r}") + value = float(beta) + if not math.isfinite(value): + raise ValueError(f"beta must be finite, got {beta!r}") + if value < 0.0 or value > 1.0: + raise ValueError(f"beta must lie in the closed interval [0, 1], got {value}") + return value + + +def normalized_support_weights(supports: Sequence[float], beta) -> list[float]: + """Normalised ``N_ij ** beta`` weights that sum to one. + + ``supports`` are the class-specific supports ``N_ij`` of the clients in + ``S_j``; they must be strictly positive and finite. Used both by the + aggregation and by the direct prototype-effect instrumentation. + """ + value = validate_beta(beta) + if supports is None: + raise ValueError("supports are required for support_weighted aggregation") + weights = [] + for support in supports: + n = float(support) + if not math.isfinite(n): + raise ValueError(f"N_ij must be finite, got {support!r}") + if n <= 0.0: + raise ValueError("support_weighted requires strictly positive N_ij") + weights.append(n ** value) + total = math.fsum(weights) + if not math.isfinite(total) or total <= 0.0: + raise ValueError("support weights must sum to a positive finite value") + return [w / total for w in weights] + + def aggregate_prototypes( prototypes: Sequence[Vector], mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, + supports: Sequence[float] | None = None, + beta=None, ) -> list[float]: """Aggregate the local prototypes ``C_ij`` of one class into ``C_bar_j``. @@ -67,28 +139,48 @@ def aggregate_prototypes( prototypes: Local prototypes ``C_ij`` for the contributing clients in ``S_j``. mode: - Only :attr:`AggregationMode.ORIGINAL_PROTEAN` is implemented in this - baseline checkpoint. Other modes raise ``NotImplementedError``. - - The baseline uses equal weights over the supplied contributor set, so the - result is the arithmetic mean of ``prototypes``. + :attr:`AggregationMode.ORIGINAL_PROTEAN` (equal mean) or + :attr:`AggregationMode.SUPPORT_WEIGHTED` (``N_ij ** beta``). + supports: + Class-specific supports ``N_ij`` aligned with ``prototypes``. Required + for ``support_weighted``; ignored by ``original_protean``. + beta: + Support-weighting exponent in ``[0, 1]``. Only valid for + ``support_weighted``; passing it with ``original_protean`` is an error. """ - # ``AggregationMode`` contains only the baseline mode, so any other value - # raises ``ValueError`` here. This prevents the proposed support-weighted - # aggregation from being used before the baseline is verified. - AggregationMode(mode) + resolved = AggregationMode(mode) dim = _validate_vectors(prototypes) - count = len(prototypes) + + if resolved is AggregationMode.ORIGINAL_PROTEAN: + if beta is not None: + raise ValueError("beta is only valid for support_weighted aggregation") + count = len(prototypes) + result = [0.0] * dim + for vec in prototypes: + for d in range(dim): + result[d] += float(vec[d]) / count + return result + + # support_weighted + if supports is None: + raise ValueError("supports are required for support_weighted aggregation") + if len(supports) != len(prototypes): + raise ValueError( + f"supports and prototypes must have equal length, got " + f"{len(supports)} vs {len(prototypes)}" + ) + weights = normalized_support_weights(supports, beta) result = [0.0] * dim - for vec in prototypes: + for weight, vec in zip(weights, prototypes): for d in range(dim): - result[d] += float(vec[d]) / count + result[d] += weight * float(vec[d]) return result def aggregate_class_prototypes( contributions, mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, + beta=None, ): """Aggregate per-class local prototypes into global prototypes. @@ -101,16 +193,21 @@ def aggregate_class_prototypes( """ if not isinstance(contributions, Mapping): raise TypeError("contributions must be a mapping class_id -> [(C_ij, N_ij), ...]") + resolved = AggregationMode(mode) result = {} for class_id in sorted(contributions, key=lambda c: int(c)): member_prototypes: list[Vector] = [] + member_supports: list[float] = [] for proto, support in contributions[class_id]: if proto is None: continue if support is None or float(support) <= 0.0: continue member_prototypes.append(proto) + member_supports.append(float(support)) if not member_prototypes: continue - result[int(class_id)] = aggregate_prototypes(member_prototypes, mode) + result[int(class_id)] = aggregate_prototypes( + member_prototypes, resolved, supports=member_supports, beta=beta + ) return result diff --git a/src/protean_research/analysis/__init__.py b/src/protean_research/analysis/__init__.py new file mode 100644 index 0000000..1ba64fa --- /dev/null +++ b/src/protean_research/analysis/__init__.py @@ -0,0 +1,24 @@ +"""Analysis utilities that do not depend on training. + +Currently this package implements the direct prototype-effect analysis +(``direct_prototype``), which recomputes global prototypes from a fixed round +snapshot of ``{C_ij, N_ij}`` for several beta without retraining. +""" + +from .direct_prototype import ( + DEFAULT_BETAS, + aggregate_snapshot, + client_weight_rows, + prototype_effect_rows, + snapshot_class_members, + snapshot_contributions, +) + +__all__ = [ + "DEFAULT_BETAS", + "aggregate_snapshot", + "client_weight_rows", + "prototype_effect_rows", + "snapshot_class_members", + "snapshot_contributions", +] diff --git a/src/protean_research/analysis/direct_prototype.py b/src/protean_research/analysis/direct_prototype.py new file mode 100644 index 0000000..9274a1e --- /dev/null +++ b/src/protean_research/analysis/direct_prototype.py @@ -0,0 +1,184 @@ +"""Direct prototype-effect analysis on a fixed ``{C_ij, N_ij}`` snapshot. + +This separates the *direct aggregation effect* (what changing beta does to the +global prototypes for a fixed set of local prototypes) from the *end-to-end +training effect* (what happens when beta also changes training). It requires no +retraining: the federated loop already stores a round snapshot of every client's +local prototypes ``C_ij`` and class supports ``N_ij``. + +The snapshot format is the one produced by +``protean_research.federated.round.run_federated``:: + + { + "round": 0, + "client_prototypes": [ {class_id: [floats], ...}, ... ], + "client_supports": [ {class_id: int, ...}, ... ], + } + +Both JSON-loaded (string class ids) and in-memory (int class ids) snapshots are +accepted. All analysis reuses the production aggregation functions, so the +reported global prototypes are exactly those a beta run would have aggregated in +that round; only the training feedback is removed. +""" + +from __future__ import annotations + +import math +from typing import Mapping, Sequence + +from ..aggregation import ( + AggregationMode, + aggregate_class_prototypes, + normalized_support_weights, +) + +DEFAULT_BETAS = (0.0, 0.25, 0.50, 0.75, 1.00) + +_SUPPORT_WEIGHTED = AggregationMode.SUPPORT_WEIGHTED + + +def _lookup(mapping: Mapping, class_id: int): + """Support both JSON (str keys) and in-memory (int keys) class mappings.""" + if class_id in mapping: + return mapping[class_id] + return mapping.get(str(class_id)) + + +def snapshot_class_members(snapshot: Mapping) -> dict[int, list[tuple[int, object, float]]]: + """Return ``class_id -> [(client_id, C_ij, N_ij), ...]`` from a snapshot.""" + try: + client_prototypes = snapshot["client_prototypes"] + client_supports = snapshot["client_supports"] + except (KeyError, TypeError) as exc: # pragma: no cover - defensive + raise ValueError("snapshot must contain client_prototypes and client_supports") from exc + if len(client_prototypes) != len(client_supports): + raise ValueError( + "snapshot client_prototypes and client_supports must have equal length" + ) + + members: dict[int, list[tuple[int, object, float]]] = {} + for client_id, (prototypes, supports) in enumerate( + zip(client_prototypes, client_supports) + ): + for raw_class_id, proto in prototypes.items(): + class_id = int(raw_class_id) + support = _lookup(supports, class_id) + members.setdefault(class_id, []).append((client_id, proto, support)) + return members + + +def snapshot_contributions(snapshot: Mapping) -> dict[int, list[tuple[object, float]]]: + """Return ``class_id -> [(C_ij, N_ij), ...]`` for the aggregation module.""" + members = snapshot_class_members(snapshot) + return { + class_id: [(proto, support) for _, proto, support in entries] + for class_id, entries in members.items() + } + + +def aggregate_snapshot(snapshot: Mapping, beta) -> dict[int, list[float]]: + """Aggregate the snapshot's local prototypes with the proposed mode.""" + return aggregate_class_prototypes( + snapshot_contributions(snapshot), _SUPPORT_WEIGHTED, beta=beta + ) + + +def _l2_distance(a: Sequence[float], b: Sequence[float]) -> float: + if len(a) != len(b): + raise ValueError("prototype vectors must share the same dimension") + return math.sqrt(math.fsum((float(x) - float(y)) ** 2 for x, y in zip(a, b))) + + +def _norm(vec: Sequence[float]) -> float: + return math.sqrt(math.fsum(float(x) ** 2 for x in vec)) + + +def prototype_effect_rows( + snapshot: Mapping, + betas: Sequence[float] = DEFAULT_BETAS, + class_names: Mapping[int, str] | None = None, +) -> list[dict]: + """Per-class rows of ``|| Cbar_j(beta) - Cbar_j(0) ||`` and prototype norms. + + One row per ``(class_id, beta)``. ``displacement`` is the direct aggregation + effect relative to the equal-weight ``beta = 0`` aggregate on the *same* + contributor set. A large displacement is not evidence of improved detection. + """ + if not betas: + raise ValueError("at least one beta is required") + contributions = snapshot_contributions(snapshot) + baseline = aggregate_class_prototypes(contributions, _SUPPORT_WEIGHTED, beta=0.0) + class_names = class_names or {} + members_by_class = snapshot_class_members(snapshot) + + rows: list[dict] = [] + for class_id in sorted(baseline): + members = [ + support + for _, _, support in members_by_class.get(class_id, []) + if support is not None and float(support) > 0.0 + ] + base_norm = _norm(baseline[class_id]) + for beta in betas: + cbar = aggregate_snapshot(snapshot, beta)[class_id] + displacement = _l2_distance(cbar, baseline[class_id]) + rows.append( + { + "class_id": class_id, + "class_name": class_names.get(class_id), + "beta": float(beta), + "n_contributors": len(members), + "support_min": min(members) if members else None, + "support_max": max(members) if members else None, + "support_mean": (sum(members) / len(members)) if members else None, + "prototype_norm": _norm(cbar), + "reference_norm_beta0": base_norm, + "displacement_from_beta0": displacement, + "relative_displacement": ( + displacement / base_norm if base_norm > 0.0 else None + ), + } + ) + return rows + + +def client_weight_rows( + snapshot: Mapping, + betas: Sequence[float] = DEFAULT_BETAS, + class_names: Mapping[int, str] | None = None, +) -> list[dict]: + """Per-``(class, beta, client)`` normalised aggregation weights. + + Only clients in ``S_j`` (positive support and a valid prototype) appear; the + reported weights sum to one within each ``(class, beta)`` group. + """ + if not betas: + raise ValueError("at least one beta is required") + members = snapshot_class_members(snapshot) + class_names = class_names or {} + + rows: list[dict] = [] + for class_id in sorted(members): + valid = [ + (client_id, support) + for client_id, _, support in members[class_id] + if support is not None and float(support) > 0.0 + ] + if not valid: + continue + for beta in betas: + weights = normalized_support_weights( + [support for _, support in valid], beta + ) + for (client_id, support), weight in zip(valid, weights): + rows.append( + { + "class_id": class_id, + "class_name": class_names.get(class_id), + "beta": float(beta), + "client_id": client_id, + "N_ij": float(support), + "weight": weight, + } + ) + return rows diff --git a/src/protean_research/federated/round.py b/src/protean_research/federated/round.py index 5705ed3..7588c97 100644 --- a/src/protean_research/federated/round.py +++ b/src/protean_research/federated/round.py @@ -38,6 +38,8 @@ class TrainingConfig: dense_units: int = 128 snapshot_round: Optional[int] = 0 device: str = "cpu" + aggregation_mode: str = "original_protean" + beta: Optional[float] = None def _evaluate(model, features, labels, global_prototypes, num_classes, rare_class_ids=None): @@ -122,6 +124,8 @@ def run_federated( [u.prototypes for u in updates], [u.supports for u in updates], num_classes, + mode=config.aggregation_mode, + beta=config.beta, ) if config.snapshot_round is not None and round_id == config.snapshot_round: diff --git a/src/protean_research/federated/server.py b/src/protean_research/federated/server.py index d7ed68b..bd9af93 100644 --- a/src/protean_research/federated/server.py +++ b/src/protean_research/federated/server.py @@ -25,10 +25,16 @@ def aggregate_prototypes( client_supports: Sequence[Mapping[int, int]], num_classes: int, mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, + beta=None, ): - """Aggregate client prototypes into ``{class_id: Cbar_j}`` (baseline).""" + """Aggregate client prototypes into ``{class_id: Cbar_j}``. + + ``mode`` selects equal ``original_protean`` averaging or the proposed + ``support_weighted`` aggregation with exponent ``beta``. The contributor set + ``S_j`` is identical for both modes. + """ return collect_global_prototypes( - client_prototypes, client_supports, num_classes, mode=mode + client_prototypes, client_supports, num_classes, mode=mode, beta=beta ) diff --git a/src/protean_research/models/prototypes.py b/src/protean_research/models/prototypes.py index 736df98..30f3512 100644 --- a/src/protean_research/models/prototypes.py +++ b/src/protean_research/models/prototypes.py @@ -41,18 +41,20 @@ def collect_global_prototypes( client_supports: Sequence[Mapping[int, int]], num_classes: int, mode: AggregationMode | str = AggregationMode.ORIGINAL_PROTEAN, + beta=None, ): """Server-side aggregation of per-client prototypes into global prototypes. Clients that did not observe a class simply do not appear in ``client_prototypes`` for that class, so ``S_j`` is built implicitly and - missing classes are never inserted as zero vectors. + missing classes are never inserted as zero vectors. ``beta`` is forwarded to + the aggregation module and is only valid for ``support_weighted``. """ contributions: dict[int, list] = {c: [] for c in range(num_classes)} for prototypes, supports in zip(client_prototypes, client_supports): for class_id, proto in prototypes.items(): contributions[int(class_id)].append((proto, supports.get(int(class_id), 0))) - return aggregate_class_prototypes(contributions, mode) + return aggregate_class_prototypes(contributions, mode, beta=beta) def nearest_prototype_predict(embeddings, global_prototypes: Mapping[int, object]): diff --git a/tests/test_baseline_aggregation.py b/tests/test_baseline_aggregation.py index 86fc5b7..566ae49 100644 --- a/tests/test_baseline_aggregation.py +++ b/tests/test_baseline_aggregation.py @@ -61,10 +61,19 @@ def test_missing_class_is_not_fabricated(): assert result == {} -def test_proposed_mode_is_not_available(): - # The support-weighted mode belongs to the later beta/controls phase. +def test_support_weighted_requires_supports_and_beta(): + # The proposed mode now exists but must be fully parameterised. with pytest.raises(ValueError): - aggregate_prototypes(PROTOTYPES, "support_weighted") + aggregate_prototypes(PROTOTYPES, "support_weighted") # no supports/beta + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, "support_weighted", supports=[1, 2, 3]) + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, "not_a_mode") + + +def test_beta_is_rejected_for_original_protean(): + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, "original_protean", beta=0.5) def test_validation_errors(): diff --git a/tests/test_prototype_effect.py b/tests/test_prototype_effect.py new file mode 100644 index 0000000..8c7ec7a --- /dev/null +++ b/tests/test_prototype_effect.py @@ -0,0 +1,142 @@ +"""Direct prototype-effect instrumentation and support_weighted end-to-end wiring.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from protean_research.analysis.direct_prototype import ( + aggregate_snapshot, + client_weight_rows, + prototype_effect_rows, + snapshot_class_members, +) + + +def make_snapshot(): + # Two classes. Class 0 has strongly imbalanced supports across 3 clients; + # class 1 has a single contributor (beta-invariant). + return { + "round": 0, + "client_prototypes": [ + {0: [0.0, 0.0], 1: [1.0, 1.0]}, + {0: [4.0, 8.0]}, + {0: [10.0, 2.0]}, + ], + "client_supports": [ + {0: 1, 1: 5}, + {0: 9}, + {0: 4}, + ], + } + + +def test_snapshot_class_members_aligns_supports(): + members = snapshot_class_members(make_snapshot()) + assert sorted(members) == [0, 1] + assert [cid for cid, _, _ in members[0]] == [0, 1, 2] + assert [n for _, _, n in members[0]] == [1, 9, 4] + assert members[1] == [(0, [1.0, 1.0], 5)] + + +def test_beta0_effect_is_zero_and_beta1_displaces(): + rows = prototype_effect_rows(make_snapshot(), betas=[0.0, 1.0]) + by_key = {(r["class_id"], r["beta"]): r for r in rows} + assert by_key[(0, 0.0)]["displacement_from_beta0"] == pytest.approx(0.0) + assert by_key[(0, 0.0)]["relative_displacement"] == pytest.approx(0.0) + assert by_key[(0, 1.0)]["displacement_from_beta0"] > 0.0 + # Single contributor -> no displacement for any beta. + assert by_key[(1, 1.0)]["displacement_from_beta0"] == pytest.approx(0.0) + assert by_key[(1, 1.0)]["n_contributors"] == 1 + + +def test_effect_beta0_matches_arithmetic_mean_aggregate(): + snapshot = make_snapshot() + aggregated = aggregate_snapshot(snapshot, 0.0) + assert aggregated[0] == pytest.approx([14.0 / 3.0, 10.0 / 3.0]) + assert aggregated[1] == pytest.approx([1.0, 1.0]) + + +def test_client_weights_sum_to_one_per_class_beta(): + rows = client_weight_rows(make_snapshot(), betas=[0.0, 0.5, 1.0]) + groups: dict[tuple, float] = {} + for row in rows: + key = (row["class_id"], row["beta"]) + groups[key] = groups.get(key, 0.0) + row["weight"] + assert groups + for total in groups.values(): + assert total == pytest.approx(1.0) + + +def test_json_style_string_class_ids_are_supported(): + snapshot = make_snapshot() + json_like = { + "round": 0, + "client_prototypes": [ + {str(c): v for c, v in prototypes.items()} + for prototypes in snapshot["client_prototypes"] + ], + "client_supports": [ + {str(c): v for c, v in supports.items()} + for supports in snapshot["client_supports"] + ], + } + rows = prototype_effect_rows(json_like, betas=[0.0, 1.0]) + assert {(r["class_id"], r["beta"]) for r in rows} == {(0, 0.0), (0, 1.0), (1, 0.0), (1, 1.0)} + + +# --------------------------------------------------------------------------- +# End-to-end wiring: support_weighted must run through the federated loop. +# --------------------------------------------------------------------------- + +torch = pytest.importorskip("torch") + +from protean_research.federated import TrainingConfig, run_federated # noqa: E402 + + +def _run_support_weighted(beta, seed_parts): + rng = np.random.default_rng(0) + num_classes, per_class, dim = 3, 10, 6 + features = rng.normal(size=(per_class * num_classes, dim)).astype(np.float32) + labels = np.repeat(np.arange(num_classes), per_class).astype(np.int64) + features = features + labels[:, None] * 2.0 + assignments = [np.arange(0, 10), np.arange(10, 20), np.arange(20, 30)] + config = TrainingConfig( + rounds=2, + local_epochs=1, + batch_size=10, + num_clients=3, + embedding_dim=8, + dense_units=8, + aggregation_mode="support_weighted", + beta=beta, + ) + return run_federated( + train_features=features, + train_labels=labels, + assignments=assignments, + val_features=features, + val_labels=labels, + num_classes=num_classes, + input_dim=dim, + config=config, + init_seed_parts=seed_parts, + ) + + +def test_support_weighted_end_to_end_is_deterministic_and_finite(): + first = _run_support_weighted(0.5, ("sw", "seed=0")) + second = _run_support_weighted(0.5, ("sw", "seed=0")) + assert set(first["global_prototypes"]) == {0, 1, 2} + for c in first["global_prototypes"]: + assert np.isfinite(np.asarray(first["global_prototypes"][c])).all() + assert np.array_equal( + np.asarray(first["global_prototypes"][c]), + np.asarray(second["global_prototypes"][c]), + ) + assert np.isfinite(first["history"][-1]["accuracy"]) + + +def test_support_weighted_rejects_bad_beta_end_to_end(): + with pytest.raises(ValueError): + _run_support_weighted(1.5, ("sw", "bad")) diff --git a/tests/test_support_weighted_aggregation.py b/tests/test_support_weighted_aggregation.py new file mode 100644 index 0000000..d28380c --- /dev/null +++ b/tests/test_support_weighted_aggregation.py @@ -0,0 +1,200 @@ +"""Invariant tests for the proposed support-weighted prototype aggregation. + +These tests pin the required arithmetic properties of ``support_weighted`` with +``w_ij = N_ij ** beta`` over the same contributor set ``S_j`` as the baseline: + +- ``beta = 0`` is the equal (arithmetic) mean over ``S_j``; +- ``beta = 1`` is the support-weighted mean; +- a hand-computed intermediate-beta fixture; +- single-contributor invariance; +- equal-support invariance; +- missing-class exclusion (``S_j`` construction); +- normalised weights sum to one; +- invalid-beta rejection. + +They do not assert that ``beta = 0`` *is* original PROTEAN; that depends on the +separately documented participant rule. +""" + +from __future__ import annotations + +import math + +import pytest + +from protean_research.aggregation import ( + BETA_CANDIDATES, + AggregationMode, + aggregate_class_prototypes, + aggregate_prototypes, + normalized_support_weights, + validate_beta, +) + +SUPPORT_WEIGHTED = AggregationMode.SUPPORT_WEIGHTED +ORIGINAL = AggregationMode.ORIGINAL_PROTEAN + +# Prototypes with strongly different supports to make weighting observable. +PROTOTYPES = [[0.0, 0.0], [4.0, 8.0], [10.0, 2.0]] +SUPPORTS = [1.0, 9.0, 4.0] + + +def arithmetic_mean(vectors): + dim = len(vectors[0]) + return [sum(v[d] for v in vectors) / len(vectors) for d in range(dim)] + + +def support_weighted_mean(vectors, supports, beta): + weights = [float(n) ** beta for n in supports] + total = sum(weights) + dim = len(vectors[0]) + return [sum(w * v[d] for w, v in zip(weights, vectors)) / total for d in range(dim)] + + +# --------------------------------------------------------------------------- +# beta = 0 +# --------------------------------------------------------------------------- + + +def test_beta0_is_equal_arithmetic_mean_over_s_j(): + out = aggregate_prototypes( + PROTOTYPES, SUPPORT_WEIGHTED, supports=SUPPORTS, beta=0.0 + ) + assert out == pytest.approx(arithmetic_mean(PROTOTYPES)) + + +def test_beta0_matches_original_protean_on_identical_s_j(): + weighted = aggregate_prototypes( + PROTOTYPES, SUPPORT_WEIGHTED, supports=SUPPORTS, beta=0.0 + ) + baseline = aggregate_prototypes(PROTOTYPES, ORIGINAL) + assert weighted == pytest.approx(baseline, rel=1e-6, abs=1e-6) + + +# --------------------------------------------------------------------------- +# beta = 1 and intermediate beta +# --------------------------------------------------------------------------- + + +def test_beta1_is_support_weighted_mean(): + out = aggregate_prototypes( + PROTOTYPES, SUPPORT_WEIGHTED, supports=SUPPORTS, beta=1.0 + ) + assert out == pytest.approx(support_weighted_mean(PROTOTYPES, SUPPORTS, 1.0)) + + +def test_intermediate_beta_fixture(): + # two contributors: C = [0, 4], N = 1 and 9; beta = 0.5 -> weights 1, 3. + out = aggregate_prototypes( + [[0.0, 0.0], [4.0, 4.0]], SUPPORT_WEIGHTED, supports=[1.0, 9.0], beta=0.5 + ) + expected = [3.0, 3.0] # (1*0 + 3*4) / 4 + assert out == pytest.approx(expected) + + +def test_intermediate_beta_matches_closed_form(): + for beta in (0.25, 0.5, 0.75): + out = aggregate_prototypes( + PROTOTYPES, SUPPORT_WEIGHTED, supports=SUPPORTS, beta=beta + ) + assert out == pytest.approx(support_weighted_mean(PROTOTYPES, SUPPORTS, beta)) + + +# --------------------------------------------------------------------------- +# invariance +# --------------------------------------------------------------------------- + + +def test_single_contributor_is_invariant_to_beta(): + single = [[1.5, -2.0, 3.0]] + for beta in BETA_CANDIDATES: + out = aggregate_prototypes(single, SUPPORT_WEIGHTED, supports=[7.0], beta=beta) + assert out == pytest.approx(single[0]) + + +def test_equal_support_is_invariant_to_beta(): + supports = [5.0, 5.0, 5.0] + reference = arithmetic_mean(PROTOTYPES) + for beta in BETA_CANDIDATES: + out = aggregate_prototypes( + PROTOTYPES, SUPPORT_WEIGHTED, supports=supports, beta=beta + ) + assert out == pytest.approx(reference, rel=1e-9, abs=1e-9) + + +# --------------------------------------------------------------------------- +# S_j construction / missing classes +# --------------------------------------------------------------------------- + + +def test_missing_class_is_excluded_from_s_j(): + contributions = { + 0: [([1.0, 1.0], 5), (None, 5), ([3.0, 3.0], 0), ([5.0, 5.0], 5)], + } + result = aggregate_class_prototypes(contributions, SUPPORT_WEIGHTED, beta=1.0) + # Only the two positive-support entries with prototypes enter: mean = 3. + assert result[0] == pytest.approx([3.0, 3.0]) + + +def test_class_with_only_missing_contributors_is_omitted(): + result = aggregate_class_prototypes( + {0: [(None, 3), ([1.0], 0)]}, SUPPORT_WEIGHTED, beta=0.5 + ) + assert result == {} + + +# --------------------------------------------------------------------------- +# weights +# --------------------------------------------------------------------------- + + +def test_normalized_weights_sum_to_one(): + for beta in BETA_CANDIDATES: + weights = normalized_support_weights(SUPPORTS, beta) + assert sum(weights) == pytest.approx(1.0) + assert all(w > 0.0 for w in weights) + + +def test_beta0_weights_are_uniform(): + weights = normalized_support_weights(SUPPORTS, 0.0) + assert weights == pytest.approx([1 / 3, 1 / 3, 1 / 3]) + + +def test_output_is_finite_for_all_candidates(): + for beta in BETA_CANDIDATES: + out = aggregate_prototypes( + PROTOTYPES, SUPPORT_WEIGHTED, supports=SUPPORTS, beta=beta + ) + assert all(math.isfinite(v) for v in out) + + +# --------------------------------------------------------------------------- +# validation / rejection +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("beta", [-0.01, 1.01, -1.0, 2.0, float("nan"), float("inf")]) +def test_invalid_beta_is_rejected(beta): + with pytest.raises(ValueError): + validate_beta(beta) + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, SUPPORT_WEIGHTED, supports=SUPPORTS, beta=beta) + + +def test_none_and_non_numeric_beta_are_rejected(): + for beta in (None, "0.5", True, [0.5]): + with pytest.raises(ValueError): + validate_beta(beta) + + +def test_supports_must_be_strictly_positive_and_aligned(): + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, SUPPORT_WEIGHTED, supports=[1, 2], beta=1.0) + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, SUPPORT_WEIGHTED, supports=[1, 0, 3], beta=1.0) + with pytest.raises(ValueError): + aggregate_prototypes(PROTOTYPES, SUPPORT_WEIGHTED, supports=None, beta=1.0) + + +def test_beta_candidates_are_the_frozen_protocol_values(): + assert BETA_CANDIDATES == (0.0, 0.25, 0.50, 0.75, 1.00)