From 111adb272e8cffb462cc168ee887f8f0d6631976 Mon Sep 17 00:00:00 2001 From: willtanoe Date: Sat, 17 Oct 2026 04:06:04 +0700 Subject: [PATCH] research: add post-hoc statistical audit and defense preparation aids - scripts/posthoc_statistical_audit.py + docs/research/POSTHOC_STATISTICAL_AUDIT.md: read-only paired audit of the completed grid (exact sign-flip / Wilcoxon, selection stability, per-class/per-client summaries); writes only posthoc__*. - kisikisithesis/: robustness audit and examiner-preparation document with deterministic read-only checks (robustness_checks.py). - artifacts/analysis/posthoc__*.csv: audit outputs (6 tables). - artifacts/analysis: track the 4 remaining beta0-snapshot effect/weight CSVs (xiiotid a0.5 seeds 1-2), completing the tracked beta0 set. - .gitignore: codify that non-beta0 prototype snapshots stay untracked. --- .gitignore | 5 + .../posthoc__paired_global_by_condition.csv | 31 ++ .../posthoc__paired_global_per_seed.csv | 91 ++++ .../analysis/posthoc__paired_test_results.csv | 31 ++ .../analysis/posthoc__per_class_summary.csv | 19 + .../analysis/posthoc__per_client_summary.csv | 7 + .../analysis/posthoc__selection_stability.csv | 7 + ...__prototype_snapshot__prototype_effect.csv | 46 ++ ..._prototype_snapshot__prototype_weights.csv | 431 +++++++++++++++++ ...__prototype_snapshot__prototype_effect.csv | 46 ++ ..._prototype_snapshot__prototype_weights.csv | 446 +++++++++++++++++ docs/research/POSTHOC_STATISTICAL_AUDIT.md | 335 +++++++++++++ .../ROBUSTNESS_AUDIT_DAN_PERSIAPAN_PENGUJI.md | 454 ++++++++++++++++++ kisikisithesis/robustness_checks.py | 169 +++++++ scripts/posthoc_statistical_audit.py | 313 ++++++++++++ 15 files changed, 2431 insertions(+) create mode 100644 artifacts/analysis/posthoc__paired_global_by_condition.csv create mode 100644 artifacts/analysis/posthoc__paired_global_per_seed.csv create mode 100644 artifacts/analysis/posthoc__paired_test_results.csv create mode 100644 artifacts/analysis/posthoc__per_class_summary.csv create mode 100644 artifacts/analysis/posthoc__per_client_summary.csv create mode 100644 artifacts/analysis/posthoc__selection_stability.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_effect.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_weights.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_effect.csv create mode 100644 artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_weights.csv create mode 100644 docs/research/POSTHOC_STATISTICAL_AUDIT.md create mode 100644 kisikisithesis/ROBUSTNESS_AUDIT_DAN_PERSIAPAN_PENGUJI.md create mode 100644 kisikisithesis/robustness_checks.py create mode 100644 scripts/posthoc_statistical_audit.py diff --git a/.gitignore b/.gitignore index 24f3aae..b3b1ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,11 @@ venv/ # NOTE: artifacts/partitions/ and artifacts/analysis/ are intentionally NOT ignored: # partition manifests are deterministic, content-hashed, and required for paired # comparisons; analysis outputs are small. They are intended to be version controlled. +# EXCEPTION: the 18 beta0-round-0 prototype snapshots (and their effect/weight +# CSVs) ARE tracked; the remaining ~76 snapshot JSONs (~28 MB, needed only to +# regenerate already-committed displacement tables) stay local-only like +# results/. They are cited by path in the audit documents; do not stage them. +artifacts/analysis/*__prototype_snapshot.json artifacts/processed/ artifacts/checkpoints/ results/ diff --git a/artifacts/analysis/posthoc__paired_global_by_condition.csv b/artifacts/analysis/posthoc__paired_global_by_condition.csv new file mode 100644 index 0000000..e1ac719 --- /dev/null +++ b/artifacts/analysis/posthoc__paired_global_by_condition.csv @@ -0,0 +1,31 @@ +dataset,alpha,selected_beta,metric,original_seed0,original_seed1,original_seed2,proposed_seed0,proposed_seed1,proposed_seed2,delta_seed0,delta_seed1,delta_seed2,delta_mean,delta_std_sample,delta_median,delta_min,delta_max,baseline_std_sample,proposed_std_sample,n_pos,n_neg,n_zero,same_sign_all_seeds +xiiotid,0.25,0.0,rare_class_macro_f1,0.6398103625587044,0.5825215614975117,0.6760389917828752,0.6413622282749385,0.5805553678969027,0.6744891962601014,0.0015518657162341132,-0.001966193600609012,-0.0015497955227737892,-0.0006547078023828959,0.0019222569702312445,-0.0015497955227737892,-0.001966193600609012,0.0015518657162341132,0.04715228886220545,0.04764177856302672,1,2,0,False +xiiotid,0.25,0.0,rare_class_recall,0.8296012943628512,0.8266995877156649,0.8095946724338723,0.830700667573069,0.8256994230374527,0.807370314447251,0.0010993732102178022,-0.0010001646782121787,-0.0022243579866212393,-0.0007083831515385386,0.0016809668681122992,-0.0010001646782121787,-0.0022243579866212393,0.0010993732102178022,0.010810973971484781,0.012283281629867224,1,2,0,False +xiiotid,0.25,0.0,macro_f1,0.6731025782780014,0.6669813754005415,0.7059686496708458,0.6745583013411162,0.6650385577782836,0.7044332575239617,0.001455723063114811,-0.0019428176222578974,-0.0015353921468841403,-0.0006741622353424089,0.0018557498263327506,-0.0015353921468841403,-0.0019428176222578974,0.001455723063114811,0.02096686007526524,0.020555132742947537,1,2,0,False +xiiotid,0.25,0.0,weighted_f1,0.9054655196913645,0.9020747305215218,0.9154163421325315,0.9061557688476052,0.899863925003673,0.9146272396181951,0.0006902491562407187,-0.0022108055178488017,-0.0007891025143363928,-0.0007698862919814919,0.0014506227982888447,-0.0007891025143363928,-0.0022108055178488017,0.0006902491562407187,0.0069343939753111275,0.007408425095578143,1,2,0,False +xiiotid,0.25,0.0,accuracy,0.8737218342944525,0.8673409854388906,0.8852306993140127,0.8745509147521049,0.8634903673133494,0.8840024319693425,0.000829080457652398,-0.0038506181255412164,-0.0012282673446701864,-0.0014166016708530016,0.0023455270348474206,-0.0012282673446701864,-0.0038506181255412164,0.000829080457652398,0.009066523301774463,0.010266545051804106,1,2,0,False +xiiotid,0.5,0.75,rare_class_macro_f1,0.6096198579015474,0.655665402810942,0.662274505162012,0.6361887569975536,0.6719373483509694,0.6883814197743454,0.026568899096006238,0.01627194554002742,0.026106914612333387,0.022982586416122348,0.0058161742702334575,0.026106914612333387,0.01627194554002742,0.026568899096006238,0.028683282930463034,0.026684712591240056,3,0,0,True +xiiotid,0.5,0.75,rare_class_recall,0.8714872155034664,0.8893650106408222,0.8700179539413673,0.8575434903523123,0.8518834947182458,0.878321192150252,-0.013943725151154163,-0.037481515922576425,0.008303238208884678,-0.014374000954948637,0.022895409598216065,-0.013943725151154163,-0.037481515922576425,0.008303238208884678,0.010770970951613777,0.013920609948849614,1,2,0,False +xiiotid,0.5,0.75,macro_f1,0.7077011542142579,0.7091760460445067,0.7100168446133323,0.7262984173916737,0.7290145924987934,0.7381943369820925,0.018597263177415724,0.019838546454286732,0.028177492368760215,0.022204434000154222,0.005209919936951091,0.019838546454286732,0.018597263177415724,0.028177492368760215,0.0011722251041387443,0.006233755140268,3,0,0,True +xiiotid,0.5,0.75,weighted_f1,0.9175853183547894,0.929132450103975,0.9274806689914715,0.9222775427473946,0.9447335103607959,0.9338838836432976,0.004692224392605193,0.015601060256820842,0.006403214651826028,0.008898833100417355,0.005867005927967981,0.006403214651826028,0.004692224392605193,0.015601060256820842,0.006244765764216191,0.011230108565482134,3,0,0,True +xiiotid,0.5,0.75,accuracy,0.8904324115186911,0.9008849666218349,0.9070447273553561,0.8886698478790893,0.9254933028723031,0.9050119448999269,-0.0017625636396018107,0.02460833625046821,-0.0020327824554292606,0.006937663385145713,0.015303848018947405,-0.0017625636396018107,-0.0020327824554292606,0.02460833625046821,0.008398091032446977,0.018450460685614026,1,2,0,False +xiiotid,0.75,0.5,rare_class_macro_f1,0.5215401253860467,0.6140607772418322,0.6760680764285393,0.5118888965076885,0.6241440493793726,0.6797040888050662,-0.00965122887835812,0.010083272137540455,0.003636012376526887,0.0013560185452364075,0.010062872924253166,0.003636012376526887,-0.00965122887835812,0.010083272137540455,0.07776445613264914,0.08548886486265879,2,1,0,False +xiiotid,0.75,0.5,rare_class_recall,0.8164397897152784,0.8832376049795277,0.8698349315628103,0.8038871133887995,0.8767262170299326,0.8656806653880937,-0.01255267632647894,-0.00651138794959516,-0.004154266174716592,-0.007739443483596897,0.004331791034347159,-0.00651138794959516,-0.01255267632647894,-0.004154266174716592,0.03533794219558259,0.039255535276369505,0,3,0,True +xiiotid,0.75,0.5,macro_f1,0.7424713724903063,0.7473954752926397,0.7656800124975965,0.741976732126316,0.7617524315559205,0.7691786586117774,-0.000494640363990273,0.014356956263280818,0.0034986461141808256,0.00578698733782379,0.007685691885683445,0.0034986461141808256,-0.000494640363990273,0.014356956263280818,0.012228464472937677,0.014060416901925107,2,1,0,False +xiiotid,0.75,0.5,weighted_f1,0.9325633790408211,0.9359372345770101,0.9489075724365953,0.9355656459425888,0.9483435473557725,0.9517397334421744,0.0030022669017677472,0.012406312778762474,0.002832161005579148,0.00608024689536979,0.005479193934763414,0.0030022669017677472,0.002832161005579148,0.012406312778762474,0.008628878095570825,0.008528481213975155,3,0,0,True +xiiotid,0.75,0.5,accuracy,0.9132536187826642,0.9138984591386161,0.9345763398861396,0.9143652007295908,0.9326111121346672,0.9373522240850943,0.0011115819469266208,0.01871265299605107,0.0027758841989546923,0.0075333730473107945,0.009717237293889463,0.0027758841989546923,0.0011115819469266208,0.01871265299605107,0.012128815595858657,0.012136684529934048,3,0,0,True +5gnidd,0.25,0.0,rare_class_macro_f1,0.8649030079014913,0.8148479924290983,0.8470722146704726,0.8598927442995293,0.8138885616043564,0.8499254383428447,-0.005010263601961995,-0.0009594308247419248,0.0028532236723721738,-0.0010388235847772487,0.003932344775851092,-0.0009594308247419248,-0.005010263601961995,0.0028532236723721738,0.02537006851997817,0.024201888695409735,1,2,0,False +5gnidd,0.25,0.0,rare_class_recall,0.9854440969271964,0.9723332887819381,0.9238897222901731,0.9852344223763528,0.9726587864975359,0.9257555179886866,-0.00020967455084353048,0.00032549771559775564,0.0018657956985135282,0.0006605396210892511,0.001077536077269848,0.00032549771559775564,-0.00020967455084353048,0.0018657956985135282,0.032423277290702865,0.03134699156678076,2,1,0,False +5gnidd,0.25,0.0,macro_f1,0.8350430568226094,0.8083728869633723,0.8148003580492102,0.8296799384783186,0.8029871664481885,0.8172541498613525,-0.005363118344290818,-0.005385720515183823,0.0024537918121422253,-0.0027650156824441385,0.004519633996652314,-0.005363118344290818,-0.005385720515183823,0.0024537918121422253,0.013918675694370832,0.01335696520619808,1,2,0,False +5gnidd,0.25,0.0,weighted_f1,0.7180410173151626,0.7163267008884987,0.6819385320735543,0.7174035576457394,0.7123024521167044,0.6836842063399526,-0.0006374596694231904,-0.004024248771794303,0.0017456742663983782,-0.0009720113916063718,0.0028994735052727187,-0.0006374596694231904,-0.004024248771794303,0.0017456742663983782,0.02036694411653514,0.018175165847202895,1,2,0,False +5gnidd,0.25,0.0,accuracy,0.745280186859777,0.7412954350123573,0.7063291347454734,0.7448442903728559,0.737779477499928,0.7080521595381142,-0.0004358964869211057,-0.0035159575124292797,0.0017230247926407483,-0.0007429430689032124,0.0026329531140602156,-0.0004358964869211057,-0.0035159575124292797,0.0017230247926407483,0.02143091560670625,0.019524710435296355,1,2,0,False +5gnidd,0.5,1.0,rare_class_macro_f1,0.8719637151922778,0.9211362509007983,0.8485250688106042,0.8809620350206805,0.8400749080738759,0.8949144498720359,0.008998319828402712,-0.08106134282692246,0.046389381061431756,-0.008557880645695995,0.06551402053172613,0.008998319828402712,-0.08106134282692246,0.046389381061431756,0.0370578200850738,0.028500884123071776,2,1,0,False +5gnidd,0.5,1.0,rare_class_recall,0.9743136055202825,0.9685663953100676,0.9758591365273065,0.9743551890147765,0.9683408980660666,0.9586158518509788,4.1583494494012996e-05,-0.00022549724400100501,-0.01724328467632774,-0.0058090661419449114,0.009903224129001456,-0.00022549724400100501,-0.01724328467632774,4.1583494494012996e-05,0.0038428107281798614,0.0079422389145739,1,2,0,False +5gnidd,0.5,1.0,macro_f1,0.8368808854359121,0.8654374620417004,0.8248673149789701,0.8504696755820977,0.8159411532871907,0.8493777051701392,0.013588790146185636,-0.04949630875450961,0.024510390191169074,-0.0037990428057183,0.039949974008521794,0.013588790146185636,-0.04949630875450961,0.024510390191169074,0.02083962881650294,0.019627422368850796,2,1,0,False +5gnidd,0.5,1.0,weighted_f1,0.7087651452654392,0.7228023713469742,0.7246221247269671,0.7169500389469496,0.7201060072251736,0.7175295566399255,0.00818489368151043,-0.002696364121800632,-0.0070925680870416175,-0.0005346795091106064,0.007864786697422122,-0.002696364121800632,-0.0070925680870416175,0.00818489368151043,0.00867754781098361,0.0016799838366213199,1,2,0,False +5gnidd,0.5,1.0,accuracy,0.7344156725348203,0.7473774246742085,0.7501326194500302,0.7428375216406157,0.7440259564021269,0.7429074295677633,0.008421849105795354,-0.003351468272081659,-0.007225189882266836,-0.0007182696828510471,0.008149095556025167,-0.003351468272081659,-0.007225189882266836,0.008421849105795354,0.008392660852506539,0.0006668791219600918,1,2,0,False +5gnidd,0.75,0.5,rare_class_macro_f1,0.9055885946257963,0.8607580175603599,0.8733557623287002,0.9044472051910231,0.9057207431633723,0.8800449253698305,-0.0011413894347732656,0.0449627256030124,0.006689163041130319,0.01683683306978982,0.024670402532432648,0.006689163041130319,-0.0011413894347732656,0.0449627256030124,0.023120839413686988,0.014470318925337908,2,1,0,False +5gnidd,0.75,0.5,rare_class_recall,0.9612851237375019,0.9590357221091453,0.9633214115686995,0.9586662658643216,0.9597228673347585,0.960715099133502,-0.0026188578731802803,0.0006871452256131949,-0.0026063124351975997,-0.0015126750275882284,0.0019051105497581826,-0.0026063124351975997,-0.0026188578731802803,0.0006871452256131949,0.002143727670860224,0.001024585149761231,1,2,0,False +5gnidd,0.75,0.5,macro_f1,0.8571092007836155,0.8298666656506455,0.8372382392225877,0.8573089581866151,0.8599385529838828,0.842201195406295,0.00019975740299960076,0.03007188733323729,0.0049629561837073455,0.011744866973314746,0.016049354258377363,0.0049629561837073455,0.00019975740299960076,0.03007188733323729,0.014091078328488028,0.009572296127013734,3,0,0,True +5gnidd,0.75,0.5,weighted_f1,0.7210591954237406,0.7299504931440421,0.7271326093968319,0.7238852910350473,0.7301094421086752,0.7284075566369677,0.002826095611306645,0.00015894896463308772,0.0012749472401357709,0.001419997272025168,0.0013394765554233625,0.0012749472401357709,0.00015894896463308772,0.002826095611306645,0.004543897028843019,0.0032168140791904,3,0,0,True +5gnidd,0.75,0.5,accuracy,0.7470484461935134,0.7540721367563544,0.7517446140054364,0.7495075603367095,0.7543805540820061,0.7529330487669476,0.002459114143196106,0.00030841732565167757,0.0011884347615112034,0.0013186554101196624,0.0010812456997565073,0.0011884347615112034,0.00030841732565167757,0.002459114143196106,0.0035777922288362054,0.0025025090963709863,3,0,0,True diff --git a/artifacts/analysis/posthoc__paired_global_per_seed.csv b/artifacts/analysis/posthoc__paired_global_per_seed.csv new file mode 100644 index 0000000..880de18 --- /dev/null +++ b/artifacts/analysis/posthoc__paired_global_per_seed.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,selected_beta,metric,original,proposed,delta,manifest_content_hash,pairing_verified +xiiotid,0.25,0,0.0,rare_class_macro_f1,0.6398103625587044,0.6413622282749385,0.0015518657162341132,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True +xiiotid,0.25,0,0.0,rare_class_recall,0.8296012943628512,0.830700667573069,0.0010993732102178022,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True +xiiotid,0.25,0,0.0,macro_f1,0.6731025782780014,0.6745583013411162,0.001455723063114811,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True +xiiotid,0.25,0,0.0,weighted_f1,0.9054655196913645,0.9061557688476052,0.0006902491562407187,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True +xiiotid,0.25,0,0.0,accuracy,0.8737218342944525,0.8745509147521049,0.000829080457652398,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True +xiiotid,0.25,1,0.0,rare_class_macro_f1,0.5825215614975117,0.5805553678969027,-0.001966193600609012,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True +xiiotid,0.25,1,0.0,rare_class_recall,0.8266995877156649,0.8256994230374527,-0.0010001646782121787,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True +xiiotid,0.25,1,0.0,macro_f1,0.6669813754005415,0.6650385577782836,-0.0019428176222578974,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True +xiiotid,0.25,1,0.0,weighted_f1,0.9020747305215218,0.899863925003673,-0.0022108055178488017,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True +xiiotid,0.25,1,0.0,accuracy,0.8673409854388906,0.8634903673133494,-0.0038506181255412164,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True +xiiotid,0.25,2,0.0,rare_class_macro_f1,0.6760389917828752,0.6744891962601014,-0.0015497955227737892,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True +xiiotid,0.25,2,0.0,rare_class_recall,0.8095946724338723,0.807370314447251,-0.0022243579866212393,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True +xiiotid,0.25,2,0.0,macro_f1,0.7059686496708458,0.7044332575239617,-0.0015353921468841403,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True +xiiotid,0.25,2,0.0,weighted_f1,0.9154163421325315,0.9146272396181951,-0.0007891025143363928,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True +xiiotid,0.25,2,0.0,accuracy,0.8852306993140127,0.8840024319693425,-0.0012282673446701864,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True +xiiotid,0.5,0,0.75,rare_class_macro_f1,0.6096198579015474,0.6361887569975536,0.026568899096006238,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +xiiotid,0.5,0,0.75,rare_class_recall,0.8714872155034664,0.8575434903523123,-0.013943725151154163,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +xiiotid,0.5,0,0.75,macro_f1,0.7077011542142579,0.7262984173916737,0.018597263177415724,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +xiiotid,0.5,0,0.75,weighted_f1,0.9175853183547894,0.9222775427473946,0.004692224392605193,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +xiiotid,0.5,0,0.75,accuracy,0.8904324115186911,0.8886698478790893,-0.0017625636396018107,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +xiiotid,0.5,1,0.75,rare_class_macro_f1,0.655665402810942,0.6719373483509694,0.01627194554002742,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +xiiotid,0.5,1,0.75,rare_class_recall,0.8893650106408222,0.8518834947182458,-0.037481515922576425,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +xiiotid,0.5,1,0.75,macro_f1,0.7091760460445067,0.7290145924987934,0.019838546454286732,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +xiiotid,0.5,1,0.75,weighted_f1,0.929132450103975,0.9447335103607959,0.015601060256820842,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +xiiotid,0.5,1,0.75,accuracy,0.9008849666218349,0.9254933028723031,0.02460833625046821,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +xiiotid,0.5,2,0.75,rare_class_macro_f1,0.662274505162012,0.6883814197743454,0.026106914612333387,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +xiiotid,0.5,2,0.75,rare_class_recall,0.8700179539413673,0.878321192150252,0.008303238208884678,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +xiiotid,0.5,2,0.75,macro_f1,0.7100168446133323,0.7381943369820925,0.028177492368760215,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +xiiotid,0.5,2,0.75,weighted_f1,0.9274806689914715,0.9338838836432976,0.006403214651826028,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +xiiotid,0.5,2,0.75,accuracy,0.9070447273553561,0.9050119448999269,-0.0020327824554292606,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +xiiotid,0.75,0,0.5,rare_class_macro_f1,0.5215401253860467,0.5118888965076885,-0.00965122887835812,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True +xiiotid,0.75,0,0.5,rare_class_recall,0.8164397897152784,0.8038871133887995,-0.01255267632647894,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True +xiiotid,0.75,0,0.5,macro_f1,0.7424713724903063,0.741976732126316,-0.000494640363990273,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True +xiiotid,0.75,0,0.5,weighted_f1,0.9325633790408211,0.9355656459425888,0.0030022669017677472,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True +xiiotid,0.75,0,0.5,accuracy,0.9132536187826642,0.9143652007295908,0.0011115819469266208,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True +xiiotid,0.75,1,0.5,rare_class_macro_f1,0.6140607772418322,0.6241440493793726,0.010083272137540455,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True +xiiotid,0.75,1,0.5,rare_class_recall,0.8832376049795277,0.8767262170299326,-0.00651138794959516,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True +xiiotid,0.75,1,0.5,macro_f1,0.7473954752926397,0.7617524315559205,0.014356956263280818,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True +xiiotid,0.75,1,0.5,weighted_f1,0.9359372345770101,0.9483435473557725,0.012406312778762474,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True +xiiotid,0.75,1,0.5,accuracy,0.9138984591386161,0.9326111121346672,0.01871265299605107,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True +xiiotid,0.75,2,0.5,rare_class_macro_f1,0.6760680764285393,0.6797040888050662,0.003636012376526887,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True +xiiotid,0.75,2,0.5,rare_class_recall,0.8698349315628103,0.8656806653880937,-0.004154266174716592,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True +xiiotid,0.75,2,0.5,macro_f1,0.7656800124975965,0.7691786586117774,0.0034986461141808256,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True +xiiotid,0.75,2,0.5,weighted_f1,0.9489075724365953,0.9517397334421744,0.002832161005579148,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True +xiiotid,0.75,2,0.5,accuracy,0.9345763398861396,0.9373522240850943,0.0027758841989546923,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True +5gnidd,0.25,0,0.0,rare_class_macro_f1,0.8649030079014913,0.8598927442995293,-0.005010263601961995,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True +5gnidd,0.25,0,0.0,rare_class_recall,0.9854440969271964,0.9852344223763528,-0.00020967455084353048,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True +5gnidd,0.25,0,0.0,macro_f1,0.8350430568226094,0.8296799384783186,-0.005363118344290818,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True +5gnidd,0.25,0,0.0,weighted_f1,0.7180410173151626,0.7174035576457394,-0.0006374596694231904,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True +5gnidd,0.25,0,0.0,accuracy,0.745280186859777,0.7448442903728559,-0.0004358964869211057,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True +5gnidd,0.25,1,0.0,rare_class_macro_f1,0.8148479924290983,0.8138885616043564,-0.0009594308247419248,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True +5gnidd,0.25,1,0.0,rare_class_recall,0.9723332887819381,0.9726587864975359,0.00032549771559775564,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True +5gnidd,0.25,1,0.0,macro_f1,0.8083728869633723,0.8029871664481885,-0.005385720515183823,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True +5gnidd,0.25,1,0.0,weighted_f1,0.7163267008884987,0.7123024521167044,-0.004024248771794303,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True +5gnidd,0.25,1,0.0,accuracy,0.7412954350123573,0.737779477499928,-0.0035159575124292797,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True +5gnidd,0.25,2,0.0,rare_class_macro_f1,0.8470722146704726,0.8499254383428447,0.0028532236723721738,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True +5gnidd,0.25,2,0.0,rare_class_recall,0.9238897222901731,0.9257555179886866,0.0018657956985135282,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True +5gnidd,0.25,2,0.0,macro_f1,0.8148003580492102,0.8172541498613525,0.0024537918121422253,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True +5gnidd,0.25,2,0.0,weighted_f1,0.6819385320735543,0.6836842063399526,0.0017456742663983782,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True +5gnidd,0.25,2,0.0,accuracy,0.7063291347454734,0.7080521595381142,0.0017230247926407483,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True +5gnidd,0.5,0,1.0,rare_class_macro_f1,0.8719637151922778,0.8809620350206805,0.008998319828402712,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True +5gnidd,0.5,0,1.0,rare_class_recall,0.9743136055202825,0.9743551890147765,4.1583494494012996e-05,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True +5gnidd,0.5,0,1.0,macro_f1,0.8368808854359121,0.8504696755820977,0.013588790146185636,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True +5gnidd,0.5,0,1.0,weighted_f1,0.7087651452654392,0.7169500389469496,0.00818489368151043,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True +5gnidd,0.5,0,1.0,accuracy,0.7344156725348203,0.7428375216406157,0.008421849105795354,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True +5gnidd,0.5,1,1.0,rare_class_macro_f1,0.9211362509007983,0.8400749080738759,-0.08106134282692246,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True +5gnidd,0.5,1,1.0,rare_class_recall,0.9685663953100676,0.9683408980660666,-0.00022549724400100501,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True +5gnidd,0.5,1,1.0,macro_f1,0.8654374620417004,0.8159411532871907,-0.04949630875450961,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True +5gnidd,0.5,1,1.0,weighted_f1,0.7228023713469742,0.7201060072251736,-0.002696364121800632,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True +5gnidd,0.5,1,1.0,accuracy,0.7473774246742085,0.7440259564021269,-0.003351468272081659,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True +5gnidd,0.5,2,1.0,rare_class_macro_f1,0.8485250688106042,0.8949144498720359,0.046389381061431756,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True +5gnidd,0.5,2,1.0,rare_class_recall,0.9758591365273065,0.9586158518509788,-0.01724328467632774,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True +5gnidd,0.5,2,1.0,macro_f1,0.8248673149789701,0.8493777051701392,0.024510390191169074,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True +5gnidd,0.5,2,1.0,weighted_f1,0.7246221247269671,0.7175295566399255,-0.0070925680870416175,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True +5gnidd,0.5,2,1.0,accuracy,0.7501326194500302,0.7429074295677633,-0.007225189882266836,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True +5gnidd,0.75,0,0.5,rare_class_macro_f1,0.9055885946257963,0.9044472051910231,-0.0011413894347732656,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True +5gnidd,0.75,0,0.5,rare_class_recall,0.9612851237375019,0.9586662658643216,-0.0026188578731802803,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True +5gnidd,0.75,0,0.5,macro_f1,0.8571092007836155,0.8573089581866151,0.00019975740299960076,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True +5gnidd,0.75,0,0.5,weighted_f1,0.7210591954237406,0.7238852910350473,0.002826095611306645,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True +5gnidd,0.75,0,0.5,accuracy,0.7470484461935134,0.7495075603367095,0.002459114143196106,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True +5gnidd,0.75,1,0.5,rare_class_macro_f1,0.8607580175603599,0.9057207431633723,0.0449627256030124,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True +5gnidd,0.75,1,0.5,rare_class_recall,0.9590357221091453,0.9597228673347585,0.0006871452256131949,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True +5gnidd,0.75,1,0.5,macro_f1,0.8298666656506455,0.8599385529838828,0.03007188733323729,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True +5gnidd,0.75,1,0.5,weighted_f1,0.7299504931440421,0.7301094421086752,0.00015894896463308772,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True +5gnidd,0.75,1,0.5,accuracy,0.7540721367563544,0.7543805540820061,0.00030841732565167757,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True +5gnidd,0.75,2,0.5,rare_class_macro_f1,0.8733557623287002,0.8800449253698305,0.006689163041130319,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True +5gnidd,0.75,2,0.5,rare_class_recall,0.9633214115686995,0.960715099133502,-0.0026063124351975997,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True +5gnidd,0.75,2,0.5,macro_f1,0.8372382392225877,0.842201195406295,0.0049629561837073455,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True +5gnidd,0.75,2,0.5,weighted_f1,0.7271326093968319,0.7284075566369677,0.0012749472401357709,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True +5gnidd,0.75,2,0.5,accuracy,0.7517446140054364,0.7529330487669476,0.0011884347615112034,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True diff --git a/artifacts/analysis/posthoc__paired_test_results.csv b/artifacts/analysis/posthoc__paired_test_results.csv new file mode 100644 index 0000000..7fb5371 --- /dev/null +++ b/artifacts/analysis/posthoc__paired_test_results.csv @@ -0,0 +1,31 @@ +dataset,alpha,selected_beta,metric,n_pairs,deltas,signflip_exact_p_two_sided,wilcoxon_exact_p_two_sided,paired_t_ci95_low,paired_t_ci95_high,min_attainable_signflip_p,note +xiiotid,0.25,0.0,rare_class_macro_f1,3,0.001552|-0.001966|-0.001550,0.75,1.0,-0.005429858832977498,0.004120443228211705,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.25,0.0,rare_class_recall,3,0.001099|-0.001000|-0.002224,0.75,1.0,-0.004884136340091787,0.00346737003701471,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.25,0.0,macro_f1,3,0.001456|-0.001943|-0.001535,0.5,1.0,-0.005284100361706733,0.003935775891021916,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.25,0.0,weighted_f1,3,0.000690|-0.002211|-0.000789,0.5,1.0,-0.004373433090107915,0.002833660506144932,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.25,0.0,accuracy,3,0.000829|-0.003851|-0.001228,0.5,1.0,-0.0072432138310324955,0.004410010489326492,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.5,0.75,rare_class_macro_f1,3,0.026569|0.016272|0.026107,0.25,0.5,0.008534408575840131,0.03743076425640456,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.5,0.75,rare_class_recall,3,-0.013944|-0.037482|0.008303,0.5,1.0,-0.07124935135394075,0.042501349444043476,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.5,0.75,macro_f1,3,0.018597|0.019839|0.028177,0.25,0.5,0.009262275411828459,0.035146592588479984,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.5,0.75,weighted_f1,3,0.004692|0.015601|0.006403,0.25,0.5,-0.00567561757777198,0.02347328377860669,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.5,0.75,accuracy,3,-0.001763|0.024608|-0.002033,1.0,1.0,-0.031079202606995104,0.04495452937728653,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.75,0.5,rare_class_macro_f1,3,-0.009651|0.010083|0.003636,0.75,1.0,-0.02364154357009798,0.026353580660570797,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.75,0.5,rare_class_recall,3,-0.012553|-0.006511|-0.004154,0.25,0.5,-0.01850020894956627,0.003021321982372477,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.75,0.5,macro_f1,3,-0.000495|0.014357|0.003499,0.5,1.0,-0.013305329712970347,0.024879304388617927,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.75,0.5,weighted_f1,3,0.003002|0.012406|0.002832,0.25,0.5,-0.007530825385598827,0.019691319176338408,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +xiiotid,0.75,0.5,accuracy,3,0.001112|0.018713|0.002776,0.25,0.5,-0.01660558256426208,0.03167232865888367,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.25,0.0,rare_class_macro_f1,3,-0.005010|-0.000959|0.002853,0.75,1.0,-0.010807309536371811,0.008729662366817314,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.25,0.0,rare_class_recall,3,-0.000210|0.000325|0.001866,0.5,1.0,-0.002016208383762044,0.0033372876259405463,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.25,0.0,macro_f1,3,-0.005363|-0.005386|0.002454,0.5,1.0,-0.013992408934881363,0.008462377569993086,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.25,0.0,weighted_f1,3,-0.000637|-0.004024|0.001746,0.75,1.0,-0.008174702869020644,0.0062306800858079004,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.25,0.0,accuracy,3,-0.000436|-0.003516|0.001723,0.75,1.0,-0.007283561191671378,0.005797675053864953,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.5,1.0,rare_class_macro_f1,3,0.008998|-0.081061|0.046389,1.0,1.0,-0.17130372966859422,0.15418796837720225,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.5,1.0,rare_class_recall,3,0.000042|-0.000225|-0.017243,0.5,1.0,-0.0304100386644281,0.018791906380538275,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.5,1.0,macro_f1,3,0.013589|-0.049496|0.024510,1.0,1.0,-0.10304027980644635,0.09544219419500975,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.5,1.0,weighted_f1,3,0.008185|-0.002696|-0.007093,1.0,1.0,-0.020071892735646125,0.01900253371742491,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.5,1.0,accuracy,3,0.008422|-0.003351|-0.007225,1.0,1.0,-0.02096174526670406,0.019525205901001964,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.75,0.5,rare_class_macro_f1,3,-0.001141|0.044963|0.006689,0.5,1.0,-0.04444784421441203,0.07812151035399167,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.75,0.5,rare_class_recall,3,-0.002619|0.000687|-0.002606,0.5,1.0,-0.006245231988471588,0.0032198819332951312,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.75,0.5,macro_f1,3,0.000200|0.030072|0.004963,0.25,0.5,-0.028123939182216617,0.05161367312884611,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.75,0.5,weighted_f1,3,0.002826|0.000159|0.001275,0.25,0.5,-0.0019074469527278436,0.004747441496778179,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" +5gnidd,0.75,0.5,accuracy,3,0.002459|0.000308|0.001188,0.25,0.5,-0.0013673078078454953,0.00400461862808482,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast" diff --git a/artifacts/analysis/posthoc__per_class_summary.csv b/artifacts/analysis/posthoc__per_class_summary.csv new file mode 100644 index 0000000..f9d8a20 --- /dev/null +++ b/artifacts/analysis/posthoc__per_class_summary.csv @@ -0,0 +1,19 @@ +dataset,class_name,n_obs,mean_delta_f1,median_delta_f1,min_delta_f1,max_delta_f1,n_positive,n_negative +xiiotid,Tampering,9,0.05199619640670891,0.004580967510456113,-0.005430499483001561,0.214955968364532,6,3 +xiiotid,Reconnaissance,9,0.02952063666312104,0.008242346598957062,-0.0003622961256807278,0.10238198044760449,8,1 +5gnidd,ICMPFlood,9,0.026821108337293526,0.010408977770148542,-0.46986271362984083,0.3772516575720237,5,4 +5gnidd,TCPConnectScan,9,0.014211458390848499,0.000706524723936286,-0.019433003409767724,0.05772198907204773,6,3 +xiiotid,Lateral_movement,9,0.010881583240329038,0.0017846016589943003,-0.014168138453098611,0.07607557487760874,6,3 +xiiotid,Weaponization,9,0.009113660779203083,0.007233534576793765,-0.004643526037137802,0.03336802894144031,8,1 +xiiotid,Exploitation,9,0.006734945561392644,6.591764883166762e-05,-0.008223665637908967,0.036914135633034896,6,3 +5gnidd,Benign,9,0.0009395838996870687,0.00028927780247300916,-0.0012807529813573248,0.004784809629670872,5,4 +5gnidd,SYNScan,9,0.0007028996039083962,0.00012358980722271262,-0.0027240713350633605,0.0033494638084979877,6,2 +xiiotid,RDOS,9,0.00011273464532779442,0.0,-0.00020998786240000733,0.0008246202062063279,4,4 +5gnidd,UDPFlood,9,-5.0275925956781244e-05,-5.101815123376152e-05,-0.0004926348932754454,0.0004015231846284939,3,6 +5gnidd,UDPScan,9,-0.00029387524907019554,-0.0009226463398589457,-0.0038642666069870213,0.004913668121620529,3,6 +5gnidd,HTTPFlood,9,-0.0008211332125179031,0.0007952922021617903,-0.0344690095854302,0.03529511546161013,6,3 +xiiotid,Normal,9,-0.002240793040637389,-0.00027963176745748886,-0.0315329514150966,0.012717262317857747,4,5 +xiiotid,Exfiltration,9,-0.006129465975398516,0.0,-0.03964725823094428,0.004487347741842007,3,3 +5gnidd,SlowrateDoS,9,-0.006597630438252566,0.0017380510468695265,-0.047165133257141556,0.028229267253973345,5,4 +xiiotid,C&C,9,-0.018037720972140073,-0.005217928136442729,-0.16416853359506542,0.0321417804096471,4,5 +5gnidd,SYNFlood,9,-0.019369709950482903,-0.006397152956185925,-0.06835732738470357,0.00727579075336604,2,7 diff --git a/artifacts/analysis/posthoc__per_client_summary.csv b/artifacts/analysis/posthoc__per_client_summary.csv new file mode 100644 index 0000000..75f9266 --- /dev/null +++ b/artifacts/analysis/posthoc__per_client_summary.csv @@ -0,0 +1,7 @@ +dataset,alpha,selected_beta,n_client_seed_obs,n_improved_gt_0.02,n_degraded_lt_-0.02,n_unchanged,mean_delta,median_delta,min_delta,max_delta,std_sample_within_condition +5gnidd,0.25,0.0,30,1,0,29,0.002111576617816057,0.0,-0.019933554817275767,0.07142857142857151,0.01470740176243374 +5gnidd,0.5,1.0,30,15,6,9,0.03903400685752252,0.02024848480689545,-0.17567567567567566,0.4040650406504065,0.11851570462408138 +5gnidd,0.75,0.5,30,8,1,21,0.02071412212202052,0.004477101314292853,-0.03633656881678371,0.16839803756061722,0.043102741088679355 +xiiotid,0.25,0.0,30,1,1,28,-0.0015178066042212891,0.0,-0.026277262664796908,0.02494573827906854,0.00807984351701968 +xiiotid,0.5,0.75,30,12,1,17,0.03024920134564162,0.012286627106224551,-0.2565947651121153,0.3235930735930736,0.08670494898387493 +xiiotid,0.75,0.5,30,8,2,20,0.01167502563325023,0.010269879504053326,-0.0697526762643042,0.06759906759906764,0.026808060695173577 diff --git a/artifacts/analysis/posthoc__selection_stability.csv b/artifacts/analysis/posthoc__selection_stability.csv new file mode 100644 index 0000000..17db8b8 --- /dev/null +++ b/artifacts/analysis/posthoc__selection_stability.csv @@ -0,0 +1,7 @@ +dataset,alpha,pooled_selected_beta,per_seed_beta_s0,per_seed_beta_s1,per_seed_beta_s2,n_seeds_agree_pooled,pooled_score_selected,second_best_beta,second_best_score,margin_best_minus_second,candidate_score_range,test_used_for_selection +xiiotid,0.25,0.0,0.75,0.5,0.0,1,0.5917038146634539,0.25,0.5886906256417319,0.0030131890217219848,0.059605114021112016,False +xiiotid,0.5,0.75,0.5,1.0,0.75,1,0.5793667895728949,1.0,0.5734790834215032,0.0058877061513916695,0.03500610197920917,False +xiiotid,0.75,0.5,0.5,0.5,0.0,2,0.47674948263663786,0.25,0.47436809609801456,0.002381386538623298,0.008144253663347922,False +5gnidd,0.25,0.0,0.25,0.0,0.0,2,0.7614534210908004,0.25,0.7492858468825279,0.012167574208272525,0.039218114350402566,False +5gnidd,0.5,1.0,1.0,0.0,1.0,2,0.6923695590578484,0.75,0.6883680316141424,0.004001527443705966,0.04729182197574977,False +5gnidd,0.75,0.5,0.5,1.0,1.0,1,0.8090814343293602,1.0,0.8043064619946296,0.004774972334730565,0.03098780223355102,False diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_effect.csv b/artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_effect.csv new file mode 100644 index 0000000..07cf04f --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_effect.csv @@ -0,0 +1,46 @@ +class_id,class_name,beta,n_contributors,support_min,support_max,support_mean,prototype_norm,reference_norm_beta0,displacement_from_beta0,relative_displacement +0,C&C,0.0,8,2,687,250.5,4.995807261779628,4.995807261779628,0.0,0.0 +0,C&C,0.25,8,2,687,250.5,4.857134771566648,4.995807261779628,0.7971996511065712,0.1595737404053873 +0,C&C,0.5,8,2,687,250.5,4.906583708196013,4.995807261779628,1.2665743125302988,0.2535274573581356 +0,C&C,0.75,8,2,687,250.5,5.010855437469484,4.995807261779628,1.56068016232047,0.3123979930651924 +0,C&C,1.0,8,2,687,250.5,5.121849269773383,4.995807261779628,1.7701252868773527,0.35432217339921773 +1,Exfiltration,0.0,10,8,6273,1549.4,7.157109420154553,7.157109420154553,0.0,0.0 +1,Exfiltration,0.25,10,8,6273,1549.4,7.739024271072487,7.157109420154553,1.112938631376158,0.1555011340531 +1,Exfiltration,0.5,10,8,6273,1549.4,8.136996752575591,7.157109420154553,1.792423075103891,0.2504395238189868 +1,Exfiltration,0.75,10,8,6273,1549.4,8.4002894954015,7.157109420154553,2.2364541806817226,0.312480087894678 +1,Exfiltration,1.0,10,8,6273,1549.4,8.580132867813264,7.157109420154553,2.5469976586314798,0.35586959889967407 +2,Exploitation,0.0,9,1,491,88.11111111111111,5.992134154687249,5.992134154687249,0.0,0.0 +2,Exploitation,0.25,9,1,491,88.11111111111111,5.852057025355164,5.992134154687249,0.6403508338020284,0.10686523653698982 +2,Exploitation,0.5,9,1,491,88.11111111111111,5.649833686058244,5.992134154687249,1.3058952653326283,0.2179349179475752 +2,Exploitation,0.75,9,1,491,88.11111111111111,5.462810593200537,5.992134154687249,2.0362822833565866,0.3398258835316192 +2,Exploitation,1.0,9,1,491,88.11111111111111,5.361136450101408,5.992134154687249,2.7586985908860333,0.46038665351443886 +3,Lateral_movement,0.0,10,49,8361,2211.7,5.836739012783497,5.836739012783497,0.0,0.0 +3,Lateral_movement,0.25,10,49,8361,2211.7,6.244285935865453,5.836739012783497,0.7731353833893619,0.13246015997906674 +3,Lateral_movement,0.5,10,49,8361,2211.7,6.616077783217527,5.836739012783497,1.3883187562912933,0.2378586318919911 +3,Lateral_movement,0.75,10,49,8361,2211.7,6.909367469065839,5.836739012783497,1.8469497276568052,0.3164352087032942 +3,Lateral_movement,1.0,10,49,8361,2211.7,7.129359871034012,5.836739012783497,2.18587302323185,0.37450244364951035 +4,Normal,0.0,10,260,89253,29277.7,5.744022110217372,5.744022110217372,0.0,0.0 +4,Normal,0.25,10,260,89253,29277.7,6.310406949046281,5.744022110217372,0.7219321390806914,0.1256840808109931 +4,Normal,0.5,10,260,89253,29277.7,6.65485903802621,5.744022110217372,1.1590902280700377,0.20179069749892972 +4,Normal,0.75,10,260,89253,29277.7,6.856645071712252,5.744022110217372,1.4245372318857714,0.24800343810512637 +4,Normal,1.0,10,260,89253,29277.7,6.981584206588257,5.744022110217372,1.5984450234076382,0.2782797476639845 +5,RDOS,0.0,10,6,47276,9888.3,7.606714687573022,7.606714687573022,0.0,0.0 +5,RDOS,0.25,10,6,47276,9888.3,8.606801712435749,7.606714687573022,1.6366469943162223,0.21515819398221792 +5,RDOS,0.5,10,6,47276,9888.3,9.183021988100103,7.606714687573022,2.444982027217194,0.3214241795096542 +5,RDOS,0.75,10,6,47276,9888.3,9.572100709915908,7.606714687573022,2.932638911884029,0.38553291826168246 +5,RDOS,1.0,10,6,47276,9888.3,9.877736968852913,7.606714687573022,3.2852506457034076,0.43188824356334454 +6,Reconnaissance,0.0,10,1842,23967,8916.1,6.600587294730691,6.600587294730691,0.0,0.0 +6,Reconnaissance,0.25,10,1842,23967,8916.1,6.619642701782661,6.600587294730691,0.19750804124277122,0.02992279814259008 +6,Reconnaissance,0.5,10,1842,23967,8916.1,6.6531594178920175,6.600587294730691,0.37937859322083467,0.057476490542545514 +6,Reconnaissance,0.75,10,1842,23967,8916.1,6.698692451193534,6.600587294730691,0.5470003591054624,0.08287146804984122 +6,Reconnaissance,1.0,10,1842,23967,8916.1,6.75347500969919,6.600587294730691,0.7025557661555466,0.10643837203947037 +7,Tampering,0.0,9,8,1228,398.44444444444446,5.065268302764301,5.065268302764301,0.0,0.0 +7,Tampering,0.25,9,8,1228,398.44444444444446,5.155842974383021,5.065268302764301,0.5957873490790528,0.11762207122452131 +7,Tampering,0.5,9,8,1228,398.44444444444446,5.314632895746181,5.065268302764301,1.0369597200237384,0.20471960378837814 +7,Tampering,0.75,9,8,1228,398.44444444444446,5.475188209465522,5.065268302764301,1.3704230949002763,0.2705529130909781 +7,Tampering,1.0,9,8,1228,398.44444444444446,5.614041117925036,5.065268302764301,1.6272957425211183,0.32126545826467745 +8,Weaponization,0.0,10,917,11526,4509.4,6.991110809530167,6.991110809530167,0.0,0.0 +8,Weaponization,0.25,10,917,11526,4509.4,7.0769252730146075,6.991110809530167,0.36048411330770536,0.05156320978581821 +8,Weaponization,0.5,10,917,11526,4509.4,7.163738177542325,6.991110809530167,0.695561899244867,0.0994923293586891 +8,Weaponization,0.75,10,917,11526,4509.4,7.238525120169953,6.991110809530167,0.9758406496021348,0.13958306143165186 +8,Weaponization,1.0,10,917,11526,4509.4,7.292875462963678,6.991110809530167,1.189062205569606,0.17008201385517965 diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_weights.csv b/artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_weights.csv new file mode 100644 index 0000000..2d4c5e3 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot__prototype_weights.csv @@ -0,0 +1,431 @@ +class_id,class_name,beta,client_id,N_ij,weight +0,C&C,0.0,0,2.0,0.125 +0,C&C,0.0,1,3.0,0.125 +0,C&C,0.0,3,687.0,0.125 +0,C&C,0.0,4,284.0,0.125 +0,C&C,0.0,5,599.0,0.125 +0,C&C,0.0,7,82.0,0.125 +0,C&C,0.0,8,236.0,0.125 +0,C&C,0.0,9,111.0,0.125 +0,C&C,0.25,0,2.0,0.044287795667630386 +0,C&C,0.25,1,3.0,0.049012502728748776 +0,C&C,0.25,3,687.0,0.19066270493474782 +0,C&C,0.25,4,284.0,0.15288192794120894 +0,C&C,0.25,5,599.0,0.1842397227452447 +0,C&C,0.25,7,82.0,0.11206758905036181 +0,C&C,0.25,8,236.0,0.145966961117219 +0,C&C,0.25,9,111.0,0.12088079581483864 +0,C&C,0.5,0,2.0,0.013387476143622101 +0,C&C,0.5,1,3.0,0.016396242747778414 +0,C&C,0.5,3,687.0,0.24812017604357262 +0,C&C,0.5,4,284.0,0.15953018988404427 +0,C&C,0.5,5,599.0,0.23168457650449506 +0,C&C,0.5,7,82.0,0.08572167297328072 +0,C&C,0.5,8,236.0,0.14542521467934644 +0,C&C,0.5,9,111.0,0.09973445102386046 +0,C&C,0.75,0,2.0,0.003736734114884092 +0,C&C,0.75,1,3.0,0.005064780649749788 +0,C&C,0.75,3,687.0,0.2981516746539658 +0,C&C,0.75,4,284.0,0.15371224135216596 +0,C&C,0.75,5,599.0,0.2690232512634536 +0,C&C,0.75,7,82.0,0.06054525177359796 +0,C&C,0.75,8,236.0,0.1337838538937557 +0,C&C,0.75,9,111.0,0.07598221229842717 +0,C&C,1.0,0,2.0,0.000998003992015968 +0,C&C,1.0,1,3.0,0.0014970059880239522 +0,C&C,1.0,3,687.0,0.34281437125748504 +0,C&C,1.0,4,284.0,0.14171656686626746 +0,C&C,1.0,5,599.0,0.29890219560878245 +0,C&C,1.0,7,82.0,0.04091816367265469 +0,C&C,1.0,8,236.0,0.11776447105788423 +0,C&C,1.0,9,111.0,0.05538922155688623 +1,Exfiltration,0.0,0,627.0,0.1 +1,Exfiltration,0.0,1,199.0,0.1 +1,Exfiltration,0.0,2,1006.0,0.1 +1,Exfiltration,0.0,3,916.0,0.1 +1,Exfiltration,0.0,4,6273.0,0.1 +1,Exfiltration,0.0,5,2759.0,0.1 +1,Exfiltration,0.0,6,2518.0,0.1 +1,Exfiltration,0.0,7,8.0,0.1 +1,Exfiltration,0.0,8,1113.0,0.1 +1,Exfiltration,0.0,9,75.0,0.1 +1,Exfiltration,0.25,0,627.0,0.0934897597942656 +1,Exfiltration,0.25,1,199.0,0.07017144629849609 +1,Exfiltration,0.25,2,1006.0,0.1052196018044885 +1,Exfiltration,0.25,3,916.0,0.1027829378876304 +1,Exfiltration,0.25,4,6273.0,0.16627079779754889 +1,Exfiltration,0.25,5,2759.0,0.13540511920488763 +1,Exfiltration,0.25,6,2518.0,0.1323460823920462 +1,Exfiltration,0.25,7,8.0,0.03142097487762249 +1,Exfiltration,0.25,8,1113.0,0.10791230019790968 +1,Exfiltration,0.25,9,75.0,0.054980979745104526 +1,Exfiltration,0.5,0,627.0,0.07636740343228143 +1,Exfiltration,0.5,1,199.0,0.04302301007227493 +1,Exfiltration,0.5,2,1006.0,0.09673268538801975 +1,Exfiltration,0.5,3,916.0,0.09230431226396064 +1,Exfiltration,0.5,4,6273.0,0.24155270083748484 +1,Exfiltration,0.5,5,2759.0,0.16019542325406966 +1,Exfiltration,0.5,6,2518.0,0.15303899917321462 +1,Exfiltration,0.5,7,8.0,0.008626194525229437 +1,Exfiltration,0.5,8,1113.0,0.10174705229246225 +1,Exfiltration,0.5,9,75.0,0.02641221876100239 +1,Exfiltration,0.75,0,627.0,0.05728357105623868 +1,Exfiltration,0.75,1,199.0,0.0242225151235567 +1,Exfiltration,0.75,2,1006.0,0.0816634858292164 +1,Exfiltration,0.75,3,916.0,0.07612039507683922 +1,Exfiltration,0.75,4,6273.0,0.32224478228192666 +1,Exfiltration,0.75,5,2759.0,0.1740376473264397 +1,Exfiltration,0.75,6,2518.0,0.1625066818573433 +1,Exfiltration,0.75,7,8.0,0.002174687802925644 +1,Exfiltration,0.75,8,1113.0,0.088094907051665 +1,Exfiltration,0.75,9,75.0,0.01165132659384861 +1,Exfiltration,1.0,0,627.0,0.04046727765586679 +1,Exfiltration,1.0,1,199.0,0.012843681425067768 +1,Exfiltration,1.0,2,1006.0,0.06492835936491545 +1,Exfiltration,1.0,3,916.0,0.059119659222925 +1,Exfiltration,1.0,4,6273.0,0.40486639989673423 +1,Exfiltration,1.0,5,2759.0,0.17806892990835163 +1,Exfiltration,1.0,6,2518.0,0.16251452175035497 +1,Exfiltration,1.0,7,8.0,0.000516328901510262 +1,Exfiltration,1.0,8,1113.0,0.0718342584226152 +1,Exfiltration,1.0,9,75.0,0.004840583451658707 +2,Exploitation,0.0,0,56.0,0.1111111111111111 +2,Exploitation,0.0,2,50.0,0.1111111111111111 +2,Exploitation,0.0,3,86.0,0.1111111111111111 +2,Exploitation,0.0,4,46.0,0.1111111111111111 +2,Exploitation,0.0,5,8.0,0.1111111111111111 +2,Exploitation,0.0,6,9.0,0.1111111111111111 +2,Exploitation,0.0,7,46.0,0.1111111111111111 +2,Exploitation,0.0,8,1.0,0.1111111111111111 +2,Exploitation,0.0,9,491.0,0.1111111111111111 +2,Exploitation,0.25,0,56.0,0.1201406619147194 +2,Exploitation,0.25,2,50.0,0.11678458309217514 +2,Exploitation,0.25,3,86.0,0.13374194241224655 +2,Exploitation,0.25,4,46.0,0.1143753593414062 +2,Exploitation,0.25,5,8.0,0.07386105563289265 +2,Exploitation,0.25,6,9.0,0.07606828780346402 +2,Exploitation,0.25,7,46.0,0.1143753593414062 +2,Exploitation,0.25,8,1.0,0.04391804644012388 +2,Exploitation,0.25,9,491.0,0.2067347040215659 +2,Exploitation,0.5,0,56.0,0.11273514564925723 +2,Exploitation,0.5,2,50.0,0.10652469978734996 +2,Exploitation,0.5,3,86.0,0.13970583403505477 +2,Exploitation,0.5,4,46.0,0.10217490265031934 +2,Exploitation,0.5,5,8.0,0.04260987991493999 +2,Exploitation,0.5,6,9.0,0.0451946025500978 +2,Exploitation,0.5,7,46.0,0.10217490265031934 +2,Exploitation,0.5,8,1.0,0.015064867516699268 +2,Exploitation,0.5,9,491.0,0.3338151652459622 +2,Exploitation,0.75,0,56.0,0.09386084897081189 +2,Exploitation,0.75,2,50.0,0.08621264334249394 +2,Exploitation,0.75,3,86.0,0.1294843545480138 +2,Exploitation,0.75,4,46.0,0.0809863510336706 +2,Exploitation,0.75,5,8.0,0.02181026528528262 +2,Exploitation,0.75,6,9.0,0.02382458475320683 +2,Exploitation,0.75,7,46.0,0.0809863510336706 +2,Exploitation,0.75,8,1.0,0.004585043473531672 +2,Exploitation,0.75,9,491.0,0.4782495575593181 +2,Exploitation,1.0,0,56.0,0.07061790668348046 +2,Exploitation,1.0,2,50.0,0.06305170239596469 +2,Exploitation,1.0,3,86.0,0.10844892812105927 +2,Exploitation,1.0,4,46.0,0.058007566204287514 +2,Exploitation,1.0,5,8.0,0.01008827238335435 +2,Exploitation,1.0,6,9.0,0.011349306431273645 +2,Exploitation,1.0,7,46.0,0.058007566204287514 +2,Exploitation,1.0,8,1.0,0.0012610340479192938 +2,Exploitation,1.0,9,491.0,0.6191677175283733 +3,Lateral_movement,0.0,0,2550.0,0.1 +3,Lateral_movement,0.0,1,3695.0,0.1 +3,Lateral_movement,0.0,2,818.0,0.1 +3,Lateral_movement,0.0,3,1239.0,0.1 +3,Lateral_movement,0.0,4,242.0,0.1 +3,Lateral_movement,0.0,5,49.0,0.1 +3,Lateral_movement,0.0,6,8361.0,0.1 +3,Lateral_movement,0.0,7,4589.0,0.1 +3,Lateral_movement,0.0,8,396.0,0.1 +3,Lateral_movement,0.0,9,178.0,0.1 +3,Lateral_movement,0.25,0,2550.0,0.1211003085961615 +3,Lateral_movement,0.25,1,3695.0,0.13286598619551734 +3,Lateral_movement,0.25,2,818.0,0.0911377978662085 +3,Lateral_movement,0.25,3,1239.0,0.10110625663303405 +3,Lateral_movement,0.25,4,242.0,0.06721466620617328 +3,Lateral_movement,0.25,5,49.0,0.04508781866114288 +3,Lateral_movement,0.25,6,8361.0,0.16295765280822522 +3,Lateral_movement,0.25,7,4589.0,0.14026190050545512 +3,Lateral_movement,0.25,8,396.0,0.07602107405963052 +3,Lateral_movement,0.25,9,178.0,0.06224653846845163 +3,Lateral_movement,0.5,0,2550.0,0.12946005533478547 +3,Lateral_movement,0.5,1,3695.0,0.15583783980228746 +3,Lateral_movement,0.5,2,818.0,0.07332335863139469 +3,Lateral_movement,0.5,3,1239.0,0.09024047056073208 +3,Lateral_movement,0.5,4,242.0,0.03988167417574212 +3,Lateral_movement,0.5,5,49.0,0.017945837798470695 +3,Lateral_movement,0.5,6,8361.0,0.234420079995533 +3,Lateral_movement,0.5,7,4589.0,0.173669969070608 +3,Lateral_movement,0.5,8,396.0,0.05101680902237561 +3,Lateral_movement,0.5,9,178.0,0.034203905608070934 +3,Lateral_movement,0.75,0,2550.0,0.12618171141712917 +3,Lateral_movement,0.75,1,3695.0,0.16664876909744766 +3,Lateral_movement,0.75,2,818.0,0.05378439105526583 +3,Lateral_movement,0.75,3,1239.0,0.07343358958802436 +3,Lateral_movement,0.75,4,242.0,0.02157509927076969 +3,Lateral_movement,0.75,5,49.0,0.006512359071280791 +3,Lateral_movement,0.75,6,8361.0,0.3074573967975947 +3,Lateral_movement,0.75,7,4589.0,0.19605585719761837 +3,Lateral_movement,0.75,8,396.0,0.031214951724883873 +3,Lateral_movement,0.75,9,178.0,0.017135874779985513 +3,Lateral_movement,1.0,0,2550.0,0.11529592621060722 +3,Lateral_movement,1.0,1,3695.0,0.16706605778360537 +3,Lateral_movement,1.0,2,818.0,0.0369851245648144 +3,Lateral_movement,1.0,3,1239.0,0.056020255911742096 +3,Lateral_movement,1.0,4,242.0,0.010941809467830176 +3,Lateral_movement,1.0,5,49.0,0.0022154903467920604 +3,Lateral_movement,1.0,6,8361.0,0.3780349957046616 +3,Lateral_movement,1.0,7,4589.0,0.20748745309038297 +3,Lateral_movement,1.0,8,396.0,0.01790477912917665 +3,Lateral_movement,1.0,9,178.0,0.008048107790387485 +4,Normal,0.0,0,89253.0,0.1 +4,Normal,0.0,1,23474.0,0.1 +4,Normal,0.0,2,60061.0,0.1 +4,Normal,0.0,3,69735.0,0.1 +4,Normal,0.0,4,18842.0,0.1 +4,Normal,0.0,5,1776.0,0.1 +4,Normal,0.0,6,9668.0,0.1 +4,Normal,0.0,7,17477.0,0.1 +4,Normal,0.0,8,260.0,0.1 +4,Normal,0.0,9,2231.0,0.1 +4,Normal,0.25,0,89253.0,0.15421903925662245 +4,Normal,0.25,1,23474.0,0.11044067762482088 +4,Normal,0.25,2,60061.0,0.1396787781299011 +4,Normal,0.25,3,69735.0,0.14499231663017753 +4,Normal,0.25,4,18842.0,0.10453555695398971 +4,Normal,0.25,5,1776.0,0.05792193046937319 +4,Normal,0.25,6,9668.0,0.0884741600033457 +4,Normal,0.25,7,17477.0,0.10258857779473632 +4,Normal,0.25,8,260.0,0.03582825236264257 +4,Normal,0.25,9,2231.0,0.06132071077439059 +4,Normal,0.5,0,89253.0,0.2082761388854636 +4,Normal,0.5,1,23474.0,0.10681239588140377 +4,Normal,0.5,2,60061.0,0.17085369909845277 +4,Normal,0.5,3,69735.0,0.1840998816096602 +4,Normal,0.5,4,18842.0,0.09569551992004474 +4,Normal,0.5,5,1776.0,0.029379850890899332 +4,Normal,0.5,6,9668.0,0.06854825876091125 +4,Normal,0.5,7,17477.0,0.09216404975621068 +4,Normal,0.5,8,260.0,0.011241254508049511 +4,Normal,0.5,9,2231.0,0.03292895068890413 +4,Normal,0.75,0,89253.0,0.2587802581458785 +4,Normal,0.75,1,23474.0,0.09503954532267954 +4,Normal,0.75,2,60061.0,0.19226863535359762 +4,Normal,0.75,3,69735.0,0.2150562797473768 +4,Normal,0.75,4,18842.0,0.08059521800192708 +4,Normal,0.75,5,1776.0,0.01371027752247949 +4,Normal,0.75,6,9668.0,0.04886146746290477 +4,Normal,0.75,7,17477.0,0.0761753028341714 +4,Normal,0.75,8,260.0,0.003244845594082718 +4,Normal,0.75,9,2231.0,0.016268170014902095 +4,Normal,1.0,0,89253.0,0.3048497662043125 +4,Normal,1.0,1,23474.0,0.08017706308897216 +4,Normal,1.0,2,60061.0,0.20514248045440728 +4,Normal,1.0,3,69735.0,0.23818469346977392 +4,Normal,1.0,4,18842.0,0.06435614819470109 +4,Normal,1.0,5,1776.0,0.0060660502703422745 +4,Normal,1.0,6,9668.0,0.033021719602291166 +4,Normal,1.0,7,17477.0,0.059693896720029234 +4,Normal,1.0,8,260.0,0.0008880478999374951 +4,Normal,1.0,9,2231.0,0.007620134095232891 +5,RDOS,0.0,0,4878.0,0.1 +5,RDOS,0.0,1,95.0,0.1 +5,RDOS,0.0,2,2816.0,0.1 +5,RDOS,0.0,3,2508.0,0.1 +5,RDOS,0.0,4,341.0,0.1 +5,RDOS,0.0,5,6.0,0.1 +5,RDOS,0.0,6,47276.0,0.1 +5,RDOS,0.0,7,20791.0,0.1 +5,RDOS,0.0,8,13932.0,0.1 +5,RDOS,0.0,9,6240.0,0.1 +5,RDOS,0.25,0,4878.0,0.10685785998076136 +5,RDOS,0.25,1,95.0,0.03991874919463753 +5,RDOS,0.25,2,2816.0,0.09314385020941883 +5,RDOS,0.25,3,2508.0,0.09048527413183693 +5,RDOS,0.25,4,341.0,0.05494581482251136 +5,RDOS,0.25,5,6.0,0.02001169319705181 +5,RDOS,0.25,6,47276.0,0.18854117287660088 +5,RDOS,0.25,7,20791.0,0.15353764253249236 +5,RDOS,0.25,8,13932.0,0.13891507888064297 +5,RDOS,0.25,9,6240.0,0.11364286417404601 +5,RDOS,0.5,0,4878.0,0.09162099667837655 +5,RDOS,0.5,1,95.0,0.012786035811850421 +5,RDOS,0.5,2,2816.0,0.06961301423833205 +5,RDOS,0.5,3,2508.0,0.06569584150494404 +5,RDOS,0.5,4,341.0,0.024224303741585254 +5,RDOS,0.5,5,6.0,0.0032132886428106787 +5,RDOS,0.5,6,47276.0,0.28522952313055794 +5,RDOS,0.5,7,20791.0,0.18915237691114564 +5,RDOS,0.5,8,13932.0,0.15483917430831698 +5,RDOS,0.5,9,6240.0,0.1036254450320805 +5,RDOS,0.75,0,4878.0,0.06981690248408114 +5,RDOS,0.75,1,95.0,0.0036397531776229367 +5,RDOS,0.75,2,2816.0,0.046238497959526094 +5,RDOS,0.75,3,2508.0,0.04239111882479618 +5,RDOS,0.75,4,341.0,0.00949172209437759 +5,RDOS,0.75,5,6.0,0.00045855630336426534 +5,RDOS,0.75,6,47276.0,0.3834950590544899 +5,RDOS,0.75,7,20791.0,0.2071027013699195 +5,RDOS,0.75,8,13932.0,0.1533872740014106 +5,RDOS,0.75,9,6240.0,0.08397841473041179 +5,RDOS,1.0,0,4878.0,0.049331027578046784 +5,RDOS,1.0,1,95.0,0.0009607313693961551 +5,RDOS,1.0,2,2816.0,0.02847810038125866 +5,RDOS,1.0,3,2508.0,0.025363308152058492 +5,RDOS,1.0,4,341.0,0.003448519968043041 +5,RDOS,1.0,5,6.0,6.067777069870453e-05 +5,RDOS,1.0,6,47276.0,0.4781003812586592 +5,RDOS,1.0,7,20791.0,0.21025858843279432 +5,RDOS,1.0,8,13932.0,0.1408937835623919 +5,RDOS,1.0,9,6240.0,0.06310488152665271 +6,Reconnaissance,0.0,0,12996.0,0.1 +6,Reconnaissance,0.0,1,5840.0,0.1 +6,Reconnaissance,0.0,2,1842.0,0.1 +6,Reconnaissance,0.0,3,14852.0,0.1 +6,Reconnaissance,0.0,4,23967.0,0.1 +6,Reconnaissance,0.0,5,9632.0,0.1 +6,Reconnaissance,0.0,6,4475.0,0.1 +6,Reconnaissance,0.0,7,2025.0,0.1 +6,Reconnaissance,0.0,8,6536.0,0.1 +6,Reconnaissance,0.0,9,6996.0,0.1 +6,Reconnaissance,0.25,0,12996.0,0.11557426335872803 +6,Reconnaissance,0.25,1,5840.0,0.09462631332339358 +6,Reconnaissance,0.25,2,1842.0,0.07091385823558452 +6,Reconnaissance,0.25,3,14852.0,0.11949643350422161 +6,Reconnaissance,0.25,4,23967.0,0.13468279712303718 +6,Reconnaissance,0.25,5,9632.0,0.10723531376462318 +6,Reconnaissance,0.25,6,4475.0,0.08853336438127579 +6,Reconnaissance,0.25,7,2025.0,0.0726130978586054 +6,Reconnaissance,0.25,8,6536.0,0.09732775799477392 +6,Reconnaissance,0.25,9,6996.0,0.09899680045575684 +6,Reconnaissance,0.5,0,12996.0,0.12887970117234407 +6,Reconnaissance,0.5,1,5840.0,0.08639449942599037 +6,Reconnaissance,0.5,2,1842.0,0.04852041223509337 +6,Reconnaissance,0.5,3,14852.0,0.13777554554880006 +6,Reconnaissance,0.5,4,23967.0,0.175019527497146 +6,Reconnaissance,0.5,5,9632.0,0.11095270790280895 +6,Reconnaissance,0.5,6,4475.0,0.07562688087950485 +6,Reconnaissance,0.5,7,2025.0,0.050873566252241084 +6,Reconnaissance,0.5,8,6536.0,0.09139778920981209 +6,Reconnaissance,0.5,9,6996.0,0.09455936987625915 +6,Reconnaissance,0.75,0,12996.0,0.13908870708132934 +6,Reconnaissance,0.75,1,5840.0,0.07633860657711282 +6,Reconnaissance,0.75,2,1842.0,0.03212934059999074 +6,Reconnaissance,0.75,3,14852.0,0.15373519339117522 +6,Reconnaissance,0.75,4,23967.0,0.22011257274836935 +6,Reconnaissance,0.75,5,9632.0,0.111102018310427 +6,Reconnaissance,0.75,6,4475.0,0.06252149767879063 +6,Reconnaissance,0.75,7,2025.0,0.03449477875278175 +6,Reconnaissance,0.75,8,6536.0,0.0830651064060061 +6,Reconnaissance,0.75,9,6996.0,0.08741217845401698 +6,Reconnaissance,1.0,0,12996.0,0.1457587958860937 +6,Reconnaissance,1.0,1,5840.0,0.06549948968719507 +6,Reconnaissance,1.0,2,1842.0,0.020659256849968034 +6,Reconnaissance,1.0,3,14852.0,0.16657507206065433 +6,Reconnaissance,1.0,4,23967.0,0.2688058680364733 +6,Reconnaissance,1.0,5,9632.0,0.10802929531970257 +6,Reconnaissance,1.0,6,4475.0,0.05019010553941746 +6,Reconnaissance,1.0,7,2025.0,0.022711723735714045 +6,Reconnaissance,1.0,8,6536.0,0.07330559325265532 +6,Reconnaissance,1.0,9,6996.0,0.07846479963212616 +7,Tampering,0.0,0,8.0,0.1111111111111111 +7,Tampering,0.0,1,1228.0,0.1111111111111111 +7,Tampering,0.0,2,260.0,0.1111111111111111 +7,Tampering,0.0,3,91.0,0.1111111111111111 +7,Tampering,0.0,4,483.0,0.1111111111111111 +7,Tampering,0.0,5,680.0,0.1111111111111111 +7,Tampering,0.0,7,618.0,0.1111111111111111 +7,Tampering,0.0,8,119.0,0.1111111111111111 +7,Tampering,0.0,9,99.0,0.1111111111111111 +7,Tampering,0.25,0,8.0,0.046790200953588545 +7,Tampering,0.25,1,1228.0,0.16469560326700045 +7,Tampering,0.25,2,260.0,0.11171866840872458 +7,Tampering,0.25,3,91.0,0.08592959437275875 +7,Tampering,0.25,4,483.0,0.13042750793401597 +7,Tampering,0.25,5,680.0,0.14207237541680212 +7,Tampering,0.25,7,618.0,0.13871695035463072 +7,Tampering,0.25,8,119.0,0.09189019094870464 +7,Tampering,0.25,9,99.0,0.0877589083437741 +7,Tampering,0.5,0,8.0,0.017980251490604452 +7,Tampering,0.5,1,1228.0,0.22276653609246694 +7,Tampering,0.5,2,260.0,0.1025032044334467 +7,Tampering,0.5,3,91.0,0.060641713545164094 +7,Tampering,0.5,4,483.0,0.1397089838245828 +7,Tampering,0.5,5,680.0,0.16576972797093445 +7,Tampering,0.5,7,618.0,0.15803198892360648 +7,Tampering,0.5,8,119.0,0.06934645250135424 +7,Tampering,0.5,9,99.0,0.06325114121783987 +7,Tampering,0.75,0,8.0,0.006480986155468525 +7,Tampering,0.75,1,1228.0,0.2826326812706583 +7,Tampering,0.75,2,260.0,0.08821727012693042 +7,Tampering,0.75,3,91.0,0.040142521414312994 +7,Tampering,0.75,4,483.0,0.1403731195760649 +7,Tampering,0.75,5,680.0,0.18142840727962103 +7,Tampering,0.75,7,618.0,0.16887484359213606 +7,Tampering,0.75,8,119.0,0.049088959242897044 +7,Tampering,0.75,9,99.0,0.04276121134191071 +7,Tampering,1.0,0,8.0,0.002230897936419409 +7,Tampering,1.0,1,1228.0,0.3424428332403793 +7,Tampering,1.0,2,260.0,0.07250418293363078 +7,Tampering,1.0,3,91.0,0.025376464026770774 +7,Tampering,1.0,4,483.0,0.1346904629113218 +7,Tampering,1.0,5,680.0,0.18962632459564974 +7,Tampering,1.0,7,618.0,0.17233686558839934 +7,Tampering,1.0,8,119.0,0.03318460680423871 +7,Tampering,1.0,9,99.0,0.027607361963190184 +8,Weaponization,0.0,0,11526.0,0.1 +8,Weaponization,0.0,1,2611.0,0.1 +8,Weaponization,0.0,2,1209.0,0.1 +8,Weaponization,0.0,3,1647.0,0.1 +8,Weaponization,0.0,4,917.0,0.1 +8,Weaponization,0.0,5,1375.0,0.1 +8,Weaponization,0.0,6,951.0,0.1 +8,Weaponization,0.0,7,6976.0,0.1 +8,Weaponization,0.0,8,6913.0,0.1 +8,Weaponization,0.0,9,10969.0,0.1 +8,Weaponization,0.25,0,11526.0,0.13739441060301907 +8,Weaponization,0.25,1,2611.0,0.09478751320998981 +8,Weaponization,0.25,2,1209.0,0.07819085944838905 +8,Weaponization,0.25,3,1647.0,0.08447395176979355 +8,Weaponization,0.25,4,917.0,0.07296956302069034 +8,Weaponization,0.25,5,1375.0,0.0807467564699872 +8,Weaponization,0.25,6,951.0,0.07363673785619201 +8,Weaponization,0.25,7,6976.0,0.12118560338330699 +8,Weaponization,0.25,8,6913.0,0.12091106619109798 +8,Weaponization,0.25,9,10969.0,0.13570353804753404 +8,Weaponization,0.5,0,11526.0,0.1779201650136799 +8,Weaponization,0.5,1,2611.0,0.08468164794018174 +8,Weaponization,0.5,2,1209.0,0.057623418014587556 +8,Weaponization,0.5,3,1647.0,0.06725625246465362 +8,Weaponization,0.5,4,917.0,0.05018460762438691 +8,Weaponization,0.5,5,1375.0,0.061452168704738123 +8,Weaponization,0.5,6,951.0,0.051106498114700416 +8,Weaponization,0.5,7,6976.0,0.13841689443045152 +8,Weaponization,0.5,8,6913.0,0.13779045794448722 +8,Weaponization,0.5,9,10969.0,0.17356788974813298 +8,Weaponization,0.75,0,11526.0,0.2181935226168662 +8,Weaponization,0.75,1,2611.0,0.07164535620438116 +8,Weaponization,0.75,2,1209.0,0.04021634686447464 +8,Weaponization,0.75,3,1647.0,0.050711107075023125 +8,Weaponization,0.75,4,917.0,0.03268585833755308 +8,Weaponization,0.75,5,1375.0,0.04429043196553467 +8,Weaponization,0.75,6,951.0,0.033590640121105526 +8,Weaponization,0.75,7,6976.0,0.14972273310126752 +8,Weaponization,0.75,8,6913.0,0.1487074785967807 +8,Weaponization,0.75,9,10969.0,0.21023652511701332 +8,Weaponization,1.0,0,11526.0,0.25559941455626023 +8,Weaponization,1.0,1,2611.0,0.05790127289661596 +8,Weaponization,1.0,2,1209.0,0.026810662172351087 +8,Weaponization,1.0,3,1647.0,0.036523706036279774 +8,Weaponization,1.0,4,917.0,0.020335299596398633 +8,Weaponization,1.0,5,1375.0,0.030491861444981595 +8,Weaponization,1.0,6,951.0,0.021089280170310906 +8,Weaponization,1.0,7,6976.0,0.15469907304741207 +8,Weaponization,1.0,8,6913.0,0.1533019913957511 +8,Weaponization,1.0,9,10969.0,0.24324743868363863 diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_effect.csv b/artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_effect.csv new file mode 100644 index 0000000..0f2713e --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_effect.csv @@ -0,0 +1,46 @@ +class_id,class_name,beta,n_contributors,support_min,support_max,support_mean,prototype_norm,reference_norm_beta0,displacement_from_beta0,relative_displacement +0,C&C,0.0,9,13,576,222.66666666666666,4.992104138405685,4.992104138405685,0.0,0.0 +0,C&C,0.25,9,13,576,222.66666666666666,4.852037998238657,4.992104138405685,0.36797724022308037,0.07371185176048838 +0,C&C,0.5,9,13,576,222.66666666666666,4.742376194761032,4.992104138405685,0.6695589690144707,0.13412359807628252 +0,C&C,0.75,9,13,576,222.66666666666666,4.65280704661058,4.992104138405685,0.9254131398589358,0.1853753676209332 +0,C&C,1.0,9,13,576,222.66666666666666,4.577031141286735,4.992104138405685,1.1548168541805874,0.23132867868205137 +1,Exfiltration,0.0,10,20,5300,1549.4,6.100872138731717,6.100872138731717,0.0,0.0 +1,Exfiltration,0.25,10,20,5300,1549.4,6.373166160422756,6.100872138731717,0.8237491288550555,0.1350215362858434 +1,Exfiltration,0.5,10,20,5300,1549.4,6.59429581012542,6.100872138731717,1.3263165170775955,0.21739785507999804 +1,Exfiltration,0.75,10,20,5300,1549.4,6.750434428594119,6.100872138731717,1.668226550089498,0.27344066752336466 +1,Exfiltration,1.0,10,20,5300,1549.4,6.8750663990264895,6.100872138731717,1.967235340132892,0.3224514947041411 +2,Exploitation,0.0,10,1,298,79.3,5.008097566090378,5.008097566090378,0.0,0.0 +2,Exploitation,0.25,10,1,298,79.3,5.047843612402878,5.008097566090378,0.8603955105014754,0.17180086832316885 +2,Exploitation,0.5,10,1,298,79.3,5.080358485205799,5.008097566090378,1.5532104750000217,0.3101398194629325 +2,Exploitation,0.75,10,1,298,79.3,5.1087633771611625,5.008097566090378,2.03138992764355,0.40562107683324844 +2,Exploitation,1.0,10,1,298,79.3,5.136319248087567,5.008097566090378,2.3385740832421744,0.4669585710702928 +3,Lateral_movement,0.0,10,11,6341,2211.7,8.481321033671277,8.481321033671277,0.0,0.0 +3,Lateral_movement,0.25,10,11,6341,2211.7,6.381682682418986,8.481321033671277,2.6907008570910045,0.31725020741565907 +3,Lateral_movement,0.5,10,11,6341,2211.7,5.888946922739037,8.481321033671277,3.5825998636295417,0.42241059493048755 +3,Lateral_movement,0.75,10,11,6341,2211.7,5.835315568632712,8.481321033671277,3.841073254381612,0.452886200054491 +3,Lateral_movement,1.0,10,11,6341,2211.7,5.867438144584891,8.481321033671277,3.9173091599439407,0.46187488298014234 +4,Normal,0.0,10,79,92031,29277.7,4.799298249108693,4.799298249108693,0.0,0.0 +4,Normal,0.25,10,79,92031,29277.7,5.712379958571236,4.799298249108693,1.1032890094562697,0.22988548579183807 +4,Normal,0.5,10,79,92031,29277.7,6.190715393209056,4.799298249108693,1.664815841256164,0.3468873478670235 +4,Normal,0.75,10,79,92031,29277.7,6.413993120649692,4.799298249108693,1.934162646704862,0.40300947061668174 +4,Normal,1.0,10,79,92031,29277.7,6.530433735916667,4.799298249108693,2.083642028161429,0.4341555619195775 +5,RDOS,0.0,10,4,48860,9888.3,6.959683596787167,6.959683596787167,0.0,0.0 +5,RDOS,0.25,10,4,48860,9888.3,7.618774665376232,6.959683596787167,1.6164653614457636,0.2322613289765041 +5,RDOS,0.5,10,4,48860,9888.3,7.820301085426912,6.959683596787167,2.353132945001506,0.3381091844588731 +5,RDOS,0.75,10,4,48860,9888.3,7.86519605437697,6.959683596787167,2.8666348425171084,0.41189154688590257 +5,RDOS,1.0,10,4,48860,9888.3,7.8810106988442605,6.959683596787167,3.322434234042245,0.4773829424624988 +6,Reconnaissance,0.0,10,208,22975,8916.1,5.415862180814956,5.415862180814956,0.0,0.0 +6,Reconnaissance,0.25,10,208,22975,8916.1,5.648965820089017,5.415862180814956,0.4294267183380856,0.07929055504020734 +6,Reconnaissance,0.5,10,208,22975,8916.1,5.808825260074657,5.415862180814956,0.7413987406482649,0.13689394521053017 +6,Reconnaissance,0.75,10,208,22975,8916.1,5.916255824906189,5.415862180814956,0.9617371210189529,0.17757784243952732 +6,Reconnaissance,1.0,10,208,22975,8916.1,5.991685616665521,5.415862180814956,1.1180725676513317,0.20644405827237813 +7,Tampering,0.0,10,26,2437,358.6,4.705216201871325,4.705216201871325,0.0,0.0 +7,Tampering,0.25,10,26,2437,358.6,4.639701461060896,4.705216201871325,0.7387359771740888,0.15700362012701646 +7,Tampering,0.5,10,26,2437,358.6,4.771752454614714,4.705216201871325,1.706873516878616,0.3627619738705674 +7,Tampering,0.75,10,26,2437,358.6,5.157225192090308,4.705216201871325,2.768796503000988,0.5884525565264784 +7,Tampering,1.0,10,26,2437,358.6,5.656404611496285,4.705216201871325,3.6982366587327884,0.7859865519594939 +8,Weaponization,0.0,10,17,23228,4509.4,5.306307987150024,5.306307987150024,0.0,0.0 +8,Weaponization,0.25,10,17,23228,4509.4,5.843513874965596,5.306307987150024,1.2689737477647698,0.23914438265509075 +8,Weaponization,0.5,10,17,23228,4509.4,6.216911060615151,5.306307987150024,2.0356955585328023,0.38363690224211017 +8,Weaponization,0.75,10,17,23228,4509.4,6.441588231782958,5.306307987150024,2.543364318490572,0.4793095924039254 +8,Weaponization,1.0,10,17,23228,4509.4,6.5954300391115535,5.306307987150024,2.934502391320979,0.5530214978903019 diff --git a/artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_weights.csv b/artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_weights.csv new file mode 100644 index 0000000..651f63c --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__seed2__support_weighted__beta0__prototype_snapshot__prototype_weights.csv @@ -0,0 +1,446 @@ +class_id,class_name,beta,client_id,N_ij,weight +0,C&C,0.0,0,576.0,0.1111111111111111 +0,C&C,0.0,1,290.0,0.1111111111111111 +0,C&C,0.0,3,211.0,0.1111111111111111 +0,C&C,0.0,4,37.0,0.1111111111111111 +0,C&C,0.0,5,252.0,0.1111111111111111 +0,C&C,0.0,6,212.0,0.1111111111111111 +0,C&C,0.0,7,13.0,0.1111111111111111 +0,C&C,0.0,8,296.0,0.1111111111111111 +0,C&C,0.0,9,117.0,0.1111111111111111 +0,C&C,0.25,0,576.0,0.15102191422892977 +0,C&C,0.25,1,290.0,0.12721368978362058 +0,C&C,0.25,3,211.0,0.1174910983303728 +0,C&C,0.25,4,37.0,0.0760299633060768 +0,C&C,0.25,5,252.0,0.12282435299375524 +0,C&C,0.25,6,212.0,0.11763005906798477 +0,C&C,0.25,7,13.0,0.05853561531663628 +0,C&C,0.25,8,296.0,0.12786664719190285 +0,C&C,0.25,9,117.0,0.10138665978072102 +0,C&C,0.5,0,576.0,0.19401845748233754 +0,C&C,0.5,1,290.0,0.1376673031078246 +0,C&C,0.5,3,211.0,0.11742837022526757 +0,C&C,0.5,4,37.0,0.04917367513999254 +0,C&C,0.5,5,252.0,0.12833114706365603 +0,C&C,0.5,6,212.0,0.11770630758500977 +0,C&C,0.5,7,13.0,0.02914764570162491 +0,C&C,0.5,8,296.0,0.1390841565894123 +0,C&C,0.5,9,117.0,0.08744293710487475 +0,C&C,0.75,0,576.0,0.23955666553058955 +0,C&C,0.75,1,290.0,0.14318248489669622 +0,C&C,0.75,3,211.0,0.11279847762533143 +0,C&C,0.75,4,37.0,0.030566287230480278 +0,C&C,0.75,5,252.0,0.12886702493762558 +0,C&C,0.75,6,212.0,0.1131991830239939 +0,C&C,0.75,7,13.0,0.013949187470554809 +0,C&C,0.75,8,296.0,0.14539858501481584 +0,C&C,0.75,9,117.0,0.07248210426991236 +0,C&C,1.0,0,576.0,0.2874251497005988 +0,C&C,1.0,1,290.0,0.14471057884231536 +0,C&C,1.0,3,211.0,0.10528942115768464 +0,C&C,1.0,4,37.0,0.018463073852295408 +0,C&C,1.0,5,252.0,0.12574850299401197 +0,C&C,1.0,6,212.0,0.10578842315369262 +0,C&C,1.0,7,13.0,0.006487025948103792 +0,C&C,1.0,8,296.0,0.14770459081836326 +0,C&C,1.0,9,117.0,0.058383233532934134 +1,Exfiltration,0.0,0,2126.0,0.1 +1,Exfiltration,0.0,1,20.0,0.1 +1,Exfiltration,0.0,2,914.0,0.1 +1,Exfiltration,0.0,3,1557.0,0.1 +1,Exfiltration,0.0,4,30.0,0.1 +1,Exfiltration,0.0,5,1430.0,0.1 +1,Exfiltration,0.0,6,5300.0,0.1 +1,Exfiltration,0.0,7,2362.0,0.1 +1,Exfiltration,0.0,8,150.0,0.1 +1,Exfiltration,0.0,9,1605.0,0.1 +1,Exfiltration,0.25,0,2126.0,0.1245754812136925 +1,Exfiltration,0.25,1,20.0,0.03879709938306384 +1,Exfiltration,0.25,2,914.0,0.10087378280431746 +1,Exfiltration,0.25,3,1557.0,0.11524282917902283 +1,Exfiltration,0.25,4,30.0,0.04293604842405325 +1,Exfiltration,0.25,5,1430.0,0.11281731714741136 +1,Exfiltration,0.25,6,5300.0,0.15653473624097708 +1,Exfiltration,0.25,7,2362.0,0.12789740466447497 +1,Exfiltration,0.25,8,150.0,0.06420436768136334 +1,Exfiltration,0.25,9,1605.0,0.11612093326162334 +1,Exfiltration,0.5,0,2126.0,0.1367087312207029 +1,Exfiltration,0.5,1,20.0,0.013259575496658765 +1,Exfiltration,0.5,2,914.0,0.08963708409730776 +1,Exfiltration,0.5,3,1557.0,0.11699274444200186 +1,Exfiltration,0.5,4,30.0,0.016239597086362405 +1,Exfiltration,0.5,5,1430.0,0.11211988440120217 +1,Exfiltration,0.5,6,5300.0,0.2158502506905477 +1,Exfiltration,0.5,7,2362.0,0.14409687724487022 +1,Exfiltration,0.5,8,150.0,0.036312843012313856 +1,Exfiltration,0.5,9,1605.0,0.1187824123080323 +1,Exfiltration,0.75,0,2126.0,0.1398719763694802 +1,Exfiltration,0.75,1,20.0,0.004225039397957745 +1,Exfiltration,0.75,2,914.0,0.07426221746657875 +1,Exfiltration,0.75,3,1557.0,0.11073240327966676 +1,Exfiltration,0.75,4,30.0,0.005726631097987726 +1,Exfiltration,0.75,5,1430.0,0.10388678215251898 +1,Exfiltration,0.75,6,5300.0,0.27750139361661635 +1,Exfiltration,0.75,7,2362.0,0.15136246307483137 +1,Exfiltration,0.75,8,150.0,0.019148145134778874 +1,Exfiltration,0.75,9,1605.0,0.11328294840958317 +1,Exfiltration,1.0,0,2126.0,0.13721440557635214 +1,Exfiltration,1.0,1,20.0,0.001290822253775655 +1,Exfiltration,1.0,2,914.0,0.058990576997547436 +1,Exfiltration,1.0,3,1557.0,0.10049051245643476 +1,Exfiltration,1.0,4,30.0,0.0019362333806634826 +1,Exfiltration,1.0,5,1430.0,0.09229379114495934 +1,Exfiltration,1.0,6,5300.0,0.3420678972505486 +1,Exfiltration,1.0,7,2362.0,0.15244610817090487 +1,Exfiltration,1.0,8,150.0,0.009681166903317414 +1,Exfiltration,1.0,9,1605.0,0.10358848586549632 +2,Exploitation,0.0,0,10.0,0.1 +2,Exploitation,0.0,1,1.0,0.1 +2,Exploitation,0.0,2,240.0,0.1 +2,Exploitation,0.0,3,2.0,0.1 +2,Exploitation,0.0,4,35.0,0.1 +2,Exploitation,0.0,5,145.0,0.1 +2,Exploitation,0.0,6,298.0,0.1 +2,Exploitation,0.0,7,49.0,0.1 +2,Exploitation,0.0,8,7.0,0.1 +2,Exploitation,0.0,9,6.0,0.1 +2,Exploitation,0.25,0,10.0,0.07472353923163849 +2,Exploitation,0.25,1,1.0,0.042020134074432634 +2,Exploitation,0.25,2,240.0,0.16539037968734396 +2,Exploitation,0.25,3,2.0,0.04997064241468357 +2,Exploitation,0.25,4,35.0,0.10220554181683487 +2,Exploitation,0.25,5,145.0,0.1458140706954984 +2,Exploitation,0.25,6,298.0,0.17458683816365086 +2,Exploitation,0.25,7,49.0,0.11117482481854002 +2,Exploitation,0.25,8,7.0,0.06834896520487059 +2,Exploitation,0.25,9,6.0,0.06576506389250662 +2,Exploitation,0.5,0,10.0,0.046242935423721356 +2,Exploitation,0.5,1,1.0,0.014623300163104301 +2,Exploitation,0.5,2,240.0,0.22654319199318068 +2,Exploitation,0.5,3,2.0,0.020680469417314797 +2,Exploitation,0.5,4,35.0,0.08651261045713868 +2,Exploitation,0.5,5,145.0,0.17608785196808927 +2,Exploitation,0.5,6,298.0,0.25243730010193305 +2,Exploitation,0.5,7,49.0,0.10236310114173011 +2,Exploitation,0.5,8,7.0,0.03868961557862425 +2,Exploitation,0.5,9,6.0,0.03581962375516356 +2,Exploitation,0.75,0,10.0,0.02512275929612515 +2,Exploitation,0.75,1,1.0,0.0044675285579663295 +2,Exploitation,0.75,2,240.0,0.27241170763423844 +2,Exploitation,0.75,3,2.0,0.007513457498874965 +2,Exploitation,0.75,4,35.0,0.06428629111250711 +2,Exploitation,0.75,5,145.0,0.18667808582049356 +2,Exploitation,0.75,6,298.0,0.32042731849990974 +2,Exploitation,0.75,7,49.0,0.08273978677620542 +2,Exploitation,0.75,8,7.0,0.019226085412864014 +2,Exploitation,0.75,9,6.0,0.017126979390815276 +2,Exploitation,1.0,0,10.0,0.012610340479192938 +2,Exploitation,1.0,1,1.0,0.0012610340479192938 +2,Exploitation,1.0,2,240.0,0.3026481715006305 +2,Exploitation,1.0,3,2.0,0.0025220680958385876 +2,Exploitation,1.0,4,35.0,0.044136191677175286 +2,Exploitation,1.0,5,145.0,0.1828499369482976 +2,Exploitation,1.0,6,298.0,0.37578814627994955 +2,Exploitation,1.0,7,49.0,0.0617906683480454 +2,Exploitation,1.0,8,7.0,0.008827238335435058 +2,Exploitation,1.0,9,6.0,0.007566204287515763 +3,Lateral_movement,0.0,0,3084.0,0.1 +3,Lateral_movement,0.0,1,11.0,0.1 +3,Lateral_movement,0.0,2,2152.0,0.1 +3,Lateral_movement,0.0,3,72.0,0.1 +3,Lateral_movement,0.0,4,566.0,0.1 +3,Lateral_movement,0.0,5,6341.0,0.1 +3,Lateral_movement,0.0,6,3130.0,0.1 +3,Lateral_movement,0.0,7,4431.0,0.1 +3,Lateral_movement,0.0,8,131.0,0.1 +3,Lateral_movement,0.0,9,2199.0,0.1 +3,Lateral_movement,0.25,0,3084.0,0.12702175800740229 +3,Lateral_movement,0.25,1,11.0,0.031041857299547067 +3,Lateral_movement,0.25,2,2152.0,0.11609408516022754 +3,Lateral_movement,0.25,3,72.0,0.0496515317395058 +3,Lateral_movement,0.25,4,566.0,0.0831387775793627 +3,Lateral_movement,0.25,5,6341.0,0.15210341508889633 +3,Lateral_movement,0.25,6,3130.0,0.12749278590878954 +3,Lateral_movement,0.25,7,4431.0,0.13906728125201873 +3,Lateral_movement,0.25,8,131.0,0.057665671116614775 +3,Lateral_movement,0.25,9,2199.0,0.11672283684763529 +3,Lateral_movement,0.5,0,3084.0,0.13953291328353135 +3,Lateral_movement,0.5,1,11.0,0.008333277031910022 +3,Lateral_movement,0.5,2,2152.0,0.1165576038262483 +3,Lateral_movement,0.5,3,72.0,0.02131992759351566 +3,Lateral_movement,0.5,4,566.0,0.0597761159716795 +3,Lateral_movement,0.5,5,6341.0,0.20007754681173798 +3,Lateral_movement,0.5,6,3130.0,0.14056967669368875 +3,Lateral_movement,0.5,7,4431.0,0.16725162738364865 +3,Lateral_movement,0.5,8,131.0,0.028757764639806005 +3,Lateral_movement,0.5,9,2199.0,0.11782354676423379 +3,Lateral_movement,0.75,0,3084.0,0.1422829526966925 +3,Lateral_movement,0.75,1,11.0,0.0020766418694166703 +3,Lateral_movement,0.75,2,2152.0,0.10862975309362505 +3,Lateral_movement,0.75,3,72.0,0.008497994868397177 +3,Lateral_movement,0.75,4,566.0,0.0398960055447025 +3,Lateral_movement,0.75,5,6341.0,0.24430671665218012 +3,Lateral_movement,0.75,6,3130.0,0.14387169007556813 +3,Lateral_movement,0.75,7,4431.0,0.18672110322562685 +3,Lateral_movement,0.75,8,131.0,0.013312835445965593 +3,Lateral_movement,0.75,9,2199.0,0.11040430652782546 +3,Lateral_movement,1.0,0,3084.0,0.13944024958176968 +3,Lateral_movement,1.0,1,11.0,0.0004973549758104626 +3,Lateral_movement,1.0,2,2152.0,0.09730071890401049 +3,Lateral_movement,1.0,3,72.0,0.0032554143871230275 +3,Lateral_movement,1.0,4,566.0,0.0255911742098838 +3,Lateral_movement,1.0,5,6341.0,0.28670253651037664 +3,Lateral_movement,1.0,6,3130.0,0.14152009766243162 +3,Lateral_movement,1.0,7,4431.0,0.2003436270741963 +3,Lateral_movement,1.0,8,131.0,0.005923045621015508 +3,Lateral_movement,1.0,9,2199.0,0.09942578107338247 +4,Normal,0.0,0,864.0,0.1 +4,Normal,0.0,1,418.0,0.1 +4,Normal,0.0,2,79.0,0.1 +4,Normal,0.0,3,33716.0,0.1 +4,Normal,0.0,4,10789.0,0.1 +4,Normal,0.0,5,1541.0,0.1 +4,Normal,0.0,6,73646.0,0.1 +4,Normal,0.0,7,57785.0,0.1 +4,Normal,0.0,8,92031.0,0.1 +4,Normal,0.0,9,21908.0,0.1 +4,Normal,0.25,0,864.0,0.05188460739258251 +4,Normal,0.25,1,418.0,0.04327172113503886 +4,Normal,0.25,2,79.0,0.028530992000834074 +4,Normal,0.25,3,33716.0,0.1296788380467858 +4,Normal,0.25,4,10789.0,0.09753385669084398 +4,Normal,0.25,5,1541.0,0.059959880532585996 +4,Normal,0.25,6,73646.0,0.15765123965898994 +4,Normal,0.25,7,57785.0,0.14837607503612354 +4,Normal,0.25,8,92031.0,0.16668389274771683 +4,Normal,0.25,9,21908.0,0.11642889675849852 +4,Normal,0.5,0,864.0,0.021795608067872552 +4,Normal,0.5,1,418.0,0.015160036935382743 +4,Normal,0.5,2,79.0,0.006590610776578489 +4,Normal,0.5,3,33716.0,0.13615391733129972 +4,Normal,0.5,4,10789.0,0.07701985826089296 +4,Normal,0.5,5,1541.0,0.029108071823932803 +4,Normal,0.5,6,73646.0,0.2012272074670356 +4,Normal,0.5,7,57785.0,0.17824595265334742 +4,Normal,0.5,8,92031.0,0.22494647347085892 +4,Normal,0.5,9,21908.0,0.10975226321279885 +4,Normal,0.75,0,864.0,0.008234413752433757 +4,Normal,0.75,1,418.0,0.0047767178535437 +4,Normal,0.75,2,79.0,0.0013692025418151716 +4,Normal,0.75,3,33716.0,0.12856549080612478 +4,Normal,0.75,4,10789.0,0.05469948948624618 +4,Normal,0.75,5,1541.0,0.01270864818588827 +4,Normal,0.75,6,73646.0,0.23099854847865048 +4,Normal,0.75,7,57785.0,0.19257890653203613 +4,Normal,0.75,8,92031.0,0.2730222139329396 +4,Normal,0.75,9,21908.0,0.093046368430322 +4,Normal,1.0,0,864.0,0.0029510514828692143 +4,Normal,1.0,1,418.0,0.0014277077775918191 +4,Normal,1.0,2,79.0,0.000269829938827162 +4,Normal,1.0,3,33716.0,0.11515931920881763 +4,Normal,1.0,4,10789.0,0.03685057227856013 +4,Normal,1.0,5,1541.0,0.005263391591552615 +4,Normal,1.0,6,73646.0,0.2515429832261414 +4,Normal,1.0,7,57785.0,0.1973686457611083 +4,Normal,1.0,8,92031.0,0.31433821645826004 +4,Normal,1.0,9,21908.0,0.0748282822762717 +5,RDOS,0.0,0,17951.0,0.1 +5,RDOS,0.0,1,7758.0,0.1 +5,RDOS,0.0,2,48860.0,0.1 +5,RDOS,0.0,3,94.0,0.1 +5,RDOS,0.0,4,1624.0,0.1 +5,RDOS,0.0,5,1142.0,0.1 +5,RDOS,0.0,6,4.0,0.1 +5,RDOS,0.0,7,10129.0,0.1 +5,RDOS,0.0,8,4382.0,0.1 +5,RDOS,0.0,9,6939.0,0.1 +5,RDOS,0.25,0,17951.0,0.14502857687379875 +5,RDOS,0.25,1,7758.0,0.117589587658176 +5,RDOS,0.25,2,48860.0,0.18628152700149078 +5,RDOS,0.25,3,94.0,0.039013362856557 +5,RDOS,0.25,4,1624.0,0.07953859769224278 +5,RDOS,0.25,5,1142.0,0.0728363123438262 +5,RDOS,0.25,6,4.0,0.017719294303109735 +5,RDOS,0.25,7,10129.0,0.12569646568603307 +5,RDOS,0.25,8,4382.0,0.10194114727488954 +5,RDOS,0.25,9,6939.0,0.11435512830987608 +5,RDOS,0.5,0,17951.0,0.17198442418661095 +5,RDOS,0.5,1,7758.0,0.11306278550208099 +5,RDOS,0.5,2,48860.0,0.2837406273058024 +5,RDOS,0.5,3,94.0,0.01244539600169817 +5,RDOS,0.5,4,1624.0,0.05172944342275919 +5,RDOS,0.5,5,1142.0,0.043378831099658635 +5,RDOS,0.5,6,4.0,0.002567289170851146 +5,RDOS,0.5,7,10129.0,0.12918975624376042 +5,RDOS,0.5,8,4382.0,0.08497300555974267 +5,RDOS,0.5,9,6939.0,0.10692844150703547 +5,RDOS,0.75,0,17951.0,0.18336531141459259 +5,RDOS,0.75,1,7758.0,0.09773790520016168 +5,RDOS,0.75,2,48860.0,0.38856689071177486 +5,RDOS,0.75,3,94.0,0.0035694112275838435 +5,RDOS,0.75,4,1624.0,0.03024755073569198 +5,RDOS,0.75,5,1142.0,0.02322738257067633 +5,RDOS,0.75,6,4.0,0.00033442265364325624 +5,RDOS,0.75,7,10129.0,0.11937836417894782 +5,RDOS,0.75,8,4382.0,0.06368028158230388 +5,RDOS,0.75,9,6939.0,0.08989247972462366 +5,RDOS,1.0,0,17951.0,0.18153777696874082 +5,RDOS,1.0,1,7758.0,0.07845635751342496 +5,RDOS,1.0,2,48860.0,0.4941193127231172 +5,RDOS,1.0,3,94.0,0.0009506184076130376 +5,RDOS,1.0,4,1624.0,0.016423449935782694 +5,RDOS,1.0,5,1142.0,0.011549002356320096 +5,RDOS,1.0,6,4.0,4.0451847132469685e-05 +5,RDOS,1.0,7,10129.0,0.10243418990119636 +5,RDOS,1.0,8,4382.0,0.044314998533620545 +5,RDOS,1.0,9,6939.0,0.0701738418130518 +6,Reconnaissance,0.0,0,1651.0,0.1 +6,Reconnaissance,0.0,1,208.0,0.1 +6,Reconnaissance,0.0,2,12985.0,0.1 +6,Reconnaissance,0.0,3,17154.0,0.1 +6,Reconnaissance,0.0,4,22975.0,0.1 +6,Reconnaissance,0.0,5,16770.0,0.1 +6,Reconnaissance,0.0,6,5123.0,0.1 +6,Reconnaissance,0.0,7,8929.0,0.1 +6,Reconnaissance,0.0,8,2708.0,0.1 +6,Reconnaissance,0.0,9,658.0,0.1 +6,Reconnaissance,0.25,0,1651.0,0.07374148107073869 +6,Reconnaissance,0.25,1,208.0,0.04393300617126853 +6,Reconnaissance,0.25,2,12985.0,0.1234910764123451 +6,Reconnaissance,0.25,3,17154.0,0.13239343160661676 +6,Reconnaissance,0.25,4,22975.0,0.14242589843429998 +6,Reconnaissance,0.25,5,16770.0,0.1316462078006376 +6,Reconnaissance,0.25,6,5123.0,0.09787148471969809 +6,Reconnaissance,0.25,7,8929.0,0.11245422928318183 +6,Reconnaissance,0.25,8,2708.0,0.08345209671284375 +6,Reconnaissance,0.25,9,658.0,0.05859108778836971 +6,Reconnaissance,0.5,0,1651.0,0.04926272195084457 +6,Reconnaissance,0.5,1,208.0,0.0174854387978552 +6,Reconnaissance,0.5,2,12985.0,0.13815475751236914 +6,Reconnaissance,0.5,3,17154.0,0.15879161684940787 +6,Reconnaissance,0.5,4,22975.0,0.18376915905719687 +6,Reconnaissance,0.5,5,16770.0,0.15700424660659393 +6,Reconnaissance,0.5,6,5123.0,0.08677748234243855 +6,Reconnaissance,0.5,7,8929.0,0.11456350164519018 +6,Reconnaissance,0.5,8,2708.0,0.06309125958277643 +6,Reconnaissance,0.5,9,658.0,0.03109981565532717 +6,Reconnaissance,0.75,0,1651.0,0.030840346291550146 +6,Reconnaissance,0.75,1,208.0,0.006521634300518785 +6,Reconnaissance,0.75,2,12985.0,0.14484063041988704 +6,Reconnaissance,0.75,3,17154.0,0.17847730325539155 +6,Reconnaissance,0.75,4,22975.0,0.22220333914755816 +6,Reconnaissance,0.75,5,16770.0,0.17547236839271976 +6,Reconnaissance,0.75,6,5123.0,0.07210281289252812 +6,Reconnaissance,0.75,7,8929.0,0.10937323511754056 +6,Reconnaissance,0.75,8,2708.0,0.044698755937912026 +6,Reconnaissance,0.75,9,658.0,0.015469574244393894 +6,Reconnaissance,1.0,0,1651.0,0.0185170646358834 +6,Reconnaissance,1.0,1,208.0,0.002332858536804208 +6,Reconnaissance,1.0,2,12985.0,0.14563542355962808 +6,Reconnaissance,1.0,3,17154.0,0.1923935352900932 +6,Reconnaissance,1.0,4,22975.0,0.25767992732248407 +6,Reconnaissance,1.0,5,16770.0,0.18808671952983927 +6,Reconnaissance,1.0,6,5123.0,0.057457857134845955 +6,Reconnaissance,1.0,7,8929.0,0.10014468209194603 +6,Reconnaissance,1.0,8,2708.0,0.030372023642624018 +6,Reconnaissance,1.0,9,658.0,0.007379908255851774 +7,Tampering,0.0,0,118.0,0.1 +7,Tampering,0.0,1,371.0,0.1 +7,Tampering,0.0,2,135.0,0.1 +7,Tampering,0.0,3,2437.0,0.1 +7,Tampering,0.0,4,76.0,0.1 +7,Tampering,0.0,5,132.0,0.1 +7,Tampering,0.0,6,146.0,0.1 +7,Tampering,0.0,7,26.0,0.1 +7,Tampering,0.0,8,64.0,0.1 +7,Tampering,0.0,9,81.0,0.1 +7,Tampering,0.25,0,118.0,0.0914906614903368 +7,Tampering,0.25,1,371.0,0.12182873736373746 +7,Tampering,0.25,2,135.0,0.09462147373200593 +7,Tampering,0.25,3,2437.0,0.19503841012134226 +7,Tampering,0.25,4,76.0,0.08196145582272057 +7,Tampering,0.25,5,132.0,0.09409136058601118 +7,Tampering,0.25,6,146.0,0.09649270466577241 +7,Tampering,0.25,7,26.0,0.06268297534976354 +7,Tampering,0.25,8,64.0,0.07851475128408372 +7,Tampering,0.25,9,81.0,0.08327746958422619 +7,Tampering,0.5,0,118.0,0.07465928078918507 +7,Tampering,0.5,1,371.0,0.13238224845001956 +7,Tampering,0.5,2,135.0,0.07985639165881428 +7,Tampering,0.5,3,2437.0,0.33928961183869 +7,Tampering,0.5,4,76.0,0.05991693570925297 +7,Tampering,0.5,5,132.0,0.07896411346733563 +7,Tampering,0.5,6,146.0,0.08304609697876314 +7,Tampering,0.5,7,26.0,0.035045275003324286 +7,Tampering,0.5,8,64.0,0.05498355110805415 +7,Tampering,0.5,9,81.0,0.06185649499656092 +7,Tampering,0.75,0,118.0,0.05307060131423309 +7,Tampering,0.75,1,371.0,0.12530630827033065 +7,Tampering,0.75,2,135.0,0.05870739760575514 +7,Tampering,0.75,3,2437.0,0.514143263422916 +7,Tampering,0.75,4,76.0,0.038155108874919706 +7,Tampering,0.75,5,132.0,0.05772619755866028 +7,Tampering,0.75,6,146.0,0.062259717357723954 +7,Tampering,0.75,7,26.0,0.017067602272501124 +7,Tampering,0.75,8,64.0,0.03354111405422001 +7,Tampering,0.75,9,81.0,0.04002268926874002 +7,Tampering,1.0,0,118.0,0.03290574456218628 +7,Tampering,1.0,1,371.0,0.10345789180145008 +7,Tampering,1.0,2,135.0,0.03764640267707752 +7,Tampering,1.0,3,2437.0,0.6795872838817624 +7,Tampering,1.0,4,76.0,0.021193530395984383 +7,Tampering,1.0,5,132.0,0.03680981595092025 +7,Tampering,1.0,6,146.0,0.04071388733965421 +7,Tampering,1.0,7,26.0,0.007250418293363078 +7,Tampering,1.0,8,64.0,0.01784718349135527 +7,Tampering,1.0,9,81.0,0.022587841606246516 +8,Weaponization,0.0,0,895.0,0.1 +8,Weaponization,0.0,1,1014.0,0.1 +8,Weaponization,0.0,2,23228.0,0.1 +8,Weaponization,0.0,3,10869.0,0.1 +8,Weaponization,0.0,4,218.0,0.1 +8,Weaponization,0.0,5,1570.0,0.1 +8,Weaponization,0.0,6,17.0,0.1 +8,Weaponization,0.0,7,2670.0,0.1 +8,Weaponization,0.0,8,38.0,0.1 +8,Weaponization,0.0,9,4575.0,0.1 +8,Weaponization,0.25,0,895.0,0.08582243760083762 +8,Weaponization,0.25,1,1014.0,0.08854306990947461 +8,Weaponization,0.25,2,23228.0,0.19370821585254064 +8,Weaponization,0.25,3,10869.0,0.16021105634310917 +8,Weaponization,0.25,4,218.0,0.06029193331621394 +8,Weaponization,0.25,5,1570.0,0.09876885061052997 +8,Weaponization,0.25,6,17.0,0.03186084700198994 +8,Weaponization,0.25,7,2670.0,0.11279059873978263 +8,Weaponization,0.25,8,38.0,0.03895749161753624 +8,Weaponization,0.25,9,4575.0,0.1290454990079852 +8,Weaponization,0.5,0,895.0,0.05954631191374364 +8,Weaponization,0.5,1,1014.0,0.06338147296578088 +8,Weaponization,0.5,2,23228.0,0.3033536738955772 +8,Weaponization,0.5,3,10869.0,0.2075095768145407 +8,Weaponization,0.5,4,218.0,0.02938810596696327 +8,Weaponization,0.5,5,1570.0,0.07886661054849604 +8,Weaponization,0.5,6,17.0,0.00820668588741752 +8,Weaponization,0.5,7,2670.0,0.10284874130321621 +8,Weaponization,0.5,8,38.0,0.0122697340296214 +8,Weaponization,0.5,9,4575.0,0.13462908667464316 +8,Weaponization,0.75,0,895.0,0.035987691170914056 +8,Weaponization,0.75,1,1014.0,0.039519839730892736 +8,Weaponization,0.75,2,23228.0,0.41380484518079425 +8,Weaponization,0.75,3,10869.0,0.2341148165504639 +8,Weaponization,0.75,4,218.0,0.012477543098555985 +8,Weaponization,0.75,5,1570.0,0.054854397303363285 +8,Weaponization,0.75,6,17.0,0.0018412951086240396 +8,Weaponization,0.75,7,2670.0,0.08169023670182742 +8,Weaponization,0.75,8,38.0,0.0033660797865626784 +8,Weaponization,0.75,9,4575.0,0.12234325536800156 +8,Weaponization,1.0,0,895.0,0.01984742981327893 +8,Weaponization,1.0,1,1014.0,0.02248636182197188 +8,Weaponization,1.0,2,23228.0,0.5151017873774781 +8,Weaponization,1.0,3,10869.0,0.24102984876036723 +8,Weaponization,1.0,4,218.0,0.004834346032731627 +8,Weaponization,1.0,5,1570.0,0.0348161617953608 +8,Weaponization,1.0,6,17.0,0.00037699028695613605 +8,Weaponization,1.0,7,2670.0,0.059209650951346074 +8,Weaponization,1.0,8,38.0,0.0008426841708431277 +8,Weaponization,1.0,9,4575.0,0.10145473898966603 diff --git a/docs/research/POSTHOC_STATISTICAL_AUDIT.md b/docs/research/POSTHOC_STATISTICAL_AUDIT.md new file mode 100644 index 0000000..ee5e04f --- /dev/null +++ b/docs/research/POSTHOC_STATISTICAL_AUDIT.md @@ -0,0 +1,335 @@ +# Post-hoc Statistical Audit of the PROTEAN Experiment Grid + +Status: **read-only post-hoc audit of completed experiments.** No training, no partition +regeneration, no modification of any existing result, manifest, dataset, configuration, source +file, or test. All numbers below are recomputed from the audited artifacts by +`scripts/posthoc_statistical_audit.py`, which writes new `posthoc__*.csv` tables under +`artifacts/analysis/` and refuses to overwrite existing files. + +Scope: determine what the completed 108-run grid can and cannot support statistically, and +provide transparent, reusable wording for the thesis limitations. + +--- + +## 1. Executive findings + +1. **The comparison is valid and paired.** All 18 `(dataset, alpha, seed)` cells pair the + `original_protean` run with the validation-selected `support_weighted` run on an identical + `manifest_content_hash`, split, and seed (`posthoc__paired_global_per_seed.csv`, + `pairing_verified = True` for every row). +2. **The paired differences are small and mostly seed-inconsistent.** For test rare-class + macro-F1, only **X-IIoTID α=0.50** has all three seed differences in the same direction + (**+0.0230** mean; paired-difference sample std **0.0058**); the other five conditions have + mixed-sign seed differences. +3. **No condition can reach statistical significance with three seeds.** The smallest attainable + two-sided exact sign-flip p-value with three nonzero differences is **0.25**, and only + X-IIoTID α=0.50 attains it. This is a **power/design limit**, not evidence of no effect. +4. **The paired-difference variability is not the group variability.** At X-IIoTID α=0.50 the + baseline and proposed cross-seed std are **0.0287** and **0.0267**, while the paired-difference + std is **0.0058** — shared seed-to-seed variation cancels in the paired difference. +5. **Selected beta is weakly separated and unstable across seeds.** The best-vs-second margin is + only **0.0024–0.0122**, and the pooled selection agrees with at most **2 of 3** seeds in every + condition (`posthoc__selection_stability.csv`). +6. **The `S_j` participant rule is materially exercised**: `|S_j| < M` in **53 of 162** + class-condition cells (verified). This is an implementation/reproduction assumption, not an + ordinary statistical limitation. +7. **Publication metrics could not be verified** because the referenced `journal/main.tex` + (and the paper's Table IV) is **not present in this repository**. Reproduced baseline + accuracies are reported instead; the discrepancy figures remain unverified. + +--- + +## 2. Data sources and pairing validation + +Primary sources (all committed, read-only): + +| Purpose | Artifact | +|---|---| +| Paired per-seed global metrics | `artifacts/analysis/deepcmp__global_per_seed.csv` | +| Per-run source of truth | `results//test_global_metrics.json`, `metadata.json` | +| Validation per-seed / pooled | `artifacts/analysis/__alpha*__multiseed__validation_per_seed.csv`, `..._validation_pooled.csv`, `..._selection_summary.json` | +| Selection margins | `artifacts/analysis/final__selection_strength.csv`, `final__pooled_validation_curves.csv` | +| Per-class | `results//test_per_class.csv` → `deepcmp__per_class.csv` | +| Per-client | `results//per_client.csv` → `deepcmp__per_client.csv` | +| `S_j` rule | `artifacts/analysis/sj_participant_rule_analysis.csv` | +| Reproduced baseline metrics | `docs/baseline/BASELINE_VALIDATION.md` | + +Pairing rule enforced in the script: same `dataset`, `alpha`, `seed`, identical +`manifest_content_hash`, and recorded `beta == validation-selected beta`. Result: **18/18 cells +verified**, written with `pairing_verified` and the manifest hash into +`posthoc__paired_global_per_seed.csv`. + +--- + +## 3. Descriptive paired results by condition + +Test split, `delta = proposed_selected_beta − original_protean`, three paired seeds per condition. +Source: `posthoc__paired_global_by_condition.csv`. The standard deviations below are three distinct +quantities: `base_std` and `prop_std` are the cross-seed spread within each group, `paired_std` is +computed directly from the three matched differences. + +### 3.1 Rare-class macro-F1 (primary comparison) + +| Dataset | alpha | beta* | Δ seed 0 | Δ seed 1 | Δ seed 2 | mean Δ | paired std | base std | prop std | +/−/0 | same sign | +|---|---|---|---|---|---|---|---|---|---|---|---| +| X-IIoTID | 0.25 | 0.00 | +0.0016 | −0.0020 | −0.0016 | −0.0007 | 0.0019 | 0.0472 | 0.0476 | 1/2/0 | no | +| X-IIoTID | 0.50 | 0.75 | +0.0266 | +0.0163 | +0.0261 | **+0.0230** | 0.0058 | 0.0287 | 0.0267 | 3/0/0 | **yes** | +| X-IIoTID | 0.75 | 0.50 | −0.0097 | +0.0101 | +0.0036 | +0.0014 | 0.0101 | 0.0778 | 0.0855 | 2/1/0 | no | +| 5G-NIDD | 0.25 | 0.00 | −0.0050 | −0.0010 | +0.0029 | −0.0010 | 0.0039 | 0.0254 | 0.0242 | 1/2/0 | no | +| 5G-NIDD | 0.50 | 1.00 | +0.0090 | −0.0811 | +0.0464 | −0.0086 | 0.0655 | 0.0371 | 0.0285 | 2/1/0 | no | +| 5G-NIDD | 0.75 | 0.50 | −0.0011 | +0.0450 | +0.0067 | +0.0168 | 0.0247 | 0.0231 | 0.0145 | 2/1/0 | no | + +**Verification of the reported `+0.0230`.** Source `final__six_condition_master.csv` +(row X-IIoTID α=0.50): original rare-F1 mean 0.642520 (std **0.028683**), selected-beta mean +0.665503 (std **0.026685**), difference **+0.022983**. The paired-difference std computed directly +from the three matched seed differences is **0.0058**. The value is exact. + +**What this does and does not establish.** X-IIoTID α=0.50 is the only condition where the +improvement appears at every seed, and only there is the paired difference small relative to its own +mean. It does **not** establish statistical significance (see §4), and it does not generalize: the +same beta direction fails at 5G-NIDD α=0.50 and is mixed at four other conditions. + +### 3.2 Other metrics + +All five metrics (rare-class macro-F1, rare-class recall, macro-F1, weighted-F1, accuracy) with the +same columns, including per-metric positive/negative/zero counts and sign agreement, are in +`posthoc__paired_global_by_condition.csv`. Highlights: + +- **Rare-class recall** is negative in **5 of 6** conditions (and in **all four** `beta>0` + conditions): the rare-class F1 gains are precision-driven. +- **Overall macro-F1** moves in the same direction as rare-class F1 in every condition, but its + paired differences are equally seed-inconsistent. +- **5G-NIDD α=0.50** has a paired-difference std of **0.0655**, larger than either group std, + because the three seed differences have mixed sign (one large negative). + +--- + +## 4. Inferential analysis and limitations + +Sources: `posthoc__paired_test_results.csv`. Only three paired seeds exist per condition. + +| Dataset | alpha | beta* | exact sign-flip p (2-sided) | exact Wilcoxon p (2-sided) | paired-t 95% CI | min attainable sign-flip p | +|---|---|---|---|---|---|---| +| X-IIoTID | 0.25 | 0.00 | 0.75 | 1.00 | [−0.0054, +0.0041] | 0.25 | +| X-IIoTID | 0.50 | 0.75 | **0.25** | 0.50 | [+0.0085, +0.0374] | 0.25 | +| X-IIoTID | 0.75 | 0.50 | 0.75 | 1.00 | [−0.0236, +0.0264] | 0.25 | +| 5G-NIDD | 0.25 | 0.00 | 0.75 | 1.00 | [−0.0108, +0.0087] | 0.25 | +| 5G-NIDD | 0.50 | 1.00 | 1.00 | 1.00 | [−0.1713, +0.1542] | 0.25 | +| 5G-NIDD | 0.75 | 0.50 | 0.50 | 1.00 | [−0.0444, +0.0781] | 0.25 | + +Test definitions and assumptions: + +- **Exact sign-flip permutation test** on the mean of the paired differences. Assumption: under the + null the sign of each paired difference is exchangeable. With `n = 3`, the null has `2^3 = 8` + sign vectors, so the smallest attainable two-sided p-value is `2/8 = 0.25`; this was verified by + enumeration, not assumed. +- **Exact Wilcoxon signed-rank** (average ranks for ties, zero differences dropped, exact null by + enumerating `2^n` sign assignments, two-sided min-rank convention). At `n = 3` the exact p-values + are coarse (0.50 or 1.00), so this test adds little. +- **Paired-t 95% CI** using `t = 4.3027` (df = 2). This is an explicitly assumption-dependent, + underpowered summary; it is not a reliable interval estimate for a population. + +Interpretation rules applied: + +- **No condition is statistically significant**, and none can be with three seeds. +- **Non-significance is not evidence of no effect**; it reflects a sample far too small to resolve + effects of this size. +- Tests are **not** pooled across the six conditions (they are not replicates of one condition) nor + across clients (clients are nested in the same trained model and seed). +- Bootstrap CIs are deliberately **not** used: three seeds cannot support a reliable resampling + distribution. + +Conclusion: **prioritize descriptive reporting** (values, direction, seed consistency) over +inferential claims for this grid. + +--- + +## 5. Beta-selection stability + +Source: `posthoc__selection_stability.csv`; per-seed selections are computed from each condition's +`validation_per_seed.csv` (per-client-mean validation rare-class macro-F1). + +| Dataset | alpha | pooled beta | per-seed beta (s0/s1/s2) | agree with pooled | best score | 2nd-best (beta, score) | margin | candidate range | +|---|---|---|---|---|---|---|---|---| +| X-IIoTID | 0.25 | 0.00 | 0.75 / 0.50 / 0.00 | 1/3 | 0.5917 | 0.25, 0.5887 | 0.0030 | 0.0596 | +| X-IIoTID | 0.50 | 0.75 | 0.50 / 1.00 / 0.75 | 1/3 | 0.5794 | 1.00, 0.5735 | 0.0059 | 0.0350 | +| X-IIoTID | 0.75 | 0.50 | 0.50 / 0.50 / 0.00 | 2/3 | 0.4767 | 0.25, 0.4744 | 0.0024 | 0.0081 | +| 5G-NIDD | 0.25 | 0.00 | 0.25 / 0.00 / 0.00 | 2/3 | 0.7615 | 0.25, 0.7493 | 0.0122 | 0.0392 | +| 5G-NIDD | 0.50 | 1.00 | 1.00 / 0.00 / 1.00 | 2/3 | 0.6924 | 0.75, 0.6884 | 0.0040 | 0.0473 | +| 5G-NIDD | 0.75 | 0.50 | 0.50 / 1.00 / 1.00 | 1/3 | 0.8091 | 1.00, 0.8043 | 0.0048 | 0.0310 | + +Verification: the reported selection margins of **0.0024–0.0122** are confirmed exactly. + +**Interpretation.** A margin of 0.002–0.012 is small relative to both the candidate-score range +(0.008–0.060) and the across-seed std of the selected beta (0.022–0.139 in earlier analyses), and the +pooled selection agrees with only 1–2 of 3 seeds. This makes the selection **fragile / weakly +separated**: it should be described as "best among the tested candidates under the pooled validation +procedure", **not** as an optimal or robust choice. A small margin does **not** prove the selected +beta is wrong; it means the data cannot discriminate among close candidates. All selections remain +**validation-only** (`test_used_for_selection = False`); test metrics are never used to select beta. + +--- + +## 6. `S_j` participant-rule audit + +Source: `artifacts/analysis/sj_participant_rule_analysis.csv` (162 class-condition cells = 18 +condition-seeds × 9 classes). Verified: **`|S_j| < M` in 53 / 162 cells** (X-IIoTID 19/7/3 and +5G-NIDD 16/4/4 for α = 0.25/0.50/0.75); all 53 are exactly the cells with ≥1 missing client +(`missing_clients` non-empty). `S_j` is the set of clients with `N_ij > 0` and a valid prototype. + +**Why this is an implementation/reproduction assumption, not an ordinary statistical limitation.** +Eq. (2)/(5.5) of the source paper sums over `M` clients but is silent on `N_ij = 0`; no official +implementation exists (see `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). The project therefore +*froze* `S_j = {i : N_ij > 0 and C_ij exists}` and used equal `1/|S_j|` weights in the baseline. In +`53/162` cells the literal `1/M` reading (e.g. substituting a zero vector for an absent prototype) +would differ from the `S_j` reading, so all results are **conditional on this rule**. More seeds +would not remove this; only the (unavailable) official code, or a documented threat/robustness +analysis across alternative participant rules, could. + +--- + +## 7. Baseline reproduction discrepancy audit + +**Reproduced baseline metrics available in this repository** +(`docs/baseline/BASELINE_VALIDATION.md`, and `final__six_condition_master.csv`): + +| Evidence | Value | +|---|---| +| X-IIoTID α=0.50 seed 0 accuracy | 0.8904 (macro-F1 0.7077, rare-F1 0.6096) | +| 5G-NIDD α=0.50 seed 0 accuracy | 0.7344 (macro-F1 0.8369, rare-F1 0.8720) | +| X-IIoTID original accuracy means (α=0.25/0.50/0.75) | 0.8754 / 0.8995 / 0.9206 | +| 5G-NIDD original accuracy means (α=0.25/0.50/0.75) | 0.7310 / 0.7440 / 0.7510 | + +**Unresolved data gap.** The task's comparison uses the paper's reported metrics ("Table IV") in +`journal/main.tex`. That file and directory **do not exist in this repository**, and no file records +the paper's numeric accuracy/F1 table (the baseline investigation records only that the paper reports +"macro accuracy, accuracy, F1, precision", without values). Therefore the claimed deviations +(X-IIoTID ≈ 5.1–9.7 percentage points below the paper; 5G-NIDD ≈ 4.6–8.4 percentage points above the +paper) **could not be verified** here. Per the constraints, this is documented as a gap rather than +reconstructed. + +**Classification.** Requires the external source (`journal/main.tex` / the published paper and +thesis). Even once sourced, such deviations must be reported as **reproduction differences**, not as +automatic evidence of an implementation bug: metric definitions, train/test split (the project uses +70/10/20 stratified, the paper 80/20), class handling (one dropped class on X-IIoTID), seed +initialization, and nearest-prototype inference all differ from the paper and can move accuracy +independently of any defect. No defect is claimed. + +--- + +## 8. Metric-selection versus headline-metric audit + +Traced in code: + +- **Selection metric** (`scripts/run_baseline.py`, line ~303): `val_rare_class_macro_f1_per_client_mean` + — the mean over eligible clients of each client's own validation rare-class macro-F1. Pooled + selection across seeds uses this (`scripts/analyze_multiseed_beta.py`). +- **Headline test metric**: `test_rare_class_macro_f1` from `rare_class_metrics(test_labels, test_pred, + num_classes, union_rare)` — a **global** rare-class macro-F1 pooled over the union of client rare + classes. + +**Consequence.** The method is *selected* on a **per-client-mean validation** rare-class macro-F1 but +*reported* on a **global test** rare-class macro-F1. These are not the same estimator: the per-client +mean weights every client equally (and can be dominated by small clients), whereas the global metric +weights by evaluation support. A beta that is best under the selection criterion need not be best +under the reported criterion. The two quantities are therefore reported separately in this thesis +(validation selection tables vs test evaluation tables) and must never be conflated; neither uses the +test set for selection. + +--- + +## 9. What can and cannot be concluded + +**Supported by the evidence.** + +- The comparison is a valid paired design with verified common manifests and splits. +- Test rare-class macro-F1 changes are **small (≤ ~0.023)** and **condition-dependent**; only + X-IIoTID α=0.50 improves in all three seeds. +- Rare-class recall tends to fall when rare-class F1 rises (precision–recall trade-off). +- The validation-selected beta is **weakly separated** and its pooled choice agrees with at most 2/3 + seeds; it is "best among tested candidates", not robust or optimal. +- The `S_j` rule is materially exercised (53/162 cells) and conditions all results. + +**Not supported / cannot be concluded.** + +- Statistical superiority or significance (min attainable sign-flip p = 0.25). +- That the method has *no* effect (non-significance ≠ equivalence). +- A universal or globally optimal beta. +- That prototype displacement implies better detection. +- Any cross-condition pooling as if the 18 cells were independent replicates. +- The paper-comparison deviation magnitudes (source absent). +- Any privacy claim from the `N_ij` flow without a threat model. + +--- + +## 10. Recommended replacement wording for thesis section A.7 (Indonesian) + +> **Keterbatasan bukti statistik.** Grid eksperimen yang telah diselesaikan terdiri atas enam +> kondisi (dua dataset × tiga nilai `α`) dengan tiga seed per kondisi, sehingga setiap perbandingan +> berpasangan antara `original_protean` dan metode usulan hanya memiliki tiga selisih. Dengan tiga +> selisih tersebut, uji permutasi tanda dua sisi yang eksak sekalipun tidak dapat menghasilkan nilai +> p di bawah 0,25; karena itu hasil eksperimen ini dilaporkan secara deskriptif (nilai tengah, +> arah perubahan, dan konsistensi antar-seed), bukan sebagai klaim signifikansi. Ketidaksignifikanan +> tidak boleh ditafsirkan sebagai bukti bahwa metode tidak berpengaruh, melainkan sebagai +> keterbatasan daya uji akibat jumlah seed yang kecil. Perbedaan hasil yang teramati juga kecil dan +> bergantung pada kondisi; hanya kondisi X-IIoTID `α`=0,50 yang konsisten membaik pada ketiga seed. +> Pemilihan `β` dilakukan sepenuhnya pada data validasi, dengan margin antara kandidat terbaik dan +> kedua terbaik yang kecil (0,0024–0,0122), sehingga nilai `β` terpilih dinyatakan sebagai yang +> terbaik di antara kandidat yang diuji pada prosedur validasi, bukan sebagai nilai optimal global +> maupun pilihan yang kokoh. Selain itu, aturan peserta agregasi `S_j` merupakan asumsi reproduksi +> yang secara material berpengaruh pada 53 dari 162 sel kelas-kondisi; seluruh hasil bersifat +> kondisional terhadap asumsi tersebut. Generalisasi juga terbatas pada dua dataset, sepuluh +> client, satu arsitektur, tiga tingkat `α`, tiga seed, dan lima kandidat `β` diskret. + +*(This is proposed wording for review; no existing thesis document was edited.)* + +--- + +## 11. Reproducibility + +```bash +# post-hoc audit (read-only; no training; writes posthoc__*.csv only) +.venv/bin/python scripts/posthoc_statistical_audit.py +``` + +The script: +1. re-verifies pairing (dataset, alpha, seed, `manifest_content_hash`, recorded beta) for all 18 + cells and aborts on any mismatch; +2. writes `posthoc__paired_global_per_seed.csv`, `posthoc__paired_global_by_condition.csv`, + `posthoc__paired_test_results.csv`, `posthoc__selection_stability.csv`, + `posthoc__per_class_summary.csv`, `posthoc__per_client_summary.csv`; +3. refuses to overwrite any existing artifact. + +Inputs are the committed `deepcmp__*`, `final__*`, `*__multiseed__validation_per_seed.csv`, and +`sj_participant_rule_analysis.csv` tables plus per-run `metadata.json`. No network, no training, no +partition regeneration. + +--- + +## 12. Unresolved data gaps + +1. **`journal/main.tex` / paper Table IV absent** — the paper's reported accuracy/F1 values cannot + be verified from the repository; the reproduction-deviation magnitudes remain unconfirmed. +2. **No official PROTEAN code** — the `S_j` participant rule and missing-class handling remain a + reproduction assumption (material in 53/162 cells). +3. **Three seeds only** — inferential power is fundamentally limited; no significant result is + attainable and bootstrap/interval estimates would be unreliable. +4. **One missing validation artifact set** for `xiiotid__alpha0.5__seed0__original_protean` + (documented in the README) — does not affect the six selections but limits that cell's + validation-side diagnostics. +5. **Metric mismatch** between the per-client validation selection criterion and the global test + headline metric cannot be removed post-hoc; it is a design/protocol property. + +--- + +## Appendix — Limitations classification (task Phase 5) + +| Item | Classification | +|---|---| +| A. Limited seeds / statistical evidence | Computable (done: descriptive + exact tests) and a **design-scope limitation** for significance | +| B. `S_j` participant rule (53/162) | **Requires source-code/protocol verification** (no official code) → documented reproduction assumption | +| C. Baseline reproduction discrepancy | **Requires the external source** (`journal/main.tex` / paper table; absent here); report as reproduction difference, not a defect | +| D. Selection margins and seed agreement | **Computable** (done; margins 0.0024–0.0122 verified) | +| E. Selection metric vs headline metric | **Computable/traceable in code** (confirmed: validation per-client mean vs global test rare-F1) | +| F. Experimental scope (2 datasets, 10 clients, 1 architecture, 3 α, 3 seeds, 5 betas) | **Design-scope limitation** — cannot be removed post-hoc; needs additional experiments | +| G. Support information / privacy (`N_ij` at server) | **Requires protocol verification (done: `N_ij` is sent) + threat-model analysis** for any privacy claim; no vulnerability is claimed here | diff --git a/kisikisithesis/ROBUSTNESS_AUDIT_DAN_PERSIAPAN_PENGUJI.md b/kisikisithesis/ROBUSTNESS_AUDIT_DAN_PERSIAPAN_PENGUJI.md new file mode 100644 index 0000000..3e0e50d --- /dev/null +++ b/kisikisithesis/ROBUSTNESS_AUDIT_DAN_PERSIAPAN_PENGUJI.md @@ -0,0 +1,454 @@ +# Audit Ketangguhan (Robustness) dan Persiapan Penguji Tesis PROTEAN + +Dokumen ini merupakan audit berbasis bukti dan bahan persiapan sidang untuk penelitian +**agregasi prototype terbobot pada PROTEAN**. Seluruh angka diambil dari artefak eksperimen yang +telah selesai; tidak ada pelatihan ulang, tidak ada perubahan pada algoritma, model, loss, +optimizer, dataset, partisi, maupun hasil eksperimen. Skrip pendukung berada di +`kisikisithesis/robustness_checks.py` dan bersifat deterministik serta hanya-baca. + +Status repositori saat audit: branch `main`, HEAD `34bab45`, tidak ada perubahan pada berkas +terlacak. Direktori `kisikisithesis/` seluruhnya baru. + +--- + +## 1. Ringkasan Eksekutif + +1. **Mekanisme yang diubah hanya agregasi prototype di sisi server.** Bobot client + `w_ij(β) = N_ij^β / Σ_{k∈S_j} N_kj^β` dengan `β ∈ {0, 0.25, 0.50, 0.75, 1.00}`; model lokal, + loss, optimizer, inferensi, dataset, dan manifest partisi tetap. +2. **Sifat yang dijamin oleh rumus** (diverifikasi sintetis, `robustness_checks.py`): + `β=0` = rata-rata aritmetik atas `S_j`; `β=1` = rata-rata terbobot dukungan; satu penyumbang + invarian terhadap `β`; dukungan seragam invarian terhadap `β`; bobot selalu berjumlah 1; + `N_ij=0` dan prototype hilang dikeluarkan dari `S_j`; `β` tak valid ditolak. +3. **Efek langsung vs efek akhir-ke-akhir berbeda.** Displacement prototype `‖C̄_j(β)−C̄_j(0)‖` + naik monoton terhadap `β` (X-IIoTID 1.305–3.422; 5G-NIDD 1.385–2.919 pada `β=1`), tetapi + displacement besar **bukan** bukti deteksi membaik. +4. **Hasil akhir-ke-akhir kecil dan bergantung kondisi.** Perbedaan rare-class macro-F1 uji: + −0,0007 / +0,0230 / +0,0014 (X-IIoTID) dan −0,0010 / −0,0086 / +0,0168 (5G-NIDD). Hanya + X-IIoTID α=0,50 yang arahnya konsisten pada ketiga seed. +5. **Daya uji statistik tidak memadai.** Dengan 3 seed, p terkecil yang mungkin untuk uji tanda + eksak dua sisi adalah **0,25** (diverifikasi); hanya X-IIoTID α=0,50 mencapainya. Maka tidak + ada klaim signifikansi. +6. **Pemilihan β rapuh secara relatif.** Margin terbaik vs terbaik kedua hanya **0,0024–0,0122**, + dan kesepakatan seleksi pooled dengan seleksi per-seed paling banyak **2 dari 3**. +7. **Asumsi reproduksi `S_j` material.** `|S_j| < M` pada **53 dari 162** sel kelas-kondisi. +8. **Reproduksi vs paper belum dapat diverifikasi** karena `journal/main.tex` (dan Table IV paper) + tidak ada di repositori ini. + +Kalimat klaim yang aman: *"Di antara kandidat β yang diuji, pembobotan dukungan sampel kelas +memberi perubahan rare-class macro-F1 yang kecil dan bergantung kondisi; hanya sebagian kondisi +menunjukkan perbaikan yang konsisten antar-seed, dan tidak ada klaim signifikansi statistik."* + +--- + +## 2. Inventaris Repositori dan Bukti + +Sumber utama (semua sudah tersimpan, hanya-baca): + +| Keperluan | Artefak | +|---|---| +| Protokol beku | `docs/research/EXPERIMENT_PROTOCOL.md`, `configs/experiments/experiment_protocol.yaml` | +| Implementasi agregasi | `src/protean_research/aggregation/prototype_aggregation.py` | +| Efek agregasi langsung | `src/protean_research/analysis/direct_prototype.py`, `scripts/analyze_prototype_effect.py` | +| Snapshot prototype | `artifacts/analysis/*prototype_snapshot.json` (108 berkas; 18 sel × 6 konfigurasi) | +| Partisi beku | `artifacts/partitions/*.json` (18 manifest) | +| Perbandingan berpasangan | `artifacts/analysis/deepcmp__*.csv` | +| Ringkasan enam kondisi | `artifacts/analysis/final__six_condition_master.csv` | +| Kurva seleksi | `final__pooled_validation_curves.csv`, `final__selection_strength.csv` | +| Sensitivitas β (tabel) | `final__prototype_displacement_by_beta.csv`, `..._by_class.csv` | +| Heterogenitas terealisasi | `final__realized_heterogeneity.csv` | +| Aturan `S_j` | `artifacts/analysis/sj_participant_rule_analysis.csv` | +| Ekuivalensi β=0 | `*__multiseed__beta0_equivalence.csv` (5 kondisi; X-IIoTID α=0,50 tidak ada) | +| Audit statistik | `docs/research/POSTHOC_STATISTICAL_AUDIT.md`, `artifacts/analysis/posthoc__*.csv` | +| Metrik per-run | `results//test_global_metrics.json`, `test_per_class.csv`, `per_client.csv` | +| Validasi per-seed | `artifacts/analysis/__alpha*__multiseed__validation_per_seed.csv` | +| Baseline tereproduksi | `docs/baseline/BASELINE_VALIDATION.md`, `PROTEAN_BASELINE_INVESTIGATION.md` | + +**Batas bukti yang harus dicatat:** +- `journal/main.tex` / Table IV paper **tidak ada** → perbandingan metrik paper tidak dapat + diverifikasi. +- `xiiotid__alpha0.5__multiseed__beta0_equivalence.csv` **tidak ada**, dan satu run + `xiiotid__alpha0.5__seed0__original_protean` kehilangan artefak validasi. +- Hanya **3 seed** per kondisi. +- Pemilihan β memakai **rata-rata per-client (validasi)**, sedangkan headline memakai + **rare-class macro-F1 global (uji)**. + +Analisis yang **dapat** dihitung dari artefak: seluruh perbandingan berpasangan, displacement +langsung, stabilitas seleksi, audit `S_j`, ringkasan per-kelas/per-client. +Analisis yang **memerlukan pelatihan baru**: menambah seed, dataset, arsitektur, atau kandidat β +kontinu. + +--- + +## 3. Penjelasan Matematis β dan Bobot Client (terverifikasi) + +Diberikan himpunan peserta `S_j = { i : N_ij > 0 dan prototype lokal C_ij valid }`. Bobot +ternormalisasi dan prototype global: + +``` +w_ij(β) = N_ij^β / Σ_{k∈S_j} N_kj^β +C̄_j(β) = Σ_{i∈S_j} w_ij(β) · C_ij +``` + +**β = 0.** `N_ij^0 = 1` sehingga `w_ij = 1/|S_j|` dan `C̄_j(0)` adalah rata-rata aritmetik atas +`S_j`. Ini adalah **kontrol setara**, bukan otomatis sama dengan baseline PROTEAN: kesetaraan hanya +berlaku bila aturan peserta/penanganan kelas-tak-tersedia juga sama. + +**β = 1.** `w_ij ∝ N_ij`. Karena setiap `C_ij` adalah rata-rata embedding kelas lokal, `C̄_j(1)` +setara dengan rata-rata gabungan seluruh embedding kelas `j` yang disertakan bila seluruh asumsi +terpenuhi. + +**0 < β < 1.** Bobot tumbuh sub-linear terhadap dukungan: client berdukungan besar lebih +berpengaruh, tetapi tidak sepenuhnya proporsional. + +**Mengapa β besar tidak menjamin deteksi lebih baik.** Agregasi hanya menghasilkan posisi +`C̄_j(β)`; inferensi memilih prototype terdekat (jarak L2). Meningkatnya bobot client berdukungan +besar dapat menggeser `C̄_j` ke arah cluster padat tetapi **kurang mewakili** variasi serangan +langka, sehingga menurunkan recall kelas langka. Jadi peningkatan `β` mengubah geometri prototype +secara deterministik, tetapi hubungannya dengan kinerja deteksi bersifat empiris dan bergantung +data. + +**N_ij = 0.** Client tidak menyumbang prototype untuk kelas tersebut dan tidak masuk `S_j`; +implementasi tidak pernah membuat vektor nol (`aggregate_class_prototypes`, baris penyaring +`support <= 0 atau proto is None`). + +**Mengapa penyebut dan aturan peserta penting.** Bila Eq. (2) paper (`1/M` atas semua client) +diterapkan harfiah dengan mengisi prototype hilang sebagai nol, hasilnya berbeda dari `1/|S_j|`. +Pada **53/162** sel kelas-kondisi, `|S_j| < M`, sehingga aturan ini material dan seluruh hasil +bersifat kondisional. + +**Sifat yang dijamin rumus** (diverifikasi): normalisasi (Σw=1), endpoint β, invariansi satu +penyumbang/dukungan seragam, pengecualian `N_ij=0`, penolakan β tak valid. +**Sifat yang bergantung data/implementasi**: besar displacement per kelas, client mana yang +dominan, dan apakah deteksi membaik. + +--- + +## 4. Temuan Sensitivitas Parameter yang Ada (efek agregasi langsung) + +Dihitung ulang dari snapshot round-0 yang tersimpan oleh `kisikisithesis/robustness_checks.py` +(tanpa pelatihan, snapshot tidak diubah). + +**Tabel kondisi (mean displacement `‖C̄_j(1)−C̄_j(0)‖`, dari +`final__prototype_displacement_by_beta.csv`):** + +| Dataset | α=0.25 | α=0.50 | α=0.75 | +|---|---|---|---| +| X-IIoTID | 3.422 (β=0.25: 1.476; β=0.50: 2.403; β=0.75: 3.003) | 2.099 | 1.305 | +| 5G-NIDD | 2.919 (1.295 / 2.055 / 2.559) | 2.340 | 1.385 | + +Displacement naik **monoton** terhadap β di setiap kondisi; makin kecil α (makin timpang), +makin besar displacement. + +**Contoh sensitivitas per kelas (5G-NIDD α=0.25 seed 0, snapshot β=0):** + +| Kelas | \|S_j\| | N_min | N_max | disp@0.5 | disp@1.0 | bobot client terbesar @β=1 | +|---|---|---|---|---|---|---| +| HTTPFlood | 9 | 1 | 48.469 | 3.895 | 4.727 | 0.492 | +| TCPConnectScan | 9 | 2 | 10.870 | 4.040 | 6.173 | 0.774 | +| SlowrateDoS | 9 | 1 | 16.517 | 3.606 | 4.216 | 0.323 | +| SYNFlood | 7 | 144 | 4.762 | 1.151 | 2.726 | 0.700 | +| Benign | 10 | 1 | 185.504 | 2.335 | 3.229 | 0.555 | + +Pada snapshot ini, mean disp@1.0 untuk kelas `|S_j|0): perolehan F1 + cenderung berasal dari presisi. +- **Uji eksak (n=3):** sign-flip p = 0.75 / **0.25** / 0.75 / 0.75 / 1.00 / 0.50; Wilcoxon p = + 1.00 / 0.50 / 1.00 / 1.00 / 1.00 / 1.00. p terkecil yang mungkin = **0.25**. +- **Confusion matrix** (5G-NIDD α=0.75 seed0): error dominan `Benign → UDPFlood` nyaris tetap + (56.899 → 57.054; **+155**, bukan perbaikan); error `HTTPFlood → SlowrateDoS` turun 547; + `Benign → TCPConnectScan` turun 520; `SlowrateDoS → HTTPFlood` naik 229. +- **Per kelas** (mean ΔF1 atas semua 9 observasi): Tampering +0.052, Reconnaissance +0.030, + ICMPFlood +0.027; SYNFlood −0.019, C&C −0.018. Konsistensi antar-seed tidak merata. +- **Per client:** redistribusi — sebagian client naik, sebagian turun; pada α=0.25 beta=0 semua + client "tidak berubah". Analisis dukungan rendah bersifat deskriptif (Pearson Δ vs fraksi + dukungan ≈ −0,09; n=120). + +**Diagnostik seleksi:** margin 0,0024–0,0122; kesepakatan pooled vs per-seed ≤2/3; pooled +per-seed: X 0.75/0.50/0.00; 0.50/1.00/0.75; 0.50/0.50/0.00 dan 5G 0.25/0.00/0.00; 1.00/0.00/1.00; +0.50/1.00/1.00. + +**Eksperimen yang dikecualikan:** kondisi X-IIoTID α=0,50 tidak memiliki artefak ekuivalensi β=0 +dan kehilangan artefak validasi untuk original seed 0; analisis langsung memakai sel yang lengkap. + +--- + +## 6. Pertanyaan Penguji dan Jawaban (≥20) + +Format: **Jawaban lisan (ID)** · *Alasan teknis* · *Bukti* · *Tindak lanjut* · *Peringatan*. + +**Q1. Apa yang berubah jika β = 0, 0,5, atau 1?** +Jawab: "β=0 berarti rata-rata sederhana atas client penyumbang; β=1 berarti kontribusi +proporsional terhadap jumlah sampel kelas; β=0,5 adalah kompromi sub-linear." *Alasan:* rumus di +§3. *Bukti:* `test_beta0_is_equal_arithmetic_mean_over_s_j`, `test_beta1_is_support_weighted_mean`; +`robustness_checks.py` (β=0 → [1.333,1.667]; β=1 → [1.857,2.286]; β=0,5 → [1.586,2.000]). +*Tindak lanjut:* "Apakah β=0 identik dengan PROTEAN asli?" → lihat Q13. *Peringatan:* perubahan +prototype tidak otomatis mengubah kinerja deteksi. + +**Q2. Mengapa memilih himpunan kandidat β ini?** +Jawab: "Lima kandidat diskret menutupi dua ujung dan tiga titik antara agar eksperimen tetap +terkendali dan terjangkau, sesuai spesifikasi riset." *Alasan:* protokol membekukan +`{0,0.25,0.5,0.75,1.0}`. *Bukti:* `EXPERIMENT_PROTOCOL.md`, `test_beta_candidates_are_the_frozen_protocol_values`. +*Tindak lanjut:* "Mengapa tidak β kontinu?" *Peringatan:* belum ada analisis β kontinu. + +**Q3. Apa yang terjadi bila partisi makin/tidak makin non-IID?** +Jawab: "Makin kecil α, dukungan kelas makin timpang dan displacement prototype makin besar; +sebaliknya pada α besar hampir seragam." *Alasan & bukti:* `final__realized_heterogeneity.csv`, +`final__prototype_displacement_by_beta.csv` (displacement β=1 turun saat α naik). *Tindak lanjut:* +"Apakah α menyebabkannya?" *Peringatan:* hanya asosiasi deskriptif; α adalah knob, bukan +heterogenitas terealisasi. + +**Q4. Bagaimana bila kelas langka absen di sebagian besar client?** +Jawab: "Kelas itu hanya diagregasi dari client yang memilikinya (`S_j`); bila tidak ada penyumbang, +kelas dihilangkan dari hasil agregasi." *Bukti:* `test_class_with_only_missing_contributors_is_omitted`, +`aggregate_class_prototypes`. *Tindak lanjut:* "Bagaimana pengaruhnya pada evaluasi rare-class?" +*Peringatan:* kelangkaan ekstrem membatasi daya metrik. + +**Q5. Apa yang terjadi ketika N_ij = 0?** +Jawab: "Client tidak masuk `S_j` dan tidak ada vektor nol yang dibuat." *Bukti:* +`aggregate_class_prototypes` (penyaring `support <= 0`), `test_missing_class_is_excluded_from_s_j`. +*Tindak lanjut:* Q14. + +**Q6. Bagaimana bila satu client jauh lebih banyak sampelnya?** +Jawab: "Pada β tinggi bobotnya mendominasi; pada β=0 semua setara." *Bukti:* synthetic dominance +[1,1000] → bobot 0,999 bagian; snapshot TCPConnectScan bobot client terbesar ≈0,774 pada β=1. +*Tindak lanjut:* "Apakah ini selalu baik?" *Peringatan:* dominasi dapat menekan variasi kelas +langka. + +**Q7. Mengapa dukungan spesifik kelas, bukan total sampel client?** +Jawab: "Bobot diterapkan per kelas, jadi relevan jumlah sampel kelas tersebut; total client tidak +mewakili dukungan kelas tertentu." *Alasan:* `N_ij` didefinisikan per kelas. *Bukti:* +`prototype_aggregation.py` menggunakan `N_ij` per kelas. *Tindak lanjut:* "Bisakah total sampel +dipakai?" → mungkin sebagai pembanding, tetapi berbeda semantik. + +**Q8. Apakah prototype yang berubah pasti memperbaiki klasifikasi?** +Jawab: "Tidak." *Bukti:* displacement monoton naik terhadap β, sedangkan respons deteksi +non-monoton (`final__prototype_displacement_by_beta.csv` vs `final__pooled_validation_curves.csv`). +*Tindak lanjut:* Q9. *Peringatan:* jangan menyamakan displacement dengan kinerja. + +**Q9. Mengapa macro-F1 bisa naik tetapi recall/presisi turun?** +Jawab: "F1 adalah rata-rata harmonik; pergeseran prototype dapat mengubah trade-off +presisi–recall." *Bukti:* rare-class recall turun di 5/6 kondisi; contoh Tampering (ΔF1 +0,144, +Δpresisi +0,212, Δrecall −0,009 di X-IIoTID α=0,50). *Tindak lanjut:* "Apakah itu bermanfaat?" + +**Q10. Bagaimana bila β terpilih tidak terbaik di uji?** +Jawab: "Mungkin saja; seleksi memakai validasi, uji tidak boleh dipakai memilih β." *Bukti:* +`test_used_for_selection = false`; `final__selection_strength.csv`. *Tindak lanjut:* Q12. + +**Q11. Mengapa hanya tiga seed, dan apa yang tidak dapat disimpulkan?** +Jawab: "Tiga seed sesuai protokol proyek; belum cukup untuk klaim signifikansi." *Bukti:* p +terkecil yang mungkin = 0,25 (`posthoc__paired_test_results.csv`). *Tindak lanjut:* Q16. +*Peringatan:* ketidaksignifikanan ≠ tidak ada efek. + +**Q12. Apa implikasi margin seleksi kecil?** +Jawab: "Seleksi rapuh; β terpilih adalah 'terbaik di antara kandidat yang diuji', bukan optimal." +*Bukti:* margin 0,0024–0,0122; kesepakatan ≤2/3. *Tindak lanjut:* "Perlu seed tambahan?" + +**Q13. Mengapa β=0 dianggap setara dengan agregasi bobot seragam asli, dan bagaimana diverifikasi?** +Jawab: "β=0 secara matematis = rata-rata seragam atas `S_j`; kesetaraan dengan baseline asli hanya +bila aturan peserta sama." *Bukti:* `test_beta0_matches_original_protean_on_identical_s_j`; +`*__multiseed__beta0_equivalence.csv` (kelas B; `aggregation_max_abs_diff` ≈ 2e−16; snapshot +round-0 byte-identik). *Tindak lanjut:* "Mengapa tidak bitwise identik?" *Peringatan:* bukan bukti +reproduksi PROTEAN asli; masih ada divergensi floating-point end-to-end. X-IIoTID α=0,50 tidak +punya artefak ekuivalensi ini (celah). + +**Q14. Bagaimana perilaku bila |S_j| < jumlah client?** +Jawab: "Hanya client di `S_j` yang diagregasi; penyebut memakai jumlah bobot kontributor." +*Bukti:* `sj_participant_rule_analysis.csv` (53/162 sel), kode `aggregate_class_prototypes`. +*Tindak lanjut:* "Apa alternatifnya?" *Peringatan:* aturan ini asumsi reproduksi. + +**Q15. Bisakah pembobotan dukungan memperburuk kinerja? Mekanisme masuk akal?** +Jawab: "Bisa, secara prinsip." *Alasan:* pada β tinggi, client berdukungan besar mendominasi; +jika variasi kelas langka hanya ada di client berdukungan kecil, prototype global bergeser menjauh +dari variasi itu → recall turun. *Bukti:* hal ini teramati pada **5G-NIDD α=0,50** (Δ = −0,0086) +dan pada α=0,25 (β=1 paling merugikan di tingkat client, dari sintesis). *Peringatan:* contoh lain +bersifat mekanisme hipotetis, bukan hasil teramati. + +**Q16. Eksperimen tambahan apa yang paling langsung menguji reprodusibilitas efek?** +Jawab: "Menambah seed pada kondisi terkuat (X-IIoTID α=0,50) dan kondisi terlemah (5G-NIDD α=0,50), +lalu seleksi dan evaluasi berpasangan; idealnya juga beberapa arsitektur." *Alasan:* daya uji +bergantung jumlah seed. *Bukti:* §5. *Tindak lanjut:* Q21. + +**Q17. Asumsi apa yang berbeda dari paper PROTEAN?** +Jawab: "Penanganan kelas tak tersedia (`S_j`), split 70/10/20 (paper 80/20), nilai λ tanpa rujukan, +seed tak diberi paper, dan preprocessing yang tidak dispesifikasikan." *Bukti:* +`PROTEAN_BASELINE_INVESTIGATION.md`, `EXPERIMENT_PROTOCOL.md`. *Peringatan:* perbedaan reproduksi, +bukan otomatis bug. + +**Q18. Apa implikasi privasi karena server menerima N_ij?** +Jawab: "Server menerima hitungan dukungan per kelas selain prototype; ini memperluas informasi yang +diungkap dibanding hanya prototype." *Alasan & bukti:* `client.py` menghasilkan `supports`; +`server.py` meneruskan `client_supports` ke agregasi. *Peringatan:* **belum ada analisis +threat-model/privasi**, jadi tidak boleh diklaim sebagai kerentanan; ini hanya deskripsi aliran data. + +**Q19. Parameter mana yang tetap, mana yang disetel, mana bagian metode usulan?** +Jawab: "Tetap: model, loss, optimizer, inferensi, dataset, partisi, arsitektur. Disetel: β. +Bagian metode usulan: bobot `N_ij^β` pada agregasi prototype server." *Bukti:* §3, +`prototype_aggregation.py`, protokol. + +**Q20. Apa yang akan menggugurkan klaim empiris utama?** +Jawab: "Bila pada seed tambahan perbaikan X-IIoTID α=0,50 tidak muncul, atau bila β terpilih tidak +stabil, maka klaim 'perbaikan bergantung kondisi' harus diperlemah; klaim keunggulan universal +sudah tidak diajukan." *Bukti:* §5. *Peringatan:* tanpa seed tambahan, ini belum diuji. + +**Q21. Mengapa tidak menjalankan β kontinu?** +Jawab: "Di luar lingkup dan biaya; grid diskret sudah dibekukan." *Bukti:* protokol. *Tindak +lanjut:* sebagai pekerjaan lanjutan. + +**Q22. Apa arti 'dukungan sampel' dan mengapa bukan ukuran kualitas?** +Jawab: "`N_ij` adalah jumlah sampel training kelas j pada client i; bukan skor kualitas prototype." +*Bukti:* `AGENTS.md` §9/§10, `prototype_aggregation.py`. + +*(22 pertanyaan; mencakup seluruh daftar yang diminta.)* + +--- + +## 7. Rencana Demonstrasi Perubahan Parameter Terkendali + +Tidak dieksekusi otomatis oleh dokumen ini; `robustness_checks.py` telah menjalankan versi +read-only untuk sebagian skenario. Semua demo memakai input sintetis kecil atau snapshot tersimpan. + +| # | Skenario | Input terkendali | Hasil yang diharapkan (rumus) | Implementasi / tes yang ada | Perlu tes baru? | Penjelasan lisan | +|---|---|---|---|---|---|---| +| 1 | Endpoint β | `p=[[1,0],[0,2],[3,3]]`, `N=[5,10,20]` | β=0 → rata-rata; β=1 → terbobot | `test_beta0_...`, `test_beta1_...` | tidak | "Dua ujung rumus terbukti langsung." | +| 2 | Normalisasi bobot | support apa pun > 0 | Σw = 1 | `test_normalized_weights_sum_to_one` | tidak | "Bobot selalu terdefinisi." | +| 3 | Client dukungan nol | `N=[5,0,7]` | client 0 di luar `S_j` | `test_missing_class_is_excluded_from_s_j` | tidak | "Tanpa fabrikasi prototype." | +| 4 | Dukungan seragam | `N=[4,4,4]` | invarian β | `test_equal_support_is_invariant_to_beta` | tidak | "Tanpa ketimpangan, β tak berpengaruh." | +| 5 | Sangat timpang | `N=[1,1000]` | bobot ≈ [0,001; 0,999] | `robustness_checks.py` (dominance) | disarankan (unit test formal) | "Dominasi client besar." | +| 6 | Ekuivalen β=0 | `S_j` identik | sama dengan rata-rata seragam | `test_beta0_matches_original_protean_on_identical_s_j`; `beta0_equivalence.csv` | tidak | "Kontrol, bukan asli." | +| 7 | Stabilitas numerik / β tak valid | β ∈ {−0.1,1.1,None,"x",∞} | ValueError | `test_invalid_beta_is_rejected`, `test_output_is_finite_...` | tidak | "Validasi ketat." | +| 8 | Sensitivitas satu client dominan terhadap prototype | snapshot nyata + client dominan | pergeseran prototype | `client_weight_rows`, `prototype_effect_rows` | tidak | "Pergeseran terukur, bukan kinerja." | + +--- + +## 8. Audit Integritas Riset dan Batasan + +**Klaim yang didukung eksperimen.** Desain berpasangan tervalidasi (manifest/split/seed sama); +perbedaan rare-F1 uji kecil dan bergantung kondisi; hanya X-IIoTID α=0,50 konsisten positif; +recall kelas langka cenderung turun; seleksi β rapuh; `S_j` material (53/162). + +**Klaim yang hanya deskriptif.** Asosiasi timpang–displacement; redistribusi manfaat antar-client; +dukungan rendah tidak dirugikan (asosiasi lemah, tanpa kausalitas). + +**Asumsi reproduksi.** `S_j` dan penanganan kelas tak tersedia; split 70/10/20; λ=1,0; preprocessing; +seed proyek; inisialisasi prototype round-0. + +**Klaim yang harus dilemahkan/dihapus.** Superioritas/signifikansi; β optimal universal; bahwa +displacement = deteksi lebih baik; keunggulan umum lintas dataset. + +**Eksperimen tambahan yang memperkuat.** Penambahan seed (terutama X-IIoTID α=0,50 dan 5G α=0,50); +dataset ketiga; arsitektur kedua; analisis β kontinu; analisis threat-model privasi `N_ij`. + +**Analisis yang tidak layak dijalankan.** Uji inferensial dengan 3 seed (daya nihil); bootstrap +interval dari 3 seed; menjadikan client sebagai replikasi independen (nested). + +**Perhatian khusus:** +- **Tiga seed** → p minimum 0,25. +- **Celah reproduksi vs paper** → `journal/main.tex`/Table IV tidak ada; belum terverifikasi. +- **`S_j`** → 53/162 sel; asumsi, bukan statistik. +- **Metrik seleksi vs headline** → per-client mean (validasi) vs global (uji). +- **Displacement vs deteksi** → dipisahkan. +- **Privasi** → deskripsi aliran `N_ij`; belum ada threat-model. + +--- + +## 9. Eksperimen Lanjutan Prioritas (dengan estimasi komputasi) + +Estimasi berdasar runtime terukur (`artifacts/analysis/*__multiseed__runtime.csv`): 5G-NIDD +≈ 62–65 menit/run, X-IIoTID ≈ 40–45 menit/run; 108 run ≈ 94,5 jam CPU (8 core). + +| Prioritas | Eksperimen | Rasional | Estimasi biaya | +|---|---|---|---| +| 1 | Tambah seed (3→10) pada X-IIoTID α=0,50 dan 5G α=0,50 (kondisi terkuat & terlemah) | menguji reprodusibilitas efek terbesar & arah negatif | 2 kondisi × 7 seed × 6 config ≈ 84 run ≈ 70–75 jam | +| 2 | Tambah seed penuh (3→10) semua 6 kondisi | daya uji dan stabilitas seleksi | ≈ 252 run ≈ 210–220 jam | +| 3 | Dataset IDS ketiga | validitas eksternal | bergantung ukuran; ≈ 40–70 jam | +| 4 | Arsitektur alternatif (mis. MLP/1D-CNN berbeda) | sensitivitas arsitektur | ≈ 90–200 jam | +| 5 | Grid β kontinu (mis. 0.0–1.0 langkah 0.1) | resolusi kandidat | 6 kondisi × 3 seed × 6 β tambahan ≈ 108 run ≈ 90 jam | +| 6 | Analisis threat-model privasi `N_ij` | klaim privasi | tanpa pelatihan (analistik) | + +Tidak disarankan: uji signifikansi atau bootstrap dari 3 seed (informasi rendah, berisiko +menyesatkan). + +--- + +## 10. Cheat Sheet Pertahanan Lisan (satu halaman) + +- **Mekanisme:** hanya agregasi prototype server; `w_ij(β)=N_ij^β/ΣN_kj^β`; `β∈{0,.25,.5,.75,1}`. +- **β=0:** rata-rata seragam atas `S_j` (kontrol; setara baseline hanya bila aturan peserta sama). +- **β=1:** terbobot dukungan; **β antara:** sub-linear. +- **Hasil utama:** Δrare-F1 uji = X {−0,0007; +0,0230; +0,0014}, 5G {−0,0010; −0,0086; +0,0168}. +- **Konsisten antar-seed:** hanya X-IIoTID α=0,50. +- **Statistik:** n=3; p minimum 0,25; **tanpa klaim signifikansi**. +- **Seleksi:** margin 0,0024–0,0122; rapuh; validasi-saja (`test_used_for_selection=false`). +- **Direct vs end-to-end:** displacement naik monoton; deteksi non-monoton; dipisahkan. +- **`S_j`:** 53/162 sel; asumsi reproduksi. +- **Confusion dominan:** `Benign→UDPFlood` tetap (+155), bukan perbaikan. +- **Jangan klaim:** superioritas universal, β optimal, displacement = deteksi lebih baik, + signifikansi, kerentanan privasi tanpa threat-model. +- **Celah:** paper Table IV tidak tersedia; X-IIoTID α=0,50 tanpa ekuivalensi β=0; + artefak validasi seed0 original hilang. + +--- + +## 11. Path Artefak dan Perintah Reproduksi + +```bash +# Pemeriksaan formula + efek agregasi langsung (read-only, tanpa pelatihan) +.venv/bin/python kisikisithesis/robustness_checks.py + +# Audit statistik post-hoc (read-only; hanya menulis posthoc__*.csv) +.venv/bin/python scripts/posthoc_statistical_audit.py + +# Analisis efek prototype langsung (read-only) - contoh +.venv/bin/python scripts/analyze_prototype_effect.py \ + --snapshot artifacts/analysis/5gnidd__alpha0.25__seed0__support_weighted__beta0__prototype_snapshot.json \ + --class-names "Benign,HTTPFlood,ICMPFlood,SYNFlood,SYNScan,SlowrateDoS,TCPConnectScan,UDPFlood,UDPScan" \ + --output-dir /tmp/opencode/proto_effect # output di luar repo bila ingin menghindari file baru + +# Uji unit agregasi & efek (read-only) +.venv/bin/python -m pytest tests/test_support_weighted_aggregation.py tests/test_baseline_aggregation.py tests/test_prototype_effect.py +``` + +Artefak kunci: `artifacts/analysis/deepcmp__*.csv`, `final__*.csv`, +`posthoc__*.csv`, `sj_participant_rule_analysis.csv`, `*__multiseed__validation_per_seed.csv`, +`*__multiseed__beta0_equivalence.csv`, `artifacts/partitions/*.json`. + +--- + +## 12. Pertanyaan Tak Terjawab dan Bukti yang Hilang + +1. **Metrik paper PROTEAN (Table IV)** tidak dapat diverifikasi: `journal/main.tex` tidak ada. +2. **Ekuivalensi β=0 untuk X-IIoTID α=0,50** tidak tersedia (berkas tidak ada). +3. **Artefak validasi seed 0 original X-IIoTID α=0,50** tidak lengkap. +4. **Daya statistik** tidak memadai (3 seed); signifikansi tidak dapat diklaim. +5. **Privasi `N_ij`** belum dianalisis dengan threat-model. +6. **β kontinu** dan **generalisasi lintas dataset/arsitektur** belum diuji. +7. **Ketidakpastian seleksi β** tidak dapat dipisahkan sepenuhnya dari variasi seed dengan n=3. diff --git a/kisikisithesis/robustness_checks.py b/kisikisithesis/robustness_checks.py new file mode 100644 index 0000000..ccb04fa --- /dev/null +++ b/kisikisithesis/robustness_checks.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Deterministic, read-only robustness checks for the PROTEAN thesis. + +Run from the repository root: + + .venv/bin/python kisikisithesis/robustness_checks.py + +Nothing is written and no experimental artifact is modified. The script: + * exercises the *actual* aggregation implementation on tiny synthetic inputs + (formula invariants); + * recomputes direct aggregation effects on persisted round-0 prototype + snapshots (no training, no snapshot mutation). + +All values are printed deterministically so the thesis report can quote them. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from protean_research.aggregation.prototype_aggregation import ( # noqa: E402 + AggregationMode, + aggregate_prototypes, + aggregate_class_prototypes, + normalized_support_weights, + validate_beta, + BETA_CANDIDATES, +) +from protean_research.analysis.direct_prototype import ( # noqa: E402 + client_weight_rows, + prototype_effect_rows, +) + +ANALYSIS = ROOT / "artifacts" / "analysis" + + +def section(title: str) -> None: + print("\n" + "=" * 74) + print(title) + print("=" * 74) + + +def check(label: str, ok: bool) -> None: + print(f" [{'PASS' if ok else 'FAIL'}] {label}") + + +def synthetic_invariants() -> None: + section("A. Synthetic formula invariants (actual implementation)") + p = [[1.0, 0.0], [0.0, 2.0], [3.0, 3.0]] + s = [5.0, 10.0, 20.0] + + eq = [sum(v[d] for v in p) / len(p) for d in range(2)] + b0 = aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED, supports=s, beta=0.0) + print(f" beta=0 aggregate = {b0}") + print(f" arithmetic mean = {eq}") + check("beta=0 equals the arithmetic mean over S_j", + all(abs(a - b) < 1e-12 for a, b in zip(b0, eq))) + + w1 = normalized_support_weights(s, 1.0) + b1 = aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED, supports=s, beta=1.0) + closed = [sum(w * v[d] for w, v in zip(w1, p)) for d in range(2)] + print(f" beta=1 weights = {[round(w, 6) for w in w1]}") + print(f" beta=1 aggregate = {b1}") + check("beta=1 equals the support-weighted closed form", + all(abs(a - b) < 1e-12 for a, b in zip(b1, closed))) + + w05 = normalized_support_weights(s, 0.5) + b05 = aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED, supports=s, beta=0.5) + print(f" beta=0.5 weights = {[round(w, 6) for w in w05]}") + print(f" beta=0.5 aggregate = {b05}") + + single = aggregate_prototypes([[7.0, 9.0]], AggregationMode.SUPPORT_WEIGHTED, + supports=[3.0], beta=1.0) + check("single contributor is beta-invariant", single == [7.0, 9.0]) + + equal = [aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED, + supports=[4.0, 4.0, 4.0], beta=b) for b in BETA_CANDIDATES] + check("equal support is beta-invariant", all(g == equal[0] for g in equal)) + + for b in BETA_CANDIDATES: + tot = sum(normalized_support_weights(s, b)) + if abs(tot - 1.0) > 1e-12: + check(f"weights sum to 1 at beta={b}", False) + break + else: + check("normalized weights sum to 1 for every beta candidate", True) + + # zero-support exclusion and missing-class handling through the S_j builder + contributions = {0: [([1.0], 5.0), ([2.0], 0.0), (None, 7.0)], 1: [(None, 0.0)]} + agg = aggregate_class_prototypes(contributions, AggregationMode.SUPPORT_WEIGHTED, beta=1.0) + print(f" S_j builder output = {agg}") + check("N_ij=0 and missing prototypes are dropped from S_j", agg.get(0) == [1.0]) + check("a class with no valid contributor is omitted", 1 not in agg) + + dom = normalized_support_weights([1.0, 1000.0], 1.0) + print(f" dominance weights [1,1000] at beta=1 = {[round(w, 6) for w in dom]}") + check("a single dominant client can dominate the aggregate at beta=1", dom[1] > 0.999) + + rejected = 0 + for bad in (-0.1, 1.1, None, "x", float("inf")): + try: + validate_beta(bad) + except (ValueError, TypeError): + rejected += 1 + check("invalid beta values are rejected", rejected == 5) + + +def snapshot_sensitivity(tag: str, path: Path, names: dict) -> None: + section(f"B. Direct aggregation effect on a persisted snapshot: {tag}") + snap = json.loads(path.read_text()) + print(f" snapshot: {path.relative_to(ROOT)} (round {snap['round']})") + rows = prototype_effect_rows(snap, betas=[0.0, 0.5, 1.0], class_names=names) + weights = client_weight_rows(snap, betas=[0.0, 0.25, 0.5, 0.75, 1.0], class_names=names) + + by_class = {} + for r in rows: + by_class.setdefault(r["class_id"], {})[r["beta"]] = r + maxw = {} + for w in weights: + if w["beta"] == 1.0: + cur = maxw.get(w["class_id"]) + if cur is None or w["weight"] > cur["weight"]: + maxw[w["class_id"]] = w + + print(f" {'class':16} {'|S_j|':>5} {'sup_min':>8} {'sup_max':>9} " + f"{'disp@0.5':>9} {'disp@1.0':>9} {'maxW@1.0':>9}") + for cid in sorted(by_class): + r = by_class[cid] + n = r[0.0]["n_contributors"] + d05 = r[0.5]["displacement_from_beta0"] + d10 = r[1.0]["displacement_from_beta0"] + mw = maxw.get(cid, {}).get("weight", float("nan")) + name = (r[0.0]["class_name"] or cid) + print(f" {str(name):16} {n:>5} {r[0.0]['support_min']:>8.1f} " + f"{r[0.0]['support_max']:>9.1f} {d05:>9.4f} {d10:>9.4f} {mw:>9.4f}") + partial = [cid for cid in by_class if by_class[cid][0.0]["n_contributors"] < 10] + full = [cid for cid in by_class if by_class[cid][0.0]["n_contributors"] == 10] + if partial: + md = sum(by_class[c][1.0]["displacement_from_beta0"] for c in partial) / len(partial) + print(f" mean disp@1.0, |S_j| None: + synthetic_invariants() + snapshot_sensitivity( + "5G-NIDD, alpha=0.25, seed=0 (beta=0 snapshot)", + ANALYSIS / "5gnidd__alpha0.25__seed0__support_weighted__beta0__prototype_snapshot.json", + {i: n for i, n in enumerate( + ["Benign", "HTTPFlood", "ICMPFlood", "SYNFlood", "SYNScan", + "SlowrateDoS", "TCPConnectScan", "UDPFlood", "UDPScan"])}) + snapshot_sensitivity( + "X-IIoTID, alpha=0.25, seed=0 (beta=0 snapshot)", + ANALYSIS / "xiiotid__alpha0.25__seed0__support_weighted__beta0__prototype_snapshot.json", + {i: n for i, n in enumerate( + ["C&C", "Exfiltration", "Exploitation", "Lateral_movement", "Normal", + "RDOS", "Reconnaissance", "Tampering", "Weaponization"])}) + print("\nREAD-ONLY CHECKS COMPLETE (no files written, no snapshots modified).") + + +if __name__ == "__main__": + main() diff --git a/scripts/posthoc_statistical_audit.py b/scripts/posthoc_statistical_audit.py new file mode 100644 index 0000000..5f115ab --- /dev/null +++ b/scripts/posthoc_statistical_audit.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Post-hoc statistical audit of the completed PROTEAN experiment grid. + +Read-only post-hoc analysis: no training, no partition regeneration, no +modification of any existing artifact. Reads the audited comparison tables and +per-run metadata under ``results/`` and writes NEW derived tables with unique +``posthoc__`` names under ``artifacts/analysis/``. + +Computes: + * paired global per-seed differences (original_protean vs selected-beta + support_weighted) and condition-level descriptive summaries; + * exploratory paired tests (exact sign-flip permutation, exact Wilcoxon + signed-rank, paired t 95% CI) with explicit small-n limitations; + * validation beta-selection stability (pooled vs per-seed); + * per-class and per-client descriptive summaries. + +Run: + .venv/bin/python scripts/posthoc_statistical_audit.py +""" + +from __future__ import annotations + +import csv +import itertools +import json +import math +import statistics +from collections import defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ANALYSIS = ROOT / "artifacts" / "analysis" +RESULTS = ROOT / "results" + +DATASETS = ["xiiotid", "5gnidd"] +ALPHAS = [0.25, 0.50, 0.75] +SEEDS = [0, 1, 2] +METRICS = ["rare_class_macro_f1", "rare_class_recall", "macro_f1", + "weighted_f1", "accuracy"] +SELECTED = {("xiiotid", 0.25): 0.0, ("xiiotid", 0.5): 0.75, ("xiiotid", 0.75): 0.5, + ("5gnidd", 0.25): 0.0, ("5gnidd", 0.5): 1.0, ("5gnidd", 0.75): 0.5} +T_ZERO = 1e-6 # numerical "effectively zero" for sign counts +T_PRACTICAL = 0.02 # practical negligible band (matches deep comparison) +T975_DF2 = 4.302652729 # t_{0.975, df=2} + + +def read_csv(p: Path) -> list[dict]: + with p.open() as fh: + return list(csv.DictReader(fh)) + + +def write_csv(name: str, rows: list[dict]) -> None: + # Guard: this script may only ever write its own posthoc__ outputs, and may + # overwrite them so the analysis is exactly reproducible on rerun. It can + # never touch any pre-existing analysis artifact. + if not name.startswith("posthoc__"): + raise SystemExit(f"refusing to write non-posthoc artifact: {name}") + path = ANALYSIS / name + with path.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + print("wrote", path) + + +def mean(v): + return statistics.mean(v) + + +def std_sample(v): + return statistics.stdev(v) if len(v) > 1 else 0.0 + + +def median(v): + return statistics.median(v) + + +# --- exploratory exact tests ------------------------------------------------ +def signflip_p(deltas): + """Exact two-sided sign-flip permutation test on the mean (n<=~20).""" + d = list(deltas) + obs = abs(sum(d)) + if not d or all(x == 0 for x in d): + return 1.0 + total = 0 + ge = 0 + for signs in itertools.product((1, -1), repeat=len(d)): + s = abs(sum(sign * x for sign, x in zip(signs, d))) + total += 1 + if s >= obs - 1e-12: + ge += 1 + return ge / total + + +def wilcoxon_exact_p(deltas): + """Exact two-sided Wilcoxon signed-rank (min-rank convention), zeros dropped.""" + d = [x for x in deltas if abs(x) > T_ZERO] + n = len(d) + if n == 0: + return 1.0 + order = sorted(range(n), key=lambda i: abs(d[i])) + ranks = [0.0] * n + i = 0 + while i < n: + j = i + while j + 1 < n and abs(d[order[j + 1]]) == abs(d[order[i]]): + j += 1 + avg = (i + j) / 2 + 1 + for k in range(i, j + 1): + ranks[order[k]] = avg + i = j + 1 + w_pos = sum(ranks[i] for i in range(n) if d[i] > 0) + w_min_obs = min(w_pos, sum(ranks) - w_pos) + dist = defaultdict(int) + for signs in itertools.product((1, -1), repeat=n): + wp = sum(ranks[i] for i in range(n) if signs[i] > 0) + dist[min(wp, sum(ranks) - wp)] += 1 + le = sum(c for w, c in dist.items() if w <= w_min_obs + 1e-12) + return min(1.0, 2 * le / (2 ** n)) + + +def paired_t_ci(deltas): + d = list(deltas) + n = len(d) + m = mean(d) + s = std_sample(d) + half = T975_DF2 * s / math.sqrt(n) if n > 1 else 0.0 + return m - half, m + half + + +# --- pairing verification --------------------------------------------------- +def manifests(ds, alpha, seed, beta): + o = json.loads((RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__original_protean" + / "metadata.json").read_text()) + s = json.loads((RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}" + / "metadata.json").read_text()) + return o, s + + +def main() -> None: + per_seed = read_csv(ANALYSIS / "deepcmp__global_per_seed.csv") + + # 1. Verify pairing and selected beta, rebuild a per-seed table. ---------- + rebuilt = [] + pair_ok = True + for ds in DATASETS: + for alpha in ALPHAS: + beta = SELECTED[(ds, alpha)] + for seed in SEEDS: + o, s = manifests(ds, alpha, seed, beta) + ok = (o["dataset"] == s["dataset"] == ds + and abs(float(o["alpha"]) - alpha) < 1e-9 + and int(o["seed"]) == int(s["seed"]) == seed + and o["manifest_content_hash"] == s["manifest_content_hash"] + and abs(float(s["beta"]) - beta) < 1e-9) + pair_ok = pair_ok and ok + for m in METRICS: + row = next(r for r in per_seed + if r["dataset"] == ds and abs(float(r["alpha"]) - alpha) < 1e-9 + and int(r["seed"]) == seed and r["metric"] == m) + rebuilt.append({ + "dataset": ds, "alpha": alpha, "seed": seed, + "selected_beta": beta, "metric": m, + "original": row["original"], "proposed": row["proposed"], + "delta": row["delta"], + "manifest_content_hash": o["manifest_content_hash"], + "pairing_verified": ok}) + if not pair_ok: + raise SystemExit("pairing verification FAILED") + print("pairing verified for all 18 condition-seed cells (manifest hash match)") + write_csv("posthoc__paired_global_per_seed.csv", rebuilt) + + # 2. Condition-level descriptive summary. ------------------------------- + by_cond = [] + tests = [] + for ds in DATASETS: + for alpha in ALPHAS: + beta = SELECTED[(ds, alpha)] + for m in METRICS: + rows = [r for r in rebuilt if r["dataset"] == ds + and abs(r["alpha"] - alpha) < 1e-9 and r["metric"] == m] + base = [float(r["original"]) for r in rows] + prop = [float(r["proposed"]) for r in rows] + dlt = [float(r["delta"]) for r in rows] + npos = sum(1 for d in dlt if d > T_ZERO) + nneg = sum(1 for d in dlt if d < -T_ZERO) + nzero = sum(1 for d in dlt if abs(d) <= T_ZERO) + by_cond.append({ + "dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m, + "original_seed0": base[0], "original_seed1": base[1], "original_seed2": base[2], + "proposed_seed0": prop[0], "proposed_seed1": prop[1], "proposed_seed2": prop[2], + "delta_seed0": dlt[0], "delta_seed1": dlt[1], "delta_seed2": dlt[2], + "delta_mean": mean(dlt), "delta_std_sample": std_sample(dlt), + "delta_median": median(dlt), "delta_min": min(dlt), "delta_max": max(dlt), + "baseline_std_sample": std_sample(base), + "proposed_std_sample": std_sample(prop), + "n_pos": npos, "n_neg": nneg, "n_zero": nzero, + "same_sign_all_seeds": (npos == 3 or nneg == 3)}) + lo, hi = paired_t_ci(dlt) + tests.append({ + "dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m, + "n_pairs": len(dlt), + "deltas": "|".join(f"{d:.6f}" for d in dlt), + "signflip_exact_p_two_sided": signflip_p(dlt), + "wilcoxon_exact_p_two_sided": wilcoxon_exact_p(dlt), + "paired_t_ci95_low": lo, "paired_t_ci95_high": hi, + "min_attainable_signflip_p": 2 / (2 ** len(dlt)), + "note": ("exploratory; 3 seeds; no inferential power; " + "beta=0 conditions are numerical, not a treatment contrast")}) + write_csv("posthoc__paired_global_by_condition.csv", by_cond) + write_csv("posthoc__paired_test_results.csv", tests) + + # 3. Beta-selection stability. ----------------------------------------- + stability = [] + for ds in DATASETS: + for alpha in ALPHAS: + vs = read_csv(ANALYSIS / f"{ds}__alpha{alpha:g}__multiseed__validation_per_seed.csv") + scores = defaultdict(dict) # seed -> beta -> score + for r in vs: + scores[int(r["seed"])][float(r["beta"])] = float(r["val_rare_macro_f1_per_client_mean"]) + per_seed_beta = {s: max(scores[s], key=lambda b: (scores[s][b], -b)) for s in scores} + pooled = {b: mean([scores[s][b] for s in SEEDS]) for b in sorted(next(iter(scores.values())))} + pooled_beta = max(pooled, key=lambda b: (pooled[b], -b)) + ordered = sorted(pooled.values(), reverse=True) + agree = sum(1 for s in SEEDS if abs(per_seed_beta[s] - pooled_beta) < 1e-9) + second_beta = sorted((b for b in pooled if abs(b - pooled_beta) > 1e-9), + key=lambda b: -pooled[b])[0] + stability.append({ + "dataset": ds, "alpha": alpha, + "pooled_selected_beta": pooled_beta, + "per_seed_beta_s0": per_seed_beta[0], + "per_seed_beta_s1": per_seed_beta[1], + "per_seed_beta_s2": per_seed_beta[2], + "n_seeds_agree_pooled": agree, + "pooled_score_selected": pooled[pooled_beta], + "second_best_beta": second_beta, + "second_best_score": pooled[second_beta], + "margin_best_minus_second": pooled[pooled_beta] - pooled[second_beta], + "candidate_score_range": ordered[0] - ordered[-1], + "test_used_for_selection": False}) + write_csv("posthoc__selection_stability.csv", stability) + + # 4. Per-class summary. -------------------------------------------------- + pc = read_csv(ANALYSIS / "deepcmp__per_class.csv") + grp = defaultdict(list) + for r in pc: + grp[(r["dataset"], r["class_name"])].append(float(r["delta_f1"])) + per_class = [] + for (ds, name), vals in sorted(grp.items(), key=lambda kv: -mean(kv[1])): + per_class.append({ + "dataset": ds, "class_name": name, "n_obs": len(vals), + "mean_delta_f1": mean(vals), "median_delta_f1": median(vals), + "min_delta_f1": min(vals), "max_delta_f1": max(vals), + "n_positive": sum(1 for v in vals if v > 0), + "n_negative": sum(1 for v in vals if v < 0)}) + write_csv("posthoc__per_class_summary.csv", per_class) + + # 5. Per-client summary (descriptive only). ------------------------------ + cl = read_csv(ANALYSIS / "deepcmp__per_client.csv") + by_condition = defaultdict(list) + for r in cl: + by_condition[(r["dataset"], float(r["alpha"]))].append(r) + per_client = [] + for (ds, alpha), rows in sorted(by_condition.items()): + d = [float(r["delta_rare_f1"]) for r in rows] + per_client.append({ + "dataset": ds, "alpha": alpha, "selected_beta": SELECTED[(ds, alpha)], + "n_client_seed_obs": len(d), + "n_improved_gt_0.02": sum(1 for x in d if x > T_PRACTICAL), + "n_degraded_lt_-0.02": sum(1 for x in d if x < -T_PRACTICAL), + "n_unchanged": sum(1 for x in d if abs(x) <= T_PRACTICAL), + "mean_delta": mean(d), "median_delta": median(d), + "min_delta": min(d), "max_delta": max(d), + "std_sample_within_condition": std_sample(d)}) + write_csv("posthoc__per_client_summary.csv", per_client) + + # --- console summary ---------------------------------------------------- + print("\n=== paired rare-class macro-F1 (delta = proposed - original) ===") + for r in by_cond: + if r["metric"] != "rare_class_macro_f1": + continue + print(f" {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} " + f"mean={float(r['delta_mean']):+.4f} std={float(r['delta_std_sample']):.4f} " + f"base_std={float(r['baseline_std_sample']):.4f} prop_std={float(r['proposed_std_sample']):.4f} " + f"pos/neg/zero={r['n_pos']}/{r['n_neg']}/{r['n_zero']} same_sign={r['same_sign_all_seeds']}") + print("\n=== exact tests (rare-class macro-F1) ===") + for r in tests: + if r["metric"] != "rare_class_macro_f1": + continue + print(f" {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} " + f"signflip_p={r['signflip_exact_p_two_sided']:.4f} " + f"wilcoxon_p={r['wilcoxon_exact_p_two_sided']:.4f} " + f"tCI=[{r['paired_t_ci95_low']:+.4f},{r['paired_t_ci95_high']:+.4f}] " + f"(min signflip p={r['min_attainable_signflip_p']:.4f})") + print("\n=== selection stability ===") + for r in stability: + print(f" {r['dataset']:8} a{r['alpha']:<4} pooled={r['pooled_selected_beta']:<4} " + f"per-seed={r['per_seed_beta_s0']}/{r['per_seed_beta_s1']}/{r['per_seed_beta_s2']} " + f"agree={r['n_seeds_agree_pooled']}/3 margin={float(r['margin_best_minus_second']):.4f} " + f"range={float(r['candidate_score_range']):.4f}") + print("\n=== S_j audit ===") + sj = read_csv(ANALYSIS / "sj_participant_rule_analysis.csv") + lt = sum(1 for r in sj if int(r["S_j"]) < int(r["M"])) + print(f" |S_j| < M cells: {lt} / {len(sj)}") + print("\n=== per-class extremes (mean delta F1) ===") + print(" top +:", ", ".join(f"{r['dataset']}/{r['class_name']} {float(r['mean_delta_f1']):+.4f}" + for r in per_class[:4])) + print(" top -:", ", ".join(f"{r['dataset']}/{r['class_name']} {float(r['mean_delta_f1']):+.4f}" + for r in per_class[-4:])) + + +if __name__ == "__main__": + main()