research: add post-hoc statistical audit and defense preparation aids
- scripts/posthoc_statistical_audit.py + docs/research/POSTHOC_STATISTICAL_AUDIT.md: read-only paired audit of the completed grid (exact sign-flip / Wilcoxon, selection stability, per-class/per-client summaries); writes only posthoc__*. - kisikisithesis/: robustness audit and examiner-preparation document with deterministic read-only checks (robustness_checks.py). - artifacts/analysis/posthoc__*.csv: audit outputs (6 tables). - artifacts/analysis: track the 4 remaining beta0-snapshot effect/weight CSVs (xiiotid a0.5 seeds 1-2), completing the tracked beta0 set. - .gitignore: codify that non-beta0 prototype snapshots stay untracked.
This commit is contained in:
willtanoe committed
1 parent
34bab45865
commit
111adb272e
15 files changed
+2431
No files matched your search
@@ -14,6 +14,11 @@ venv/
|
||||
# NOTE: artifacts/partitions/ and artifacts/analysis/ are intentionally NOT ignored:
|
||||
# partition manifests are deterministic, content-hashed, and required for paired
|
||||
# comparisons; analysis outputs are small. They are intended to be version controlled.
|
||||
# EXCEPTION: the 18 beta0-round-0 prototype snapshots (and their effect/weight
|
||||
# CSVs) ARE tracked; the remaining ~76 snapshot JSONs (~28 MB, needed only to
|
||||
# regenerate already-committed displacement tables) stay local-only like
|
||||
# results/. They are cited by path in the audit documents; do not stage them.
|
||||
artifacts/analysis/*__prototype_snapshot.json
|
||||
artifacts/processed/
|
||||
artifacts/checkpoints/
|
||||
results/
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
dataset,alpha,selected_beta,metric,original_seed0,original_seed1,original_seed2,proposed_seed0,proposed_seed1,proposed_seed2,delta_seed0,delta_seed1,delta_seed2,delta_mean,delta_std_sample,delta_median,delta_min,delta_max,baseline_std_sample,proposed_std_sample,n_pos,n_neg,n_zero,same_sign_all_seeds
|
||||
xiiotid,0.25,0.0,rare_class_macro_f1,0.6398103625587044,0.5825215614975117,0.6760389917828752,0.6413622282749385,0.5805553678969027,0.6744891962601014,0.0015518657162341132,-0.001966193600609012,-0.0015497955227737892,-0.0006547078023828959,0.0019222569702312445,-0.0015497955227737892,-0.001966193600609012,0.0015518657162341132,0.04715228886220545,0.04764177856302672,1,2,0,False
|
||||
xiiotid,0.25,0.0,rare_class_recall,0.8296012943628512,0.8266995877156649,0.8095946724338723,0.830700667573069,0.8256994230374527,0.807370314447251,0.0010993732102178022,-0.0010001646782121787,-0.0022243579866212393,-0.0007083831515385386,0.0016809668681122992,-0.0010001646782121787,-0.0022243579866212393,0.0010993732102178022,0.010810973971484781,0.012283281629867224,1,2,0,False
|
||||
xiiotid,0.25,0.0,macro_f1,0.6731025782780014,0.6669813754005415,0.7059686496708458,0.6745583013411162,0.6650385577782836,0.7044332575239617,0.001455723063114811,-0.0019428176222578974,-0.0015353921468841403,-0.0006741622353424089,0.0018557498263327506,-0.0015353921468841403,-0.0019428176222578974,0.001455723063114811,0.02096686007526524,0.020555132742947537,1,2,0,False
|
||||
xiiotid,0.25,0.0,weighted_f1,0.9054655196913645,0.9020747305215218,0.9154163421325315,0.9061557688476052,0.899863925003673,0.9146272396181951,0.0006902491562407187,-0.0022108055178488017,-0.0007891025143363928,-0.0007698862919814919,0.0014506227982888447,-0.0007891025143363928,-0.0022108055178488017,0.0006902491562407187,0.0069343939753111275,0.007408425095578143,1,2,0,False
|
||||
xiiotid,0.25,0.0,accuracy,0.8737218342944525,0.8673409854388906,0.8852306993140127,0.8745509147521049,0.8634903673133494,0.8840024319693425,0.000829080457652398,-0.0038506181255412164,-0.0012282673446701864,-0.0014166016708530016,0.0023455270348474206,-0.0012282673446701864,-0.0038506181255412164,0.000829080457652398,0.009066523301774463,0.010266545051804106,1,2,0,False
|
||||
xiiotid,0.5,0.75,rare_class_macro_f1,0.6096198579015474,0.655665402810942,0.662274505162012,0.6361887569975536,0.6719373483509694,0.6883814197743454,0.026568899096006238,0.01627194554002742,0.026106914612333387,0.022982586416122348,0.0058161742702334575,0.026106914612333387,0.01627194554002742,0.026568899096006238,0.028683282930463034,0.026684712591240056,3,0,0,True
|
||||
xiiotid,0.5,0.75,rare_class_recall,0.8714872155034664,0.8893650106408222,0.8700179539413673,0.8575434903523123,0.8518834947182458,0.878321192150252,-0.013943725151154163,-0.037481515922576425,0.008303238208884678,-0.014374000954948637,0.022895409598216065,-0.013943725151154163,-0.037481515922576425,0.008303238208884678,0.010770970951613777,0.013920609948849614,1,2,0,False
|
||||
xiiotid,0.5,0.75,macro_f1,0.7077011542142579,0.7091760460445067,0.7100168446133323,0.7262984173916737,0.7290145924987934,0.7381943369820925,0.018597263177415724,0.019838546454286732,0.028177492368760215,0.022204434000154222,0.005209919936951091,0.019838546454286732,0.018597263177415724,0.028177492368760215,0.0011722251041387443,0.006233755140268,3,0,0,True
|
||||
xiiotid,0.5,0.75,weighted_f1,0.9175853183547894,0.929132450103975,0.9274806689914715,0.9222775427473946,0.9447335103607959,0.9338838836432976,0.004692224392605193,0.015601060256820842,0.006403214651826028,0.008898833100417355,0.005867005927967981,0.006403214651826028,0.004692224392605193,0.015601060256820842,0.006244765764216191,0.011230108565482134,3,0,0,True
|
||||
xiiotid,0.5,0.75,accuracy,0.8904324115186911,0.9008849666218349,0.9070447273553561,0.8886698478790893,0.9254933028723031,0.9050119448999269,-0.0017625636396018107,0.02460833625046821,-0.0020327824554292606,0.006937663385145713,0.015303848018947405,-0.0017625636396018107,-0.0020327824554292606,0.02460833625046821,0.008398091032446977,0.018450460685614026,1,2,0,False
|
||||
xiiotid,0.75,0.5,rare_class_macro_f1,0.5215401253860467,0.6140607772418322,0.6760680764285393,0.5118888965076885,0.6241440493793726,0.6797040888050662,-0.00965122887835812,0.010083272137540455,0.003636012376526887,0.0013560185452364075,0.010062872924253166,0.003636012376526887,-0.00965122887835812,0.010083272137540455,0.07776445613264914,0.08548886486265879,2,1,0,False
|
||||
xiiotid,0.75,0.5,rare_class_recall,0.8164397897152784,0.8832376049795277,0.8698349315628103,0.8038871133887995,0.8767262170299326,0.8656806653880937,-0.01255267632647894,-0.00651138794959516,-0.004154266174716592,-0.007739443483596897,0.004331791034347159,-0.00651138794959516,-0.01255267632647894,-0.004154266174716592,0.03533794219558259,0.039255535276369505,0,3,0,True
|
||||
xiiotid,0.75,0.5,macro_f1,0.7424713724903063,0.7473954752926397,0.7656800124975965,0.741976732126316,0.7617524315559205,0.7691786586117774,-0.000494640363990273,0.014356956263280818,0.0034986461141808256,0.00578698733782379,0.007685691885683445,0.0034986461141808256,-0.000494640363990273,0.014356956263280818,0.012228464472937677,0.014060416901925107,2,1,0,False
|
||||
xiiotid,0.75,0.5,weighted_f1,0.9325633790408211,0.9359372345770101,0.9489075724365953,0.9355656459425888,0.9483435473557725,0.9517397334421744,0.0030022669017677472,0.012406312778762474,0.002832161005579148,0.00608024689536979,0.005479193934763414,0.0030022669017677472,0.002832161005579148,0.012406312778762474,0.008628878095570825,0.008528481213975155,3,0,0,True
|
||||
xiiotid,0.75,0.5,accuracy,0.9132536187826642,0.9138984591386161,0.9345763398861396,0.9143652007295908,0.9326111121346672,0.9373522240850943,0.0011115819469266208,0.01871265299605107,0.0027758841989546923,0.0075333730473107945,0.009717237293889463,0.0027758841989546923,0.0011115819469266208,0.01871265299605107,0.012128815595858657,0.012136684529934048,3,0,0,True
|
||||
5gnidd,0.25,0.0,rare_class_macro_f1,0.8649030079014913,0.8148479924290983,0.8470722146704726,0.8598927442995293,0.8138885616043564,0.8499254383428447,-0.005010263601961995,-0.0009594308247419248,0.0028532236723721738,-0.0010388235847772487,0.003932344775851092,-0.0009594308247419248,-0.005010263601961995,0.0028532236723721738,0.02537006851997817,0.024201888695409735,1,2,0,False
|
||||
5gnidd,0.25,0.0,rare_class_recall,0.9854440969271964,0.9723332887819381,0.9238897222901731,0.9852344223763528,0.9726587864975359,0.9257555179886866,-0.00020967455084353048,0.00032549771559775564,0.0018657956985135282,0.0006605396210892511,0.001077536077269848,0.00032549771559775564,-0.00020967455084353048,0.0018657956985135282,0.032423277290702865,0.03134699156678076,2,1,0,False
|
||||
5gnidd,0.25,0.0,macro_f1,0.8350430568226094,0.8083728869633723,0.8148003580492102,0.8296799384783186,0.8029871664481885,0.8172541498613525,-0.005363118344290818,-0.005385720515183823,0.0024537918121422253,-0.0027650156824441385,0.004519633996652314,-0.005363118344290818,-0.005385720515183823,0.0024537918121422253,0.013918675694370832,0.01335696520619808,1,2,0,False
|
||||
5gnidd,0.25,0.0,weighted_f1,0.7180410173151626,0.7163267008884987,0.6819385320735543,0.7174035576457394,0.7123024521167044,0.6836842063399526,-0.0006374596694231904,-0.004024248771794303,0.0017456742663983782,-0.0009720113916063718,0.0028994735052727187,-0.0006374596694231904,-0.004024248771794303,0.0017456742663983782,0.02036694411653514,0.018175165847202895,1,2,0,False
|
||||
5gnidd,0.25,0.0,accuracy,0.745280186859777,0.7412954350123573,0.7063291347454734,0.7448442903728559,0.737779477499928,0.7080521595381142,-0.0004358964869211057,-0.0035159575124292797,0.0017230247926407483,-0.0007429430689032124,0.0026329531140602156,-0.0004358964869211057,-0.0035159575124292797,0.0017230247926407483,0.02143091560670625,0.019524710435296355,1,2,0,False
|
||||
5gnidd,0.5,1.0,rare_class_macro_f1,0.8719637151922778,0.9211362509007983,0.8485250688106042,0.8809620350206805,0.8400749080738759,0.8949144498720359,0.008998319828402712,-0.08106134282692246,0.046389381061431756,-0.008557880645695995,0.06551402053172613,0.008998319828402712,-0.08106134282692246,0.046389381061431756,0.0370578200850738,0.028500884123071776,2,1,0,False
|
||||
5gnidd,0.5,1.0,rare_class_recall,0.9743136055202825,0.9685663953100676,0.9758591365273065,0.9743551890147765,0.9683408980660666,0.9586158518509788,4.1583494494012996e-05,-0.00022549724400100501,-0.01724328467632774,-0.0058090661419449114,0.009903224129001456,-0.00022549724400100501,-0.01724328467632774,4.1583494494012996e-05,0.0038428107281798614,0.0079422389145739,1,2,0,False
|
||||
5gnidd,0.5,1.0,macro_f1,0.8368808854359121,0.8654374620417004,0.8248673149789701,0.8504696755820977,0.8159411532871907,0.8493777051701392,0.013588790146185636,-0.04949630875450961,0.024510390191169074,-0.0037990428057183,0.039949974008521794,0.013588790146185636,-0.04949630875450961,0.024510390191169074,0.02083962881650294,0.019627422368850796,2,1,0,False
|
||||
5gnidd,0.5,1.0,weighted_f1,0.7087651452654392,0.7228023713469742,0.7246221247269671,0.7169500389469496,0.7201060072251736,0.7175295566399255,0.00818489368151043,-0.002696364121800632,-0.0070925680870416175,-0.0005346795091106064,0.007864786697422122,-0.002696364121800632,-0.0070925680870416175,0.00818489368151043,0.00867754781098361,0.0016799838366213199,1,2,0,False
|
||||
5gnidd,0.5,1.0,accuracy,0.7344156725348203,0.7473774246742085,0.7501326194500302,0.7428375216406157,0.7440259564021269,0.7429074295677633,0.008421849105795354,-0.003351468272081659,-0.007225189882266836,-0.0007182696828510471,0.008149095556025167,-0.003351468272081659,-0.007225189882266836,0.008421849105795354,0.008392660852506539,0.0006668791219600918,1,2,0,False
|
||||
5gnidd,0.75,0.5,rare_class_macro_f1,0.9055885946257963,0.8607580175603599,0.8733557623287002,0.9044472051910231,0.9057207431633723,0.8800449253698305,-0.0011413894347732656,0.0449627256030124,0.006689163041130319,0.01683683306978982,0.024670402532432648,0.006689163041130319,-0.0011413894347732656,0.0449627256030124,0.023120839413686988,0.014470318925337908,2,1,0,False
|
||||
5gnidd,0.75,0.5,rare_class_recall,0.9612851237375019,0.9590357221091453,0.9633214115686995,0.9586662658643216,0.9597228673347585,0.960715099133502,-0.0026188578731802803,0.0006871452256131949,-0.0026063124351975997,-0.0015126750275882284,0.0019051105497581826,-0.0026063124351975997,-0.0026188578731802803,0.0006871452256131949,0.002143727670860224,0.001024585149761231,1,2,0,False
|
||||
5gnidd,0.75,0.5,macro_f1,0.8571092007836155,0.8298666656506455,0.8372382392225877,0.8573089581866151,0.8599385529838828,0.842201195406295,0.00019975740299960076,0.03007188733323729,0.0049629561837073455,0.011744866973314746,0.016049354258377363,0.0049629561837073455,0.00019975740299960076,0.03007188733323729,0.014091078328488028,0.009572296127013734,3,0,0,True
|
||||
5gnidd,0.75,0.5,weighted_f1,0.7210591954237406,0.7299504931440421,0.7271326093968319,0.7238852910350473,0.7301094421086752,0.7284075566369677,0.002826095611306645,0.00015894896463308772,0.0012749472401357709,0.001419997272025168,0.0013394765554233625,0.0012749472401357709,0.00015894896463308772,0.002826095611306645,0.004543897028843019,0.0032168140791904,3,0,0,True
|
||||
5gnidd,0.75,0.5,accuracy,0.7470484461935134,0.7540721367563544,0.7517446140054364,0.7495075603367095,0.7543805540820061,0.7529330487669476,0.002459114143196106,0.00030841732565167757,0.0011884347615112034,0.0013186554101196624,0.0010812456997565073,0.0011884347615112034,0.00030841732565167757,0.002459114143196106,0.0035777922288362054,0.0025025090963709863,3,0,0,True
|
||||
|
@@ -0,0 +1,91 @@
|
||||
dataset,alpha,seed,selected_beta,metric,original,proposed,delta,manifest_content_hash,pairing_verified
|
||||
xiiotid,0.25,0,0.0,rare_class_macro_f1,0.6398103625587044,0.6413622282749385,0.0015518657162341132,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True
|
||||
xiiotid,0.25,0,0.0,rare_class_recall,0.8296012943628512,0.830700667573069,0.0010993732102178022,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True
|
||||
xiiotid,0.25,0,0.0,macro_f1,0.6731025782780014,0.6745583013411162,0.001455723063114811,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True
|
||||
xiiotid,0.25,0,0.0,weighted_f1,0.9054655196913645,0.9061557688476052,0.0006902491562407187,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True
|
||||
xiiotid,0.25,0,0.0,accuracy,0.8737218342944525,0.8745509147521049,0.000829080457652398,839ba18e2d87245d198be8f5f89d4a24f13cdb6fc64c004d406c6d4f8246f3da,True
|
||||
xiiotid,0.25,1,0.0,rare_class_macro_f1,0.5825215614975117,0.5805553678969027,-0.001966193600609012,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True
|
||||
xiiotid,0.25,1,0.0,rare_class_recall,0.8266995877156649,0.8256994230374527,-0.0010001646782121787,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True
|
||||
xiiotid,0.25,1,0.0,macro_f1,0.6669813754005415,0.6650385577782836,-0.0019428176222578974,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True
|
||||
xiiotid,0.25,1,0.0,weighted_f1,0.9020747305215218,0.899863925003673,-0.0022108055178488017,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True
|
||||
xiiotid,0.25,1,0.0,accuracy,0.8673409854388906,0.8634903673133494,-0.0038506181255412164,84c20b4bd6cd20010fa35d139a0416cb06e79c82fbd8e1d57441cafdcdfbf15d,True
|
||||
xiiotid,0.25,2,0.0,rare_class_macro_f1,0.6760389917828752,0.6744891962601014,-0.0015497955227737892,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True
|
||||
xiiotid,0.25,2,0.0,rare_class_recall,0.8095946724338723,0.807370314447251,-0.0022243579866212393,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True
|
||||
xiiotid,0.25,2,0.0,macro_f1,0.7059686496708458,0.7044332575239617,-0.0015353921468841403,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True
|
||||
xiiotid,0.25,2,0.0,weighted_f1,0.9154163421325315,0.9146272396181951,-0.0007891025143363928,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True
|
||||
xiiotid,0.25,2,0.0,accuracy,0.8852306993140127,0.8840024319693425,-0.0012282673446701864,d6e68ee13cb11f98cfb4aea668f3e905d3ebc94884632352c0045fc5f86df28c,True
|
||||
xiiotid,0.5,0,0.75,rare_class_macro_f1,0.6096198579015474,0.6361887569975536,0.026568899096006238,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True
|
||||
xiiotid,0.5,0,0.75,rare_class_recall,0.8714872155034664,0.8575434903523123,-0.013943725151154163,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True
|
||||
xiiotid,0.5,0,0.75,macro_f1,0.7077011542142579,0.7262984173916737,0.018597263177415724,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True
|
||||
xiiotid,0.5,0,0.75,weighted_f1,0.9175853183547894,0.9222775427473946,0.004692224392605193,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True
|
||||
xiiotid,0.5,0,0.75,accuracy,0.8904324115186911,0.8886698478790893,-0.0017625636396018107,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True
|
||||
xiiotid,0.5,1,0.75,rare_class_macro_f1,0.655665402810942,0.6719373483509694,0.01627194554002742,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True
|
||||
xiiotid,0.5,1,0.75,rare_class_recall,0.8893650106408222,0.8518834947182458,-0.037481515922576425,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True
|
||||
xiiotid,0.5,1,0.75,macro_f1,0.7091760460445067,0.7290145924987934,0.019838546454286732,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True
|
||||
xiiotid,0.5,1,0.75,weighted_f1,0.929132450103975,0.9447335103607959,0.015601060256820842,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True
|
||||
xiiotid,0.5,1,0.75,accuracy,0.9008849666218349,0.9254933028723031,0.02460833625046821,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True
|
||||
xiiotid,0.5,2,0.75,rare_class_macro_f1,0.662274505162012,0.6883814197743454,0.026106914612333387,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True
|
||||
xiiotid,0.5,2,0.75,rare_class_recall,0.8700179539413673,0.878321192150252,0.008303238208884678,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True
|
||||
xiiotid,0.5,2,0.75,macro_f1,0.7100168446133323,0.7381943369820925,0.028177492368760215,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True
|
||||
xiiotid,0.5,2,0.75,weighted_f1,0.9274806689914715,0.9338838836432976,0.006403214651826028,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True
|
||||
xiiotid,0.5,2,0.75,accuracy,0.9070447273553561,0.9050119448999269,-0.0020327824554292606,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True
|
||||
xiiotid,0.75,0,0.5,rare_class_macro_f1,0.5215401253860467,0.5118888965076885,-0.00965122887835812,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True
|
||||
xiiotid,0.75,0,0.5,rare_class_recall,0.8164397897152784,0.8038871133887995,-0.01255267632647894,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True
|
||||
xiiotid,0.75,0,0.5,macro_f1,0.7424713724903063,0.741976732126316,-0.000494640363990273,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True
|
||||
xiiotid,0.75,0,0.5,weighted_f1,0.9325633790408211,0.9355656459425888,0.0030022669017677472,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True
|
||||
xiiotid,0.75,0,0.5,accuracy,0.9132536187826642,0.9143652007295908,0.0011115819469266208,759f7e504a7d442101238726334d121a746ea4a8ff366428c1c174e47f358723,True
|
||||
xiiotid,0.75,1,0.5,rare_class_macro_f1,0.6140607772418322,0.6241440493793726,0.010083272137540455,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True
|
||||
xiiotid,0.75,1,0.5,rare_class_recall,0.8832376049795277,0.8767262170299326,-0.00651138794959516,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True
|
||||
xiiotid,0.75,1,0.5,macro_f1,0.7473954752926397,0.7617524315559205,0.014356956263280818,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True
|
||||
xiiotid,0.75,1,0.5,weighted_f1,0.9359372345770101,0.9483435473557725,0.012406312778762474,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True
|
||||
xiiotid,0.75,1,0.5,accuracy,0.9138984591386161,0.9326111121346672,0.01871265299605107,4a50ba219de20af14f6ad73976111659fba2d5b25cd5eeb1a2f21b76de0d9303,True
|
||||
xiiotid,0.75,2,0.5,rare_class_macro_f1,0.6760680764285393,0.6797040888050662,0.003636012376526887,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True
|
||||
xiiotid,0.75,2,0.5,rare_class_recall,0.8698349315628103,0.8656806653880937,-0.004154266174716592,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True
|
||||
xiiotid,0.75,2,0.5,macro_f1,0.7656800124975965,0.7691786586117774,0.0034986461141808256,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True
|
||||
xiiotid,0.75,2,0.5,weighted_f1,0.9489075724365953,0.9517397334421744,0.002832161005579148,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True
|
||||
xiiotid,0.75,2,0.5,accuracy,0.9345763398861396,0.9373522240850943,0.0027758841989546923,922b7bb15169302be7dd4d4eb1644d077537872e1e91fe59b7525b7e3343cc1c,True
|
||||
5gnidd,0.25,0,0.0,rare_class_macro_f1,0.8649030079014913,0.8598927442995293,-0.005010263601961995,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True
|
||||
5gnidd,0.25,0,0.0,rare_class_recall,0.9854440969271964,0.9852344223763528,-0.00020967455084353048,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True
|
||||
5gnidd,0.25,0,0.0,macro_f1,0.8350430568226094,0.8296799384783186,-0.005363118344290818,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True
|
||||
5gnidd,0.25,0,0.0,weighted_f1,0.7180410173151626,0.7174035576457394,-0.0006374596694231904,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True
|
||||
5gnidd,0.25,0,0.0,accuracy,0.745280186859777,0.7448442903728559,-0.0004358964869211057,ab6d8a8b63d478c4f6cfd88f96ccc7e16e1902f5e2c0ceac0f38ea26cfcda788,True
|
||||
5gnidd,0.25,1,0.0,rare_class_macro_f1,0.8148479924290983,0.8138885616043564,-0.0009594308247419248,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True
|
||||
5gnidd,0.25,1,0.0,rare_class_recall,0.9723332887819381,0.9726587864975359,0.00032549771559775564,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True
|
||||
5gnidd,0.25,1,0.0,macro_f1,0.8083728869633723,0.8029871664481885,-0.005385720515183823,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True
|
||||
5gnidd,0.25,1,0.0,weighted_f1,0.7163267008884987,0.7123024521167044,-0.004024248771794303,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True
|
||||
5gnidd,0.25,1,0.0,accuracy,0.7412954350123573,0.737779477499928,-0.0035159575124292797,74df102e4edb4509ba32b2625aa6a99f4bbd1dbbac0fc61d3bcccd50f21475ba,True
|
||||
5gnidd,0.25,2,0.0,rare_class_macro_f1,0.8470722146704726,0.8499254383428447,0.0028532236723721738,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True
|
||||
5gnidd,0.25,2,0.0,rare_class_recall,0.9238897222901731,0.9257555179886866,0.0018657956985135282,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True
|
||||
5gnidd,0.25,2,0.0,macro_f1,0.8148003580492102,0.8172541498613525,0.0024537918121422253,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True
|
||||
5gnidd,0.25,2,0.0,weighted_f1,0.6819385320735543,0.6836842063399526,0.0017456742663983782,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True
|
||||
5gnidd,0.25,2,0.0,accuracy,0.7063291347454734,0.7080521595381142,0.0017230247926407483,061c094d1913f942267bdb5654d3c0d8cfef3fa592be6cd5a1845a5eb5cdc8f2,True
|
||||
5gnidd,0.5,0,1.0,rare_class_macro_f1,0.8719637151922778,0.8809620350206805,0.008998319828402712,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True
|
||||
5gnidd,0.5,0,1.0,rare_class_recall,0.9743136055202825,0.9743551890147765,4.1583494494012996e-05,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True
|
||||
5gnidd,0.5,0,1.0,macro_f1,0.8368808854359121,0.8504696755820977,0.013588790146185636,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True
|
||||
5gnidd,0.5,0,1.0,weighted_f1,0.7087651452654392,0.7169500389469496,0.00818489368151043,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True
|
||||
5gnidd,0.5,0,1.0,accuracy,0.7344156725348203,0.7428375216406157,0.008421849105795354,0732582cf4510089212717816d2151f5706c7b361123be32cc353bdf4b7f78e6,True
|
||||
5gnidd,0.5,1,1.0,rare_class_macro_f1,0.9211362509007983,0.8400749080738759,-0.08106134282692246,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True
|
||||
5gnidd,0.5,1,1.0,rare_class_recall,0.9685663953100676,0.9683408980660666,-0.00022549724400100501,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True
|
||||
5gnidd,0.5,1,1.0,macro_f1,0.8654374620417004,0.8159411532871907,-0.04949630875450961,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True
|
||||
5gnidd,0.5,1,1.0,weighted_f1,0.7228023713469742,0.7201060072251736,-0.002696364121800632,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True
|
||||
5gnidd,0.5,1,1.0,accuracy,0.7473774246742085,0.7440259564021269,-0.003351468272081659,f43906e111a8342c102237fe21090b64cadcc6e94c77db0deee1497e87e86801,True
|
||||
5gnidd,0.5,2,1.0,rare_class_macro_f1,0.8485250688106042,0.8949144498720359,0.046389381061431756,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True
|
||||
5gnidd,0.5,2,1.0,rare_class_recall,0.9758591365273065,0.9586158518509788,-0.01724328467632774,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True
|
||||
5gnidd,0.5,2,1.0,macro_f1,0.8248673149789701,0.8493777051701392,0.024510390191169074,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True
|
||||
5gnidd,0.5,2,1.0,weighted_f1,0.7246221247269671,0.7175295566399255,-0.0070925680870416175,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True
|
||||
5gnidd,0.5,2,1.0,accuracy,0.7501326194500302,0.7429074295677633,-0.007225189882266836,fe8b4fc6f0486cf4fadf97ea0e096e8c7439fadb1f4e747336313b3bfd52d023,True
|
||||
5gnidd,0.75,0,0.5,rare_class_macro_f1,0.9055885946257963,0.9044472051910231,-0.0011413894347732656,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True
|
||||
5gnidd,0.75,0,0.5,rare_class_recall,0.9612851237375019,0.9586662658643216,-0.0026188578731802803,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True
|
||||
5gnidd,0.75,0,0.5,macro_f1,0.8571092007836155,0.8573089581866151,0.00019975740299960076,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True
|
||||
5gnidd,0.75,0,0.5,weighted_f1,0.7210591954237406,0.7238852910350473,0.002826095611306645,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True
|
||||
5gnidd,0.75,0,0.5,accuracy,0.7470484461935134,0.7495075603367095,0.002459114143196106,96cd9b90b71895da4fcfb8c575c76d3a07ca14e824d8190ca6299eedeba6d6a5,True
|
||||
5gnidd,0.75,1,0.5,rare_class_macro_f1,0.8607580175603599,0.9057207431633723,0.0449627256030124,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True
|
||||
5gnidd,0.75,1,0.5,rare_class_recall,0.9590357221091453,0.9597228673347585,0.0006871452256131949,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True
|
||||
5gnidd,0.75,1,0.5,macro_f1,0.8298666656506455,0.8599385529838828,0.03007188733323729,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True
|
||||
5gnidd,0.75,1,0.5,weighted_f1,0.7299504931440421,0.7301094421086752,0.00015894896463308772,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True
|
||||
5gnidd,0.75,1,0.5,accuracy,0.7540721367563544,0.7543805540820061,0.00030841732565167757,007c23564a44919bba2520d09645e6a99c2a60dd347e8ff8d69a59aaa8ca6db3,True
|
||||
5gnidd,0.75,2,0.5,rare_class_macro_f1,0.8733557623287002,0.8800449253698305,0.006689163041130319,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True
|
||||
5gnidd,0.75,2,0.5,rare_class_recall,0.9633214115686995,0.960715099133502,-0.0026063124351975997,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True
|
||||
5gnidd,0.75,2,0.5,macro_f1,0.8372382392225877,0.842201195406295,0.0049629561837073455,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True
|
||||
5gnidd,0.75,2,0.5,weighted_f1,0.7271326093968319,0.7284075566369677,0.0012749472401357709,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True
|
||||
5gnidd,0.75,2,0.5,accuracy,0.7517446140054364,0.7529330487669476,0.0011884347615112034,3134bea2d47a7336c725e8d1a904b1e15c115336c0f018d685e1780cd952590d,True
|
||||
|
@@ -0,0 +1,31 @@
|
||||
dataset,alpha,selected_beta,metric,n_pairs,deltas,signflip_exact_p_two_sided,wilcoxon_exact_p_two_sided,paired_t_ci95_low,paired_t_ci95_high,min_attainable_signflip_p,note
|
||||
xiiotid,0.25,0.0,rare_class_macro_f1,3,0.001552|-0.001966|-0.001550,0.75,1.0,-0.005429858832977498,0.004120443228211705,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.25,0.0,rare_class_recall,3,0.001099|-0.001000|-0.002224,0.75,1.0,-0.004884136340091787,0.00346737003701471,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.25,0.0,macro_f1,3,0.001456|-0.001943|-0.001535,0.5,1.0,-0.005284100361706733,0.003935775891021916,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.25,0.0,weighted_f1,3,0.000690|-0.002211|-0.000789,0.5,1.0,-0.004373433090107915,0.002833660506144932,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.25,0.0,accuracy,3,0.000829|-0.003851|-0.001228,0.5,1.0,-0.0072432138310324955,0.004410010489326492,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.5,0.75,rare_class_macro_f1,3,0.026569|0.016272|0.026107,0.25,0.5,0.008534408575840131,0.03743076425640456,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.5,0.75,rare_class_recall,3,-0.013944|-0.037482|0.008303,0.5,1.0,-0.07124935135394075,0.042501349444043476,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.5,0.75,macro_f1,3,0.018597|0.019839|0.028177,0.25,0.5,0.009262275411828459,0.035146592588479984,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.5,0.75,weighted_f1,3,0.004692|0.015601|0.006403,0.25,0.5,-0.00567561757777198,0.02347328377860669,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.5,0.75,accuracy,3,-0.001763|0.024608|-0.002033,1.0,1.0,-0.031079202606995104,0.04495452937728653,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.75,0.5,rare_class_macro_f1,3,-0.009651|0.010083|0.003636,0.75,1.0,-0.02364154357009798,0.026353580660570797,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.75,0.5,rare_class_recall,3,-0.012553|-0.006511|-0.004154,0.25,0.5,-0.01850020894956627,0.003021321982372477,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.75,0.5,macro_f1,3,-0.000495|0.014357|0.003499,0.5,1.0,-0.013305329712970347,0.024879304388617927,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.75,0.5,weighted_f1,3,0.003002|0.012406|0.002832,0.25,0.5,-0.007530825385598827,0.019691319176338408,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
xiiotid,0.75,0.5,accuracy,3,0.001112|0.018713|0.002776,0.25,0.5,-0.01660558256426208,0.03167232865888367,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.25,0.0,rare_class_macro_f1,3,-0.005010|-0.000959|0.002853,0.75,1.0,-0.010807309536371811,0.008729662366817314,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.25,0.0,rare_class_recall,3,-0.000210|0.000325|0.001866,0.5,1.0,-0.002016208383762044,0.0033372876259405463,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.25,0.0,macro_f1,3,-0.005363|-0.005386|0.002454,0.5,1.0,-0.013992408934881363,0.008462377569993086,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.25,0.0,weighted_f1,3,-0.000637|-0.004024|0.001746,0.75,1.0,-0.008174702869020644,0.0062306800858079004,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.25,0.0,accuracy,3,-0.000436|-0.003516|0.001723,0.75,1.0,-0.007283561191671378,0.005797675053864953,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.5,1.0,rare_class_macro_f1,3,0.008998|-0.081061|0.046389,1.0,1.0,-0.17130372966859422,0.15418796837720225,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.5,1.0,rare_class_recall,3,0.000042|-0.000225|-0.017243,0.5,1.0,-0.0304100386644281,0.018791906380538275,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.5,1.0,macro_f1,3,0.013589|-0.049496|0.024510,1.0,1.0,-0.10304027980644635,0.09544219419500975,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.5,1.0,weighted_f1,3,0.008185|-0.002696|-0.007093,1.0,1.0,-0.020071892735646125,0.01900253371742491,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.5,1.0,accuracy,3,0.008422|-0.003351|-0.007225,1.0,1.0,-0.02096174526670406,0.019525205901001964,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.75,0.5,rare_class_macro_f1,3,-0.001141|0.044963|0.006689,0.5,1.0,-0.04444784421441203,0.07812151035399167,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.75,0.5,rare_class_recall,3,-0.002619|0.000687|-0.002606,0.5,1.0,-0.006245231988471588,0.0032198819332951312,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.75,0.5,macro_f1,3,0.000200|0.030072|0.004963,0.25,0.5,-0.028123939182216617,0.05161367312884611,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.75,0.5,weighted_f1,3,0.002826|0.000159|0.001275,0.25,0.5,-0.0019074469527278436,0.004747441496778179,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
5gnidd,0.75,0.5,accuracy,3,0.002459|0.000308|0.001188,0.25,0.5,-0.0013673078078454953,0.00400461862808482,0.25,"exploratory; 3 seeds; no inferential power; beta=0 conditions are numerical, not a treatment contrast"
|
||||
|
@@ -0,0 +1,19 @@
|
||||
dataset,class_name,n_obs,mean_delta_f1,median_delta_f1,min_delta_f1,max_delta_f1,n_positive,n_negative
|
||||
xiiotid,Tampering,9,0.05199619640670891,0.004580967510456113,-0.005430499483001561,0.214955968364532,6,3
|
||||
xiiotid,Reconnaissance,9,0.02952063666312104,0.008242346598957062,-0.0003622961256807278,0.10238198044760449,8,1
|
||||
5gnidd,ICMPFlood,9,0.026821108337293526,0.010408977770148542,-0.46986271362984083,0.3772516575720237,5,4
|
||||
5gnidd,TCPConnectScan,9,0.014211458390848499,0.000706524723936286,-0.019433003409767724,0.05772198907204773,6,3
|
||||
xiiotid,Lateral_movement,9,0.010881583240329038,0.0017846016589943003,-0.014168138453098611,0.07607557487760874,6,3
|
||||
xiiotid,Weaponization,9,0.009113660779203083,0.007233534576793765,-0.004643526037137802,0.03336802894144031,8,1
|
||||
xiiotid,Exploitation,9,0.006734945561392644,6.591764883166762e-05,-0.008223665637908967,0.036914135633034896,6,3
|
||||
5gnidd,Benign,9,0.0009395838996870687,0.00028927780247300916,-0.0012807529813573248,0.004784809629670872,5,4
|
||||
5gnidd,SYNScan,9,0.0007028996039083962,0.00012358980722271262,-0.0027240713350633605,0.0033494638084979877,6,2
|
||||
xiiotid,RDOS,9,0.00011273464532779442,0.0,-0.00020998786240000733,0.0008246202062063279,4,4
|
||||
5gnidd,UDPFlood,9,-5.0275925956781244e-05,-5.101815123376152e-05,-0.0004926348932754454,0.0004015231846284939,3,6
|
||||
5gnidd,UDPScan,9,-0.00029387524907019554,-0.0009226463398589457,-0.0038642666069870213,0.004913668121620529,3,6
|
||||
5gnidd,HTTPFlood,9,-0.0008211332125179031,0.0007952922021617903,-0.0344690095854302,0.03529511546161013,6,3
|
||||
xiiotid,Normal,9,-0.002240793040637389,-0.00027963176745748886,-0.0315329514150966,0.012717262317857747,4,5
|
||||
xiiotid,Exfiltration,9,-0.006129465975398516,0.0,-0.03964725823094428,0.004487347741842007,3,3
|
||||
5gnidd,SlowrateDoS,9,-0.006597630438252566,0.0017380510468695265,-0.047165133257141556,0.028229267253973345,5,4
|
||||
xiiotid,C&C,9,-0.018037720972140073,-0.005217928136442729,-0.16416853359506542,0.0321417804096471,4,5
|
||||
5gnidd,SYNFlood,9,-0.019369709950482903,-0.006397152956185925,-0.06835732738470357,0.00727579075336604,2,7
|
||||
|
@@ -0,0 +1,7 @@
|
||||
dataset,alpha,selected_beta,n_client_seed_obs,n_improved_gt_0.02,n_degraded_lt_-0.02,n_unchanged,mean_delta,median_delta,min_delta,max_delta,std_sample_within_condition
|
||||
5gnidd,0.25,0.0,30,1,0,29,0.002111576617816057,0.0,-0.019933554817275767,0.07142857142857151,0.01470740176243374
|
||||
5gnidd,0.5,1.0,30,15,6,9,0.03903400685752252,0.02024848480689545,-0.17567567567567566,0.4040650406504065,0.11851570462408138
|
||||
5gnidd,0.75,0.5,30,8,1,21,0.02071412212202052,0.004477101314292853,-0.03633656881678371,0.16839803756061722,0.043102741088679355
|
||||
xiiotid,0.25,0.0,30,1,1,28,-0.0015178066042212891,0.0,-0.026277262664796908,0.02494573827906854,0.00807984351701968
|
||||
xiiotid,0.5,0.75,30,12,1,17,0.03024920134564162,0.012286627106224551,-0.2565947651121153,0.3235930735930736,0.08670494898387493
|
||||
xiiotid,0.75,0.5,30,8,2,20,0.01167502563325023,0.010269879504053326,-0.0697526762643042,0.06759906759906764,0.026808060695173577
|
||||
|
@@ -0,0 +1,7 @@
|
||||
dataset,alpha,pooled_selected_beta,per_seed_beta_s0,per_seed_beta_s1,per_seed_beta_s2,n_seeds_agree_pooled,pooled_score_selected,second_best_beta,second_best_score,margin_best_minus_second,candidate_score_range,test_used_for_selection
|
||||
xiiotid,0.25,0.0,0.75,0.5,0.0,1,0.5917038146634539,0.25,0.5886906256417319,0.0030131890217219848,0.059605114021112016,False
|
||||
xiiotid,0.5,0.75,0.5,1.0,0.75,1,0.5793667895728949,1.0,0.5734790834215032,0.0058877061513916695,0.03500610197920917,False
|
||||
xiiotid,0.75,0.5,0.5,0.5,0.0,2,0.47674948263663786,0.25,0.47436809609801456,0.002381386538623298,0.008144253663347922,False
|
||||
5gnidd,0.25,0.0,0.25,0.0,0.0,2,0.7614534210908004,0.25,0.7492858468825279,0.012167574208272525,0.039218114350402566,False
|
||||
5gnidd,0.5,1.0,1.0,0.0,1.0,2,0.6923695590578484,0.75,0.6883680316141424,0.004001527443705966,0.04729182197574977,False
|
||||
5gnidd,0.75,0.5,0.5,1.0,1.0,1,0.8090814343293602,1.0,0.8043064619946296,0.004774972334730565,0.03098780223355102,False
|
||||
|
+46
@@ -0,0 +1,46 @@
|
||||
class_id,class_name,beta,n_contributors,support_min,support_max,support_mean,prototype_norm,reference_norm_beta0,displacement_from_beta0,relative_displacement
|
||||
0,C&C,0.0,8,2,687,250.5,4.995807261779628,4.995807261779628,0.0,0.0
|
||||
0,C&C,0.25,8,2,687,250.5,4.857134771566648,4.995807261779628,0.7971996511065712,0.1595737404053873
|
||||
0,C&C,0.5,8,2,687,250.5,4.906583708196013,4.995807261779628,1.2665743125302988,0.2535274573581356
|
||||
0,C&C,0.75,8,2,687,250.5,5.010855437469484,4.995807261779628,1.56068016232047,0.3123979930651924
|
||||
0,C&C,1.0,8,2,687,250.5,5.121849269773383,4.995807261779628,1.7701252868773527,0.35432217339921773
|
||||
1,Exfiltration,0.0,10,8,6273,1549.4,7.157109420154553,7.157109420154553,0.0,0.0
|
||||
1,Exfiltration,0.25,10,8,6273,1549.4,7.739024271072487,7.157109420154553,1.112938631376158,0.1555011340531
|
||||
1,Exfiltration,0.5,10,8,6273,1549.4,8.136996752575591,7.157109420154553,1.792423075103891,0.2504395238189868
|
||||
1,Exfiltration,0.75,10,8,6273,1549.4,8.4002894954015,7.157109420154553,2.2364541806817226,0.312480087894678
|
||||
1,Exfiltration,1.0,10,8,6273,1549.4,8.580132867813264,7.157109420154553,2.5469976586314798,0.35586959889967407
|
||||
2,Exploitation,0.0,9,1,491,88.11111111111111,5.992134154687249,5.992134154687249,0.0,0.0
|
||||
2,Exploitation,0.25,9,1,491,88.11111111111111,5.852057025355164,5.992134154687249,0.6403508338020284,0.10686523653698982
|
||||
2,Exploitation,0.5,9,1,491,88.11111111111111,5.649833686058244,5.992134154687249,1.3058952653326283,0.2179349179475752
|
||||
2,Exploitation,0.75,9,1,491,88.11111111111111,5.462810593200537,5.992134154687249,2.0362822833565866,0.3398258835316192
|
||||
2,Exploitation,1.0,9,1,491,88.11111111111111,5.361136450101408,5.992134154687249,2.7586985908860333,0.46038665351443886
|
||||
3,Lateral_movement,0.0,10,49,8361,2211.7,5.836739012783497,5.836739012783497,0.0,0.0
|
||||
3,Lateral_movement,0.25,10,49,8361,2211.7,6.244285935865453,5.836739012783497,0.7731353833893619,0.13246015997906674
|
||||
3,Lateral_movement,0.5,10,49,8361,2211.7,6.616077783217527,5.836739012783497,1.3883187562912933,0.2378586318919911
|
||||
3,Lateral_movement,0.75,10,49,8361,2211.7,6.909367469065839,5.836739012783497,1.8469497276568052,0.3164352087032942
|
||||
3,Lateral_movement,1.0,10,49,8361,2211.7,7.129359871034012,5.836739012783497,2.18587302323185,0.37450244364951035
|
||||
4,Normal,0.0,10,260,89253,29277.7,5.744022110217372,5.744022110217372,0.0,0.0
|
||||
4,Normal,0.25,10,260,89253,29277.7,6.310406949046281,5.744022110217372,0.7219321390806914,0.1256840808109931
|
||||
4,Normal,0.5,10,260,89253,29277.7,6.65485903802621,5.744022110217372,1.1590902280700377,0.20179069749892972
|
||||
4,Normal,0.75,10,260,89253,29277.7,6.856645071712252,5.744022110217372,1.4245372318857714,0.24800343810512637
|
||||
4,Normal,1.0,10,260,89253,29277.7,6.981584206588257,5.744022110217372,1.5984450234076382,0.2782797476639845
|
||||
5,RDOS,0.0,10,6,47276,9888.3,7.606714687573022,7.606714687573022,0.0,0.0
|
||||
5,RDOS,0.25,10,6,47276,9888.3,8.606801712435749,7.606714687573022,1.6366469943162223,0.21515819398221792
|
||||
5,RDOS,0.5,10,6,47276,9888.3,9.183021988100103,7.606714687573022,2.444982027217194,0.3214241795096542
|
||||
5,RDOS,0.75,10,6,47276,9888.3,9.572100709915908,7.606714687573022,2.932638911884029,0.38553291826168246
|
||||
5,RDOS,1.0,10,6,47276,9888.3,9.877736968852913,7.606714687573022,3.2852506457034076,0.43188824356334454
|
||||
6,Reconnaissance,0.0,10,1842,23967,8916.1,6.600587294730691,6.600587294730691,0.0,0.0
|
||||
6,Reconnaissance,0.25,10,1842,23967,8916.1,6.619642701782661,6.600587294730691,0.19750804124277122,0.02992279814259008
|
||||
6,Reconnaissance,0.5,10,1842,23967,8916.1,6.6531594178920175,6.600587294730691,0.37937859322083467,0.057476490542545514
|
||||
6,Reconnaissance,0.75,10,1842,23967,8916.1,6.698692451193534,6.600587294730691,0.5470003591054624,0.08287146804984122
|
||||
6,Reconnaissance,1.0,10,1842,23967,8916.1,6.75347500969919,6.600587294730691,0.7025557661555466,0.10643837203947037
|
||||
7,Tampering,0.0,9,8,1228,398.44444444444446,5.065268302764301,5.065268302764301,0.0,0.0
|
||||
7,Tampering,0.25,9,8,1228,398.44444444444446,5.155842974383021,5.065268302764301,0.5957873490790528,0.11762207122452131
|
||||
7,Tampering,0.5,9,8,1228,398.44444444444446,5.314632895746181,5.065268302764301,1.0369597200237384,0.20471960378837814
|
||||
7,Tampering,0.75,9,8,1228,398.44444444444446,5.475188209465522,5.065268302764301,1.3704230949002763,0.2705529130909781
|
||||
7,Tampering,1.0,9,8,1228,398.44444444444446,5.614041117925036,5.065268302764301,1.6272957425211183,0.32126545826467745
|
||||
8,Weaponization,0.0,10,917,11526,4509.4,6.991110809530167,6.991110809530167,0.0,0.0
|
||||
8,Weaponization,0.25,10,917,11526,4509.4,7.0769252730146075,6.991110809530167,0.36048411330770536,0.05156320978581821
|
||||
8,Weaponization,0.5,10,917,11526,4509.4,7.163738177542325,6.991110809530167,0.695561899244867,0.0994923293586891
|
||||
8,Weaponization,0.75,10,917,11526,4509.4,7.238525120169953,6.991110809530167,0.9758406496021348,0.13958306143165186
|
||||
8,Weaponization,1.0,10,917,11526,4509.4,7.292875462963678,6.991110809530167,1.189062205569606,0.17008201385517965
|
||||
|
+431
@@ -0,0 +1,431 @@
|
||||
class_id,class_name,beta,client_id,N_ij,weight
|
||||
0,C&C,0.0,0,2.0,0.125
|
||||
0,C&C,0.0,1,3.0,0.125
|
||||
0,C&C,0.0,3,687.0,0.125
|
||||
0,C&C,0.0,4,284.0,0.125
|
||||
0,C&C,0.0,5,599.0,0.125
|
||||
0,C&C,0.0,7,82.0,0.125
|
||||
0,C&C,0.0,8,236.0,0.125
|
||||
0,C&C,0.0,9,111.0,0.125
|
||||
0,C&C,0.25,0,2.0,0.044287795667630386
|
||||
0,C&C,0.25,1,3.0,0.049012502728748776
|
||||
0,C&C,0.25,3,687.0,0.19066270493474782
|
||||
0,C&C,0.25,4,284.0,0.15288192794120894
|
||||
0,C&C,0.25,5,599.0,0.1842397227452447
|
||||
0,C&C,0.25,7,82.0,0.11206758905036181
|
||||
0,C&C,0.25,8,236.0,0.145966961117219
|
||||
0,C&C,0.25,9,111.0,0.12088079581483864
|
||||
0,C&C,0.5,0,2.0,0.013387476143622101
|
||||
0,C&C,0.5,1,3.0,0.016396242747778414
|
||||
0,C&C,0.5,3,687.0,0.24812017604357262
|
||||
0,C&C,0.5,4,284.0,0.15953018988404427
|
||||
0,C&C,0.5,5,599.0,0.23168457650449506
|
||||
0,C&C,0.5,7,82.0,0.08572167297328072
|
||||
0,C&C,0.5,8,236.0,0.14542521467934644
|
||||
0,C&C,0.5,9,111.0,0.09973445102386046
|
||||
0,C&C,0.75,0,2.0,0.003736734114884092
|
||||
0,C&C,0.75,1,3.0,0.005064780649749788
|
||||
0,C&C,0.75,3,687.0,0.2981516746539658
|
||||
0,C&C,0.75,4,284.0,0.15371224135216596
|
||||
0,C&C,0.75,5,599.0,0.2690232512634536
|
||||
0,C&C,0.75,7,82.0,0.06054525177359796
|
||||
0,C&C,0.75,8,236.0,0.1337838538937557
|
||||
0,C&C,0.75,9,111.0,0.07598221229842717
|
||||
0,C&C,1.0,0,2.0,0.000998003992015968
|
||||
0,C&C,1.0,1,3.0,0.0014970059880239522
|
||||
0,C&C,1.0,3,687.0,0.34281437125748504
|
||||
0,C&C,1.0,4,284.0,0.14171656686626746
|
||||
0,C&C,1.0,5,599.0,0.29890219560878245
|
||||
0,C&C,1.0,7,82.0,0.04091816367265469
|
||||
0,C&C,1.0,8,236.0,0.11776447105788423
|
||||
0,C&C,1.0,9,111.0,0.05538922155688623
|
||||
1,Exfiltration,0.0,0,627.0,0.1
|
||||
1,Exfiltration,0.0,1,199.0,0.1
|
||||
1,Exfiltration,0.0,2,1006.0,0.1
|
||||
1,Exfiltration,0.0,3,916.0,0.1
|
||||
1,Exfiltration,0.0,4,6273.0,0.1
|
||||
1,Exfiltration,0.0,5,2759.0,0.1
|
||||
1,Exfiltration,0.0,6,2518.0,0.1
|
||||
1,Exfiltration,0.0,7,8.0,0.1
|
||||
1,Exfiltration,0.0,8,1113.0,0.1
|
||||
1,Exfiltration,0.0,9,75.0,0.1
|
||||
1,Exfiltration,0.25,0,627.0,0.0934897597942656
|
||||
1,Exfiltration,0.25,1,199.0,0.07017144629849609
|
||||
1,Exfiltration,0.25,2,1006.0,0.1052196018044885
|
||||
1,Exfiltration,0.25,3,916.0,0.1027829378876304
|
||||
1,Exfiltration,0.25,4,6273.0,0.16627079779754889
|
||||
1,Exfiltration,0.25,5,2759.0,0.13540511920488763
|
||||
1,Exfiltration,0.25,6,2518.0,0.1323460823920462
|
||||
1,Exfiltration,0.25,7,8.0,0.03142097487762249
|
||||
1,Exfiltration,0.25,8,1113.0,0.10791230019790968
|
||||
1,Exfiltration,0.25,9,75.0,0.054980979745104526
|
||||
1,Exfiltration,0.5,0,627.0,0.07636740343228143
|
||||
1,Exfiltration,0.5,1,199.0,0.04302301007227493
|
||||
1,Exfiltration,0.5,2,1006.0,0.09673268538801975
|
||||
1,Exfiltration,0.5,3,916.0,0.09230431226396064
|
||||
1,Exfiltration,0.5,4,6273.0,0.24155270083748484
|
||||
1,Exfiltration,0.5,5,2759.0,0.16019542325406966
|
||||
1,Exfiltration,0.5,6,2518.0,0.15303899917321462
|
||||
1,Exfiltration,0.5,7,8.0,0.008626194525229437
|
||||
1,Exfiltration,0.5,8,1113.0,0.10174705229246225
|
||||
1,Exfiltration,0.5,9,75.0,0.02641221876100239
|
||||
1,Exfiltration,0.75,0,627.0,0.05728357105623868
|
||||
1,Exfiltration,0.75,1,199.0,0.0242225151235567
|
||||
1,Exfiltration,0.75,2,1006.0,0.0816634858292164
|
||||
1,Exfiltration,0.75,3,916.0,0.07612039507683922
|
||||
1,Exfiltration,0.75,4,6273.0,0.32224478228192666
|
||||
1,Exfiltration,0.75,5,2759.0,0.1740376473264397
|
||||
1,Exfiltration,0.75,6,2518.0,0.1625066818573433
|
||||
1,Exfiltration,0.75,7,8.0,0.002174687802925644
|
||||
1,Exfiltration,0.75,8,1113.0,0.088094907051665
|
||||
1,Exfiltration,0.75,9,75.0,0.01165132659384861
|
||||
1,Exfiltration,1.0,0,627.0,0.04046727765586679
|
||||
1,Exfiltration,1.0,1,199.0,0.012843681425067768
|
||||
1,Exfiltration,1.0,2,1006.0,0.06492835936491545
|
||||
1,Exfiltration,1.0,3,916.0,0.059119659222925
|
||||
1,Exfiltration,1.0,4,6273.0,0.40486639989673423
|
||||
1,Exfiltration,1.0,5,2759.0,0.17806892990835163
|
||||
1,Exfiltration,1.0,6,2518.0,0.16251452175035497
|
||||
1,Exfiltration,1.0,7,8.0,0.000516328901510262
|
||||
1,Exfiltration,1.0,8,1113.0,0.0718342584226152
|
||||
1,Exfiltration,1.0,9,75.0,0.004840583451658707
|
||||
2,Exploitation,0.0,0,56.0,0.1111111111111111
|
||||
2,Exploitation,0.0,2,50.0,0.1111111111111111
|
||||
2,Exploitation,0.0,3,86.0,0.1111111111111111
|
||||
2,Exploitation,0.0,4,46.0,0.1111111111111111
|
||||
2,Exploitation,0.0,5,8.0,0.1111111111111111
|
||||
2,Exploitation,0.0,6,9.0,0.1111111111111111
|
||||
2,Exploitation,0.0,7,46.0,0.1111111111111111
|
||||
2,Exploitation,0.0,8,1.0,0.1111111111111111
|
||||
2,Exploitation,0.0,9,491.0,0.1111111111111111
|
||||
2,Exploitation,0.25,0,56.0,0.1201406619147194
|
||||
2,Exploitation,0.25,2,50.0,0.11678458309217514
|
||||
2,Exploitation,0.25,3,86.0,0.13374194241224655
|
||||
2,Exploitation,0.25,4,46.0,0.1143753593414062
|
||||
2,Exploitation,0.25,5,8.0,0.07386105563289265
|
||||
2,Exploitation,0.25,6,9.0,0.07606828780346402
|
||||
2,Exploitation,0.25,7,46.0,0.1143753593414062
|
||||
2,Exploitation,0.25,8,1.0,0.04391804644012388
|
||||
2,Exploitation,0.25,9,491.0,0.2067347040215659
|
||||
2,Exploitation,0.5,0,56.0,0.11273514564925723
|
||||
2,Exploitation,0.5,2,50.0,0.10652469978734996
|
||||
2,Exploitation,0.5,3,86.0,0.13970583403505477
|
||||
2,Exploitation,0.5,4,46.0,0.10217490265031934
|
||||
2,Exploitation,0.5,5,8.0,0.04260987991493999
|
||||
2,Exploitation,0.5,6,9.0,0.0451946025500978
|
||||
2,Exploitation,0.5,7,46.0,0.10217490265031934
|
||||
2,Exploitation,0.5,8,1.0,0.015064867516699268
|
||||
2,Exploitation,0.5,9,491.0,0.3338151652459622
|
||||
2,Exploitation,0.75,0,56.0,0.09386084897081189
|
||||
2,Exploitation,0.75,2,50.0,0.08621264334249394
|
||||
2,Exploitation,0.75,3,86.0,0.1294843545480138
|
||||
2,Exploitation,0.75,4,46.0,0.0809863510336706
|
||||
2,Exploitation,0.75,5,8.0,0.02181026528528262
|
||||
2,Exploitation,0.75,6,9.0,0.02382458475320683
|
||||
2,Exploitation,0.75,7,46.0,0.0809863510336706
|
||||
2,Exploitation,0.75,8,1.0,0.004585043473531672
|
||||
2,Exploitation,0.75,9,491.0,0.4782495575593181
|
||||
2,Exploitation,1.0,0,56.0,0.07061790668348046
|
||||
2,Exploitation,1.0,2,50.0,0.06305170239596469
|
||||
2,Exploitation,1.0,3,86.0,0.10844892812105927
|
||||
2,Exploitation,1.0,4,46.0,0.058007566204287514
|
||||
2,Exploitation,1.0,5,8.0,0.01008827238335435
|
||||
2,Exploitation,1.0,6,9.0,0.011349306431273645
|
||||
2,Exploitation,1.0,7,46.0,0.058007566204287514
|
||||
2,Exploitation,1.0,8,1.0,0.0012610340479192938
|
||||
2,Exploitation,1.0,9,491.0,0.6191677175283733
|
||||
3,Lateral_movement,0.0,0,2550.0,0.1
|
||||
3,Lateral_movement,0.0,1,3695.0,0.1
|
||||
3,Lateral_movement,0.0,2,818.0,0.1
|
||||
3,Lateral_movement,0.0,3,1239.0,0.1
|
||||
3,Lateral_movement,0.0,4,242.0,0.1
|
||||
3,Lateral_movement,0.0,5,49.0,0.1
|
||||
3,Lateral_movement,0.0,6,8361.0,0.1
|
||||
3,Lateral_movement,0.0,7,4589.0,0.1
|
||||
3,Lateral_movement,0.0,8,396.0,0.1
|
||||
3,Lateral_movement,0.0,9,178.0,0.1
|
||||
3,Lateral_movement,0.25,0,2550.0,0.1211003085961615
|
||||
3,Lateral_movement,0.25,1,3695.0,0.13286598619551734
|
||||
3,Lateral_movement,0.25,2,818.0,0.0911377978662085
|
||||
3,Lateral_movement,0.25,3,1239.0,0.10110625663303405
|
||||
3,Lateral_movement,0.25,4,242.0,0.06721466620617328
|
||||
3,Lateral_movement,0.25,5,49.0,0.04508781866114288
|
||||
3,Lateral_movement,0.25,6,8361.0,0.16295765280822522
|
||||
3,Lateral_movement,0.25,7,4589.0,0.14026190050545512
|
||||
3,Lateral_movement,0.25,8,396.0,0.07602107405963052
|
||||
3,Lateral_movement,0.25,9,178.0,0.06224653846845163
|
||||
3,Lateral_movement,0.5,0,2550.0,0.12946005533478547
|
||||
3,Lateral_movement,0.5,1,3695.0,0.15583783980228746
|
||||
3,Lateral_movement,0.5,2,818.0,0.07332335863139469
|
||||
3,Lateral_movement,0.5,3,1239.0,0.09024047056073208
|
||||
3,Lateral_movement,0.5,4,242.0,0.03988167417574212
|
||||
3,Lateral_movement,0.5,5,49.0,0.017945837798470695
|
||||
3,Lateral_movement,0.5,6,8361.0,0.234420079995533
|
||||
3,Lateral_movement,0.5,7,4589.0,0.173669969070608
|
||||
3,Lateral_movement,0.5,8,396.0,0.05101680902237561
|
||||
3,Lateral_movement,0.5,9,178.0,0.034203905608070934
|
||||
3,Lateral_movement,0.75,0,2550.0,0.12618171141712917
|
||||
3,Lateral_movement,0.75,1,3695.0,0.16664876909744766
|
||||
3,Lateral_movement,0.75,2,818.0,0.05378439105526583
|
||||
3,Lateral_movement,0.75,3,1239.0,0.07343358958802436
|
||||
3,Lateral_movement,0.75,4,242.0,0.02157509927076969
|
||||
3,Lateral_movement,0.75,5,49.0,0.006512359071280791
|
||||
3,Lateral_movement,0.75,6,8361.0,0.3074573967975947
|
||||
3,Lateral_movement,0.75,7,4589.0,0.19605585719761837
|
||||
3,Lateral_movement,0.75,8,396.0,0.031214951724883873
|
||||
3,Lateral_movement,0.75,9,178.0,0.017135874779985513
|
||||
3,Lateral_movement,1.0,0,2550.0,0.11529592621060722
|
||||
3,Lateral_movement,1.0,1,3695.0,0.16706605778360537
|
||||
3,Lateral_movement,1.0,2,818.0,0.0369851245648144
|
||||
3,Lateral_movement,1.0,3,1239.0,0.056020255911742096
|
||||
3,Lateral_movement,1.0,4,242.0,0.010941809467830176
|
||||
3,Lateral_movement,1.0,5,49.0,0.0022154903467920604
|
||||
3,Lateral_movement,1.0,6,8361.0,0.3780349957046616
|
||||
3,Lateral_movement,1.0,7,4589.0,0.20748745309038297
|
||||
3,Lateral_movement,1.0,8,396.0,0.01790477912917665
|
||||
3,Lateral_movement,1.0,9,178.0,0.008048107790387485
|
||||
4,Normal,0.0,0,89253.0,0.1
|
||||
4,Normal,0.0,1,23474.0,0.1
|
||||
4,Normal,0.0,2,60061.0,0.1
|
||||
4,Normal,0.0,3,69735.0,0.1
|
||||
4,Normal,0.0,4,18842.0,0.1
|
||||
4,Normal,0.0,5,1776.0,0.1
|
||||
4,Normal,0.0,6,9668.0,0.1
|
||||
4,Normal,0.0,7,17477.0,0.1
|
||||
4,Normal,0.0,8,260.0,0.1
|
||||
4,Normal,0.0,9,2231.0,0.1
|
||||
4,Normal,0.25,0,89253.0,0.15421903925662245
|
||||
4,Normal,0.25,1,23474.0,0.11044067762482088
|
||||
4,Normal,0.25,2,60061.0,0.1396787781299011
|
||||
4,Normal,0.25,3,69735.0,0.14499231663017753
|
||||
4,Normal,0.25,4,18842.0,0.10453555695398971
|
||||
4,Normal,0.25,5,1776.0,0.05792193046937319
|
||||
4,Normal,0.25,6,9668.0,0.0884741600033457
|
||||
4,Normal,0.25,7,17477.0,0.10258857779473632
|
||||
4,Normal,0.25,8,260.0,0.03582825236264257
|
||||
4,Normal,0.25,9,2231.0,0.06132071077439059
|
||||
4,Normal,0.5,0,89253.0,0.2082761388854636
|
||||
4,Normal,0.5,1,23474.0,0.10681239588140377
|
||||
4,Normal,0.5,2,60061.0,0.17085369909845277
|
||||
4,Normal,0.5,3,69735.0,0.1840998816096602
|
||||
4,Normal,0.5,4,18842.0,0.09569551992004474
|
||||
4,Normal,0.5,5,1776.0,0.029379850890899332
|
||||
4,Normal,0.5,6,9668.0,0.06854825876091125
|
||||
4,Normal,0.5,7,17477.0,0.09216404975621068
|
||||
4,Normal,0.5,8,260.0,0.011241254508049511
|
||||
4,Normal,0.5,9,2231.0,0.03292895068890413
|
||||
4,Normal,0.75,0,89253.0,0.2587802581458785
|
||||
4,Normal,0.75,1,23474.0,0.09503954532267954
|
||||
4,Normal,0.75,2,60061.0,0.19226863535359762
|
||||
4,Normal,0.75,3,69735.0,0.2150562797473768
|
||||
4,Normal,0.75,4,18842.0,0.08059521800192708
|
||||
4,Normal,0.75,5,1776.0,0.01371027752247949
|
||||
4,Normal,0.75,6,9668.0,0.04886146746290477
|
||||
4,Normal,0.75,7,17477.0,0.0761753028341714
|
||||
4,Normal,0.75,8,260.0,0.003244845594082718
|
||||
4,Normal,0.75,9,2231.0,0.016268170014902095
|
||||
4,Normal,1.0,0,89253.0,0.3048497662043125
|
||||
4,Normal,1.0,1,23474.0,0.08017706308897216
|
||||
4,Normal,1.0,2,60061.0,0.20514248045440728
|
||||
4,Normal,1.0,3,69735.0,0.23818469346977392
|
||||
4,Normal,1.0,4,18842.0,0.06435614819470109
|
||||
4,Normal,1.0,5,1776.0,0.0060660502703422745
|
||||
4,Normal,1.0,6,9668.0,0.033021719602291166
|
||||
4,Normal,1.0,7,17477.0,0.059693896720029234
|
||||
4,Normal,1.0,8,260.0,0.0008880478999374951
|
||||
4,Normal,1.0,9,2231.0,0.007620134095232891
|
||||
5,RDOS,0.0,0,4878.0,0.1
|
||||
5,RDOS,0.0,1,95.0,0.1
|
||||
5,RDOS,0.0,2,2816.0,0.1
|
||||
5,RDOS,0.0,3,2508.0,0.1
|
||||
5,RDOS,0.0,4,341.0,0.1
|
||||
5,RDOS,0.0,5,6.0,0.1
|
||||
5,RDOS,0.0,6,47276.0,0.1
|
||||
5,RDOS,0.0,7,20791.0,0.1
|
||||
5,RDOS,0.0,8,13932.0,0.1
|
||||
5,RDOS,0.0,9,6240.0,0.1
|
||||
5,RDOS,0.25,0,4878.0,0.10685785998076136
|
||||
5,RDOS,0.25,1,95.0,0.03991874919463753
|
||||
5,RDOS,0.25,2,2816.0,0.09314385020941883
|
||||
5,RDOS,0.25,3,2508.0,0.09048527413183693
|
||||
5,RDOS,0.25,4,341.0,0.05494581482251136
|
||||
5,RDOS,0.25,5,6.0,0.02001169319705181
|
||||
5,RDOS,0.25,6,47276.0,0.18854117287660088
|
||||
5,RDOS,0.25,7,20791.0,0.15353764253249236
|
||||
5,RDOS,0.25,8,13932.0,0.13891507888064297
|
||||
5,RDOS,0.25,9,6240.0,0.11364286417404601
|
||||
5,RDOS,0.5,0,4878.0,0.09162099667837655
|
||||
5,RDOS,0.5,1,95.0,0.012786035811850421
|
||||
5,RDOS,0.5,2,2816.0,0.06961301423833205
|
||||
5,RDOS,0.5,3,2508.0,0.06569584150494404
|
||||
5,RDOS,0.5,4,341.0,0.024224303741585254
|
||||
5,RDOS,0.5,5,6.0,0.0032132886428106787
|
||||
5,RDOS,0.5,6,47276.0,0.28522952313055794
|
||||
5,RDOS,0.5,7,20791.0,0.18915237691114564
|
||||
5,RDOS,0.5,8,13932.0,0.15483917430831698
|
||||
5,RDOS,0.5,9,6240.0,0.1036254450320805
|
||||
5,RDOS,0.75,0,4878.0,0.06981690248408114
|
||||
5,RDOS,0.75,1,95.0,0.0036397531776229367
|
||||
5,RDOS,0.75,2,2816.0,0.046238497959526094
|
||||
5,RDOS,0.75,3,2508.0,0.04239111882479618
|
||||
5,RDOS,0.75,4,341.0,0.00949172209437759
|
||||
5,RDOS,0.75,5,6.0,0.00045855630336426534
|
||||
5,RDOS,0.75,6,47276.0,0.3834950590544899
|
||||
5,RDOS,0.75,7,20791.0,0.2071027013699195
|
||||
5,RDOS,0.75,8,13932.0,0.1533872740014106
|
||||
5,RDOS,0.75,9,6240.0,0.08397841473041179
|
||||
5,RDOS,1.0,0,4878.0,0.049331027578046784
|
||||
5,RDOS,1.0,1,95.0,0.0009607313693961551
|
||||
5,RDOS,1.0,2,2816.0,0.02847810038125866
|
||||
5,RDOS,1.0,3,2508.0,0.025363308152058492
|
||||
5,RDOS,1.0,4,341.0,0.003448519968043041
|
||||
5,RDOS,1.0,5,6.0,6.067777069870453e-05
|
||||
5,RDOS,1.0,6,47276.0,0.4781003812586592
|
||||
5,RDOS,1.0,7,20791.0,0.21025858843279432
|
||||
5,RDOS,1.0,8,13932.0,0.1408937835623919
|
||||
5,RDOS,1.0,9,6240.0,0.06310488152665271
|
||||
6,Reconnaissance,0.0,0,12996.0,0.1
|
||||
6,Reconnaissance,0.0,1,5840.0,0.1
|
||||
6,Reconnaissance,0.0,2,1842.0,0.1
|
||||
6,Reconnaissance,0.0,3,14852.0,0.1
|
||||
6,Reconnaissance,0.0,4,23967.0,0.1
|
||||
6,Reconnaissance,0.0,5,9632.0,0.1
|
||||
6,Reconnaissance,0.0,6,4475.0,0.1
|
||||
6,Reconnaissance,0.0,7,2025.0,0.1
|
||||
6,Reconnaissance,0.0,8,6536.0,0.1
|
||||
6,Reconnaissance,0.0,9,6996.0,0.1
|
||||
6,Reconnaissance,0.25,0,12996.0,0.11557426335872803
|
||||
6,Reconnaissance,0.25,1,5840.0,0.09462631332339358
|
||||
6,Reconnaissance,0.25,2,1842.0,0.07091385823558452
|
||||
6,Reconnaissance,0.25,3,14852.0,0.11949643350422161
|
||||
6,Reconnaissance,0.25,4,23967.0,0.13468279712303718
|
||||
6,Reconnaissance,0.25,5,9632.0,0.10723531376462318
|
||||
6,Reconnaissance,0.25,6,4475.0,0.08853336438127579
|
||||
6,Reconnaissance,0.25,7,2025.0,0.0726130978586054
|
||||
6,Reconnaissance,0.25,8,6536.0,0.09732775799477392
|
||||
6,Reconnaissance,0.25,9,6996.0,0.09899680045575684
|
||||
6,Reconnaissance,0.5,0,12996.0,0.12887970117234407
|
||||
6,Reconnaissance,0.5,1,5840.0,0.08639449942599037
|
||||
6,Reconnaissance,0.5,2,1842.0,0.04852041223509337
|
||||
6,Reconnaissance,0.5,3,14852.0,0.13777554554880006
|
||||
6,Reconnaissance,0.5,4,23967.0,0.175019527497146
|
||||
6,Reconnaissance,0.5,5,9632.0,0.11095270790280895
|
||||
6,Reconnaissance,0.5,6,4475.0,0.07562688087950485
|
||||
6,Reconnaissance,0.5,7,2025.0,0.050873566252241084
|
||||
6,Reconnaissance,0.5,8,6536.0,0.09139778920981209
|
||||
6,Reconnaissance,0.5,9,6996.0,0.09455936987625915
|
||||
6,Reconnaissance,0.75,0,12996.0,0.13908870708132934
|
||||
6,Reconnaissance,0.75,1,5840.0,0.07633860657711282
|
||||
6,Reconnaissance,0.75,2,1842.0,0.03212934059999074
|
||||
6,Reconnaissance,0.75,3,14852.0,0.15373519339117522
|
||||
6,Reconnaissance,0.75,4,23967.0,0.22011257274836935
|
||||
6,Reconnaissance,0.75,5,9632.0,0.111102018310427
|
||||
6,Reconnaissance,0.75,6,4475.0,0.06252149767879063
|
||||
6,Reconnaissance,0.75,7,2025.0,0.03449477875278175
|
||||
6,Reconnaissance,0.75,8,6536.0,0.0830651064060061
|
||||
6,Reconnaissance,0.75,9,6996.0,0.08741217845401698
|
||||
6,Reconnaissance,1.0,0,12996.0,0.1457587958860937
|
||||
6,Reconnaissance,1.0,1,5840.0,0.06549948968719507
|
||||
6,Reconnaissance,1.0,2,1842.0,0.020659256849968034
|
||||
6,Reconnaissance,1.0,3,14852.0,0.16657507206065433
|
||||
6,Reconnaissance,1.0,4,23967.0,0.2688058680364733
|
||||
6,Reconnaissance,1.0,5,9632.0,0.10802929531970257
|
||||
6,Reconnaissance,1.0,6,4475.0,0.05019010553941746
|
||||
6,Reconnaissance,1.0,7,2025.0,0.022711723735714045
|
||||
6,Reconnaissance,1.0,8,6536.0,0.07330559325265532
|
||||
6,Reconnaissance,1.0,9,6996.0,0.07846479963212616
|
||||
7,Tampering,0.0,0,8.0,0.1111111111111111
|
||||
7,Tampering,0.0,1,1228.0,0.1111111111111111
|
||||
7,Tampering,0.0,2,260.0,0.1111111111111111
|
||||
7,Tampering,0.0,3,91.0,0.1111111111111111
|
||||
7,Tampering,0.0,4,483.0,0.1111111111111111
|
||||
7,Tampering,0.0,5,680.0,0.1111111111111111
|
||||
7,Tampering,0.0,7,618.0,0.1111111111111111
|
||||
7,Tampering,0.0,8,119.0,0.1111111111111111
|
||||
7,Tampering,0.0,9,99.0,0.1111111111111111
|
||||
7,Tampering,0.25,0,8.0,0.046790200953588545
|
||||
7,Tampering,0.25,1,1228.0,0.16469560326700045
|
||||
7,Tampering,0.25,2,260.0,0.11171866840872458
|
||||
7,Tampering,0.25,3,91.0,0.08592959437275875
|
||||
7,Tampering,0.25,4,483.0,0.13042750793401597
|
||||
7,Tampering,0.25,5,680.0,0.14207237541680212
|
||||
7,Tampering,0.25,7,618.0,0.13871695035463072
|
||||
7,Tampering,0.25,8,119.0,0.09189019094870464
|
||||
7,Tampering,0.25,9,99.0,0.0877589083437741
|
||||
7,Tampering,0.5,0,8.0,0.017980251490604452
|
||||
7,Tampering,0.5,1,1228.0,0.22276653609246694
|
||||
7,Tampering,0.5,2,260.0,0.1025032044334467
|
||||
7,Tampering,0.5,3,91.0,0.060641713545164094
|
||||
7,Tampering,0.5,4,483.0,0.1397089838245828
|
||||
7,Tampering,0.5,5,680.0,0.16576972797093445
|
||||
7,Tampering,0.5,7,618.0,0.15803198892360648
|
||||
7,Tampering,0.5,8,119.0,0.06934645250135424
|
||||
7,Tampering,0.5,9,99.0,0.06325114121783987
|
||||
7,Tampering,0.75,0,8.0,0.006480986155468525
|
||||
7,Tampering,0.75,1,1228.0,0.2826326812706583
|
||||
7,Tampering,0.75,2,260.0,0.08821727012693042
|
||||
7,Tampering,0.75,3,91.0,0.040142521414312994
|
||||
7,Tampering,0.75,4,483.0,0.1403731195760649
|
||||
7,Tampering,0.75,5,680.0,0.18142840727962103
|
||||
7,Tampering,0.75,7,618.0,0.16887484359213606
|
||||
7,Tampering,0.75,8,119.0,0.049088959242897044
|
||||
7,Tampering,0.75,9,99.0,0.04276121134191071
|
||||
7,Tampering,1.0,0,8.0,0.002230897936419409
|
||||
7,Tampering,1.0,1,1228.0,0.3424428332403793
|
||||
7,Tampering,1.0,2,260.0,0.07250418293363078
|
||||
7,Tampering,1.0,3,91.0,0.025376464026770774
|
||||
7,Tampering,1.0,4,483.0,0.1346904629113218
|
||||
7,Tampering,1.0,5,680.0,0.18962632459564974
|
||||
7,Tampering,1.0,7,618.0,0.17233686558839934
|
||||
7,Tampering,1.0,8,119.0,0.03318460680423871
|
||||
7,Tampering,1.0,9,99.0,0.027607361963190184
|
||||
8,Weaponization,0.0,0,11526.0,0.1
|
||||
8,Weaponization,0.0,1,2611.0,0.1
|
||||
8,Weaponization,0.0,2,1209.0,0.1
|
||||
8,Weaponization,0.0,3,1647.0,0.1
|
||||
8,Weaponization,0.0,4,917.0,0.1
|
||||
8,Weaponization,0.0,5,1375.0,0.1
|
||||
8,Weaponization,0.0,6,951.0,0.1
|
||||
8,Weaponization,0.0,7,6976.0,0.1
|
||||
8,Weaponization,0.0,8,6913.0,0.1
|
||||
8,Weaponization,0.0,9,10969.0,0.1
|
||||
8,Weaponization,0.25,0,11526.0,0.13739441060301907
|
||||
8,Weaponization,0.25,1,2611.0,0.09478751320998981
|
||||
8,Weaponization,0.25,2,1209.0,0.07819085944838905
|
||||
8,Weaponization,0.25,3,1647.0,0.08447395176979355
|
||||
8,Weaponization,0.25,4,917.0,0.07296956302069034
|
||||
8,Weaponization,0.25,5,1375.0,0.0807467564699872
|
||||
8,Weaponization,0.25,6,951.0,0.07363673785619201
|
||||
8,Weaponization,0.25,7,6976.0,0.12118560338330699
|
||||
8,Weaponization,0.25,8,6913.0,0.12091106619109798
|
||||
8,Weaponization,0.25,9,10969.0,0.13570353804753404
|
||||
8,Weaponization,0.5,0,11526.0,0.1779201650136799
|
||||
8,Weaponization,0.5,1,2611.0,0.08468164794018174
|
||||
8,Weaponization,0.5,2,1209.0,0.057623418014587556
|
||||
8,Weaponization,0.5,3,1647.0,0.06725625246465362
|
||||
8,Weaponization,0.5,4,917.0,0.05018460762438691
|
||||
8,Weaponization,0.5,5,1375.0,0.061452168704738123
|
||||
8,Weaponization,0.5,6,951.0,0.051106498114700416
|
||||
8,Weaponization,0.5,7,6976.0,0.13841689443045152
|
||||
8,Weaponization,0.5,8,6913.0,0.13779045794448722
|
||||
8,Weaponization,0.5,9,10969.0,0.17356788974813298
|
||||
8,Weaponization,0.75,0,11526.0,0.2181935226168662
|
||||
8,Weaponization,0.75,1,2611.0,0.07164535620438116
|
||||
8,Weaponization,0.75,2,1209.0,0.04021634686447464
|
||||
8,Weaponization,0.75,3,1647.0,0.050711107075023125
|
||||
8,Weaponization,0.75,4,917.0,0.03268585833755308
|
||||
8,Weaponization,0.75,5,1375.0,0.04429043196553467
|
||||
8,Weaponization,0.75,6,951.0,0.033590640121105526
|
||||
8,Weaponization,0.75,7,6976.0,0.14972273310126752
|
||||
8,Weaponization,0.75,8,6913.0,0.1487074785967807
|
||||
8,Weaponization,0.75,9,10969.0,0.21023652511701332
|
||||
8,Weaponization,1.0,0,11526.0,0.25559941455626023
|
||||
8,Weaponization,1.0,1,2611.0,0.05790127289661596
|
||||
8,Weaponization,1.0,2,1209.0,0.026810662172351087
|
||||
8,Weaponization,1.0,3,1647.0,0.036523706036279774
|
||||
8,Weaponization,1.0,4,917.0,0.020335299596398633
|
||||
8,Weaponization,1.0,5,1375.0,0.030491861444981595
|
||||
8,Weaponization,1.0,6,951.0,0.021089280170310906
|
||||
8,Weaponization,1.0,7,6976.0,0.15469907304741207
|
||||
8,Weaponization,1.0,8,6913.0,0.1533019913957511
|
||||
8,Weaponization,1.0,9,10969.0,0.24324743868363863
|
||||
|
+46
@@ -0,0 +1,46 @@
|
||||
class_id,class_name,beta,n_contributors,support_min,support_max,support_mean,prototype_norm,reference_norm_beta0,displacement_from_beta0,relative_displacement
|
||||
0,C&C,0.0,9,13,576,222.66666666666666,4.992104138405685,4.992104138405685,0.0,0.0
|
||||
0,C&C,0.25,9,13,576,222.66666666666666,4.852037998238657,4.992104138405685,0.36797724022308037,0.07371185176048838
|
||||
0,C&C,0.5,9,13,576,222.66666666666666,4.742376194761032,4.992104138405685,0.6695589690144707,0.13412359807628252
|
||||
0,C&C,0.75,9,13,576,222.66666666666666,4.65280704661058,4.992104138405685,0.9254131398589358,0.1853753676209332
|
||||
0,C&C,1.0,9,13,576,222.66666666666666,4.577031141286735,4.992104138405685,1.1548168541805874,0.23132867868205137
|
||||
1,Exfiltration,0.0,10,20,5300,1549.4,6.100872138731717,6.100872138731717,0.0,0.0
|
||||
1,Exfiltration,0.25,10,20,5300,1549.4,6.373166160422756,6.100872138731717,0.8237491288550555,0.1350215362858434
|
||||
1,Exfiltration,0.5,10,20,5300,1549.4,6.59429581012542,6.100872138731717,1.3263165170775955,0.21739785507999804
|
||||
1,Exfiltration,0.75,10,20,5300,1549.4,6.750434428594119,6.100872138731717,1.668226550089498,0.27344066752336466
|
||||
1,Exfiltration,1.0,10,20,5300,1549.4,6.8750663990264895,6.100872138731717,1.967235340132892,0.3224514947041411
|
||||
2,Exploitation,0.0,10,1,298,79.3,5.008097566090378,5.008097566090378,0.0,0.0
|
||||
2,Exploitation,0.25,10,1,298,79.3,5.047843612402878,5.008097566090378,0.8603955105014754,0.17180086832316885
|
||||
2,Exploitation,0.5,10,1,298,79.3,5.080358485205799,5.008097566090378,1.5532104750000217,0.3101398194629325
|
||||
2,Exploitation,0.75,10,1,298,79.3,5.1087633771611625,5.008097566090378,2.03138992764355,0.40562107683324844
|
||||
2,Exploitation,1.0,10,1,298,79.3,5.136319248087567,5.008097566090378,2.3385740832421744,0.4669585710702928
|
||||
3,Lateral_movement,0.0,10,11,6341,2211.7,8.481321033671277,8.481321033671277,0.0,0.0
|
||||
3,Lateral_movement,0.25,10,11,6341,2211.7,6.381682682418986,8.481321033671277,2.6907008570910045,0.31725020741565907
|
||||
3,Lateral_movement,0.5,10,11,6341,2211.7,5.888946922739037,8.481321033671277,3.5825998636295417,0.42241059493048755
|
||||
3,Lateral_movement,0.75,10,11,6341,2211.7,5.835315568632712,8.481321033671277,3.841073254381612,0.452886200054491
|
||||
3,Lateral_movement,1.0,10,11,6341,2211.7,5.867438144584891,8.481321033671277,3.9173091599439407,0.46187488298014234
|
||||
4,Normal,0.0,10,79,92031,29277.7,4.799298249108693,4.799298249108693,0.0,0.0
|
||||
4,Normal,0.25,10,79,92031,29277.7,5.712379958571236,4.799298249108693,1.1032890094562697,0.22988548579183807
|
||||
4,Normal,0.5,10,79,92031,29277.7,6.190715393209056,4.799298249108693,1.664815841256164,0.3468873478670235
|
||||
4,Normal,0.75,10,79,92031,29277.7,6.413993120649692,4.799298249108693,1.934162646704862,0.40300947061668174
|
||||
4,Normal,1.0,10,79,92031,29277.7,6.530433735916667,4.799298249108693,2.083642028161429,0.4341555619195775
|
||||
5,RDOS,0.0,10,4,48860,9888.3,6.959683596787167,6.959683596787167,0.0,0.0
|
||||
5,RDOS,0.25,10,4,48860,9888.3,7.618774665376232,6.959683596787167,1.6164653614457636,0.2322613289765041
|
||||
5,RDOS,0.5,10,4,48860,9888.3,7.820301085426912,6.959683596787167,2.353132945001506,0.3381091844588731
|
||||
5,RDOS,0.75,10,4,48860,9888.3,7.86519605437697,6.959683596787167,2.8666348425171084,0.41189154688590257
|
||||
5,RDOS,1.0,10,4,48860,9888.3,7.8810106988442605,6.959683596787167,3.322434234042245,0.4773829424624988
|
||||
6,Reconnaissance,0.0,10,208,22975,8916.1,5.415862180814956,5.415862180814956,0.0,0.0
|
||||
6,Reconnaissance,0.25,10,208,22975,8916.1,5.648965820089017,5.415862180814956,0.4294267183380856,0.07929055504020734
|
||||
6,Reconnaissance,0.5,10,208,22975,8916.1,5.808825260074657,5.415862180814956,0.7413987406482649,0.13689394521053017
|
||||
6,Reconnaissance,0.75,10,208,22975,8916.1,5.916255824906189,5.415862180814956,0.9617371210189529,0.17757784243952732
|
||||
6,Reconnaissance,1.0,10,208,22975,8916.1,5.991685616665521,5.415862180814956,1.1180725676513317,0.20644405827237813
|
||||
7,Tampering,0.0,10,26,2437,358.6,4.705216201871325,4.705216201871325,0.0,0.0
|
||||
7,Tampering,0.25,10,26,2437,358.6,4.639701461060896,4.705216201871325,0.7387359771740888,0.15700362012701646
|
||||
7,Tampering,0.5,10,26,2437,358.6,4.771752454614714,4.705216201871325,1.706873516878616,0.3627619738705674
|
||||
7,Tampering,0.75,10,26,2437,358.6,5.157225192090308,4.705216201871325,2.768796503000988,0.5884525565264784
|
||||
7,Tampering,1.0,10,26,2437,358.6,5.656404611496285,4.705216201871325,3.6982366587327884,0.7859865519594939
|
||||
8,Weaponization,0.0,10,17,23228,4509.4,5.306307987150024,5.306307987150024,0.0,0.0
|
||||
8,Weaponization,0.25,10,17,23228,4509.4,5.843513874965596,5.306307987150024,1.2689737477647698,0.23914438265509075
|
||||
8,Weaponization,0.5,10,17,23228,4509.4,6.216911060615151,5.306307987150024,2.0356955585328023,0.38363690224211017
|
||||
8,Weaponization,0.75,10,17,23228,4509.4,6.441588231782958,5.306307987150024,2.543364318490572,0.4793095924039254
|
||||
8,Weaponization,1.0,10,17,23228,4509.4,6.5954300391115535,5.306307987150024,2.934502391320979,0.5530214978903019
|
||||
|
+446
@@ -0,0 +1,446 @@
|
||||
class_id,class_name,beta,client_id,N_ij,weight
|
||||
0,C&C,0.0,0,576.0,0.1111111111111111
|
||||
0,C&C,0.0,1,290.0,0.1111111111111111
|
||||
0,C&C,0.0,3,211.0,0.1111111111111111
|
||||
0,C&C,0.0,4,37.0,0.1111111111111111
|
||||
0,C&C,0.0,5,252.0,0.1111111111111111
|
||||
0,C&C,0.0,6,212.0,0.1111111111111111
|
||||
0,C&C,0.0,7,13.0,0.1111111111111111
|
||||
0,C&C,0.0,8,296.0,0.1111111111111111
|
||||
0,C&C,0.0,9,117.0,0.1111111111111111
|
||||
0,C&C,0.25,0,576.0,0.15102191422892977
|
||||
0,C&C,0.25,1,290.0,0.12721368978362058
|
||||
0,C&C,0.25,3,211.0,0.1174910983303728
|
||||
0,C&C,0.25,4,37.0,0.0760299633060768
|
||||
0,C&C,0.25,5,252.0,0.12282435299375524
|
||||
0,C&C,0.25,6,212.0,0.11763005906798477
|
||||
0,C&C,0.25,7,13.0,0.05853561531663628
|
||||
0,C&C,0.25,8,296.0,0.12786664719190285
|
||||
0,C&C,0.25,9,117.0,0.10138665978072102
|
||||
0,C&C,0.5,0,576.0,0.19401845748233754
|
||||
0,C&C,0.5,1,290.0,0.1376673031078246
|
||||
0,C&C,0.5,3,211.0,0.11742837022526757
|
||||
0,C&C,0.5,4,37.0,0.04917367513999254
|
||||
0,C&C,0.5,5,252.0,0.12833114706365603
|
||||
0,C&C,0.5,6,212.0,0.11770630758500977
|
||||
0,C&C,0.5,7,13.0,0.02914764570162491
|
||||
0,C&C,0.5,8,296.0,0.1390841565894123
|
||||
0,C&C,0.5,9,117.0,0.08744293710487475
|
||||
0,C&C,0.75,0,576.0,0.23955666553058955
|
||||
0,C&C,0.75,1,290.0,0.14318248489669622
|
||||
0,C&C,0.75,3,211.0,0.11279847762533143
|
||||
0,C&C,0.75,4,37.0,0.030566287230480278
|
||||
0,C&C,0.75,5,252.0,0.12886702493762558
|
||||
0,C&C,0.75,6,212.0,0.1131991830239939
|
||||
0,C&C,0.75,7,13.0,0.013949187470554809
|
||||
0,C&C,0.75,8,296.0,0.14539858501481584
|
||||
0,C&C,0.75,9,117.0,0.07248210426991236
|
||||
0,C&C,1.0,0,576.0,0.2874251497005988
|
||||
0,C&C,1.0,1,290.0,0.14471057884231536
|
||||
0,C&C,1.0,3,211.0,0.10528942115768464
|
||||
0,C&C,1.0,4,37.0,0.018463073852295408
|
||||
0,C&C,1.0,5,252.0,0.12574850299401197
|
||||
0,C&C,1.0,6,212.0,0.10578842315369262
|
||||
0,C&C,1.0,7,13.0,0.006487025948103792
|
||||
0,C&C,1.0,8,296.0,0.14770459081836326
|
||||
0,C&C,1.0,9,117.0,0.058383233532934134
|
||||
1,Exfiltration,0.0,0,2126.0,0.1
|
||||
1,Exfiltration,0.0,1,20.0,0.1
|
||||
1,Exfiltration,0.0,2,914.0,0.1
|
||||
1,Exfiltration,0.0,3,1557.0,0.1
|
||||
1,Exfiltration,0.0,4,30.0,0.1
|
||||
1,Exfiltration,0.0,5,1430.0,0.1
|
||||
1,Exfiltration,0.0,6,5300.0,0.1
|
||||
1,Exfiltration,0.0,7,2362.0,0.1
|
||||
1,Exfiltration,0.0,8,150.0,0.1
|
||||
1,Exfiltration,0.0,9,1605.0,0.1
|
||||
1,Exfiltration,0.25,0,2126.0,0.1245754812136925
|
||||
1,Exfiltration,0.25,1,20.0,0.03879709938306384
|
||||
1,Exfiltration,0.25,2,914.0,0.10087378280431746
|
||||
1,Exfiltration,0.25,3,1557.0,0.11524282917902283
|
||||
1,Exfiltration,0.25,4,30.0,0.04293604842405325
|
||||
1,Exfiltration,0.25,5,1430.0,0.11281731714741136
|
||||
1,Exfiltration,0.25,6,5300.0,0.15653473624097708
|
||||
1,Exfiltration,0.25,7,2362.0,0.12789740466447497
|
||||
1,Exfiltration,0.25,8,150.0,0.06420436768136334
|
||||
1,Exfiltration,0.25,9,1605.0,0.11612093326162334
|
||||
1,Exfiltration,0.5,0,2126.0,0.1367087312207029
|
||||
1,Exfiltration,0.5,1,20.0,0.013259575496658765
|
||||
1,Exfiltration,0.5,2,914.0,0.08963708409730776
|
||||
1,Exfiltration,0.5,3,1557.0,0.11699274444200186
|
||||
1,Exfiltration,0.5,4,30.0,0.016239597086362405
|
||||
1,Exfiltration,0.5,5,1430.0,0.11211988440120217
|
||||
1,Exfiltration,0.5,6,5300.0,0.2158502506905477
|
||||
1,Exfiltration,0.5,7,2362.0,0.14409687724487022
|
||||
1,Exfiltration,0.5,8,150.0,0.036312843012313856
|
||||
1,Exfiltration,0.5,9,1605.0,0.1187824123080323
|
||||
1,Exfiltration,0.75,0,2126.0,0.1398719763694802
|
||||
1,Exfiltration,0.75,1,20.0,0.004225039397957745
|
||||
1,Exfiltration,0.75,2,914.0,0.07426221746657875
|
||||
1,Exfiltration,0.75,3,1557.0,0.11073240327966676
|
||||
1,Exfiltration,0.75,4,30.0,0.005726631097987726
|
||||
1,Exfiltration,0.75,5,1430.0,0.10388678215251898
|
||||
1,Exfiltration,0.75,6,5300.0,0.27750139361661635
|
||||
1,Exfiltration,0.75,7,2362.0,0.15136246307483137
|
||||
1,Exfiltration,0.75,8,150.0,0.019148145134778874
|
||||
1,Exfiltration,0.75,9,1605.0,0.11328294840958317
|
||||
1,Exfiltration,1.0,0,2126.0,0.13721440557635214
|
||||
1,Exfiltration,1.0,1,20.0,0.001290822253775655
|
||||
1,Exfiltration,1.0,2,914.0,0.058990576997547436
|
||||
1,Exfiltration,1.0,3,1557.0,0.10049051245643476
|
||||
1,Exfiltration,1.0,4,30.0,0.0019362333806634826
|
||||
1,Exfiltration,1.0,5,1430.0,0.09229379114495934
|
||||
1,Exfiltration,1.0,6,5300.0,0.3420678972505486
|
||||
1,Exfiltration,1.0,7,2362.0,0.15244610817090487
|
||||
1,Exfiltration,1.0,8,150.0,0.009681166903317414
|
||||
1,Exfiltration,1.0,9,1605.0,0.10358848586549632
|
||||
2,Exploitation,0.0,0,10.0,0.1
|
||||
2,Exploitation,0.0,1,1.0,0.1
|
||||
2,Exploitation,0.0,2,240.0,0.1
|
||||
2,Exploitation,0.0,3,2.0,0.1
|
||||
2,Exploitation,0.0,4,35.0,0.1
|
||||
2,Exploitation,0.0,5,145.0,0.1
|
||||
2,Exploitation,0.0,6,298.0,0.1
|
||||
2,Exploitation,0.0,7,49.0,0.1
|
||||
2,Exploitation,0.0,8,7.0,0.1
|
||||
2,Exploitation,0.0,9,6.0,0.1
|
||||
2,Exploitation,0.25,0,10.0,0.07472353923163849
|
||||
2,Exploitation,0.25,1,1.0,0.042020134074432634
|
||||
2,Exploitation,0.25,2,240.0,0.16539037968734396
|
||||
2,Exploitation,0.25,3,2.0,0.04997064241468357
|
||||
2,Exploitation,0.25,4,35.0,0.10220554181683487
|
||||
2,Exploitation,0.25,5,145.0,0.1458140706954984
|
||||
2,Exploitation,0.25,6,298.0,0.17458683816365086
|
||||
2,Exploitation,0.25,7,49.0,0.11117482481854002
|
||||
2,Exploitation,0.25,8,7.0,0.06834896520487059
|
||||
2,Exploitation,0.25,9,6.0,0.06576506389250662
|
||||
2,Exploitation,0.5,0,10.0,0.046242935423721356
|
||||
2,Exploitation,0.5,1,1.0,0.014623300163104301
|
||||
2,Exploitation,0.5,2,240.0,0.22654319199318068
|
||||
2,Exploitation,0.5,3,2.0,0.020680469417314797
|
||||
2,Exploitation,0.5,4,35.0,0.08651261045713868
|
||||
2,Exploitation,0.5,5,145.0,0.17608785196808927
|
||||
2,Exploitation,0.5,6,298.0,0.25243730010193305
|
||||
2,Exploitation,0.5,7,49.0,0.10236310114173011
|
||||
2,Exploitation,0.5,8,7.0,0.03868961557862425
|
||||
2,Exploitation,0.5,9,6.0,0.03581962375516356
|
||||
2,Exploitation,0.75,0,10.0,0.02512275929612515
|
||||
2,Exploitation,0.75,1,1.0,0.0044675285579663295
|
||||
2,Exploitation,0.75,2,240.0,0.27241170763423844
|
||||
2,Exploitation,0.75,3,2.0,0.007513457498874965
|
||||
2,Exploitation,0.75,4,35.0,0.06428629111250711
|
||||
2,Exploitation,0.75,5,145.0,0.18667808582049356
|
||||
2,Exploitation,0.75,6,298.0,0.32042731849990974
|
||||
2,Exploitation,0.75,7,49.0,0.08273978677620542
|
||||
2,Exploitation,0.75,8,7.0,0.019226085412864014
|
||||
2,Exploitation,0.75,9,6.0,0.017126979390815276
|
||||
2,Exploitation,1.0,0,10.0,0.012610340479192938
|
||||
2,Exploitation,1.0,1,1.0,0.0012610340479192938
|
||||
2,Exploitation,1.0,2,240.0,0.3026481715006305
|
||||
2,Exploitation,1.0,3,2.0,0.0025220680958385876
|
||||
2,Exploitation,1.0,4,35.0,0.044136191677175286
|
||||
2,Exploitation,1.0,5,145.0,0.1828499369482976
|
||||
2,Exploitation,1.0,6,298.0,0.37578814627994955
|
||||
2,Exploitation,1.0,7,49.0,0.0617906683480454
|
||||
2,Exploitation,1.0,8,7.0,0.008827238335435058
|
||||
2,Exploitation,1.0,9,6.0,0.007566204287515763
|
||||
3,Lateral_movement,0.0,0,3084.0,0.1
|
||||
3,Lateral_movement,0.0,1,11.0,0.1
|
||||
3,Lateral_movement,0.0,2,2152.0,0.1
|
||||
3,Lateral_movement,0.0,3,72.0,0.1
|
||||
3,Lateral_movement,0.0,4,566.0,0.1
|
||||
3,Lateral_movement,0.0,5,6341.0,0.1
|
||||
3,Lateral_movement,0.0,6,3130.0,0.1
|
||||
3,Lateral_movement,0.0,7,4431.0,0.1
|
||||
3,Lateral_movement,0.0,8,131.0,0.1
|
||||
3,Lateral_movement,0.0,9,2199.0,0.1
|
||||
3,Lateral_movement,0.25,0,3084.0,0.12702175800740229
|
||||
3,Lateral_movement,0.25,1,11.0,0.031041857299547067
|
||||
3,Lateral_movement,0.25,2,2152.0,0.11609408516022754
|
||||
3,Lateral_movement,0.25,3,72.0,0.0496515317395058
|
||||
3,Lateral_movement,0.25,4,566.0,0.0831387775793627
|
||||
3,Lateral_movement,0.25,5,6341.0,0.15210341508889633
|
||||
3,Lateral_movement,0.25,6,3130.0,0.12749278590878954
|
||||
3,Lateral_movement,0.25,7,4431.0,0.13906728125201873
|
||||
3,Lateral_movement,0.25,8,131.0,0.057665671116614775
|
||||
3,Lateral_movement,0.25,9,2199.0,0.11672283684763529
|
||||
3,Lateral_movement,0.5,0,3084.0,0.13953291328353135
|
||||
3,Lateral_movement,0.5,1,11.0,0.008333277031910022
|
||||
3,Lateral_movement,0.5,2,2152.0,0.1165576038262483
|
||||
3,Lateral_movement,0.5,3,72.0,0.02131992759351566
|
||||
3,Lateral_movement,0.5,4,566.0,0.0597761159716795
|
||||
3,Lateral_movement,0.5,5,6341.0,0.20007754681173798
|
||||
3,Lateral_movement,0.5,6,3130.0,0.14056967669368875
|
||||
3,Lateral_movement,0.5,7,4431.0,0.16725162738364865
|
||||
3,Lateral_movement,0.5,8,131.0,0.028757764639806005
|
||||
3,Lateral_movement,0.5,9,2199.0,0.11782354676423379
|
||||
3,Lateral_movement,0.75,0,3084.0,0.1422829526966925
|
||||
3,Lateral_movement,0.75,1,11.0,0.0020766418694166703
|
||||
3,Lateral_movement,0.75,2,2152.0,0.10862975309362505
|
||||
3,Lateral_movement,0.75,3,72.0,0.008497994868397177
|
||||
3,Lateral_movement,0.75,4,566.0,0.0398960055447025
|
||||
3,Lateral_movement,0.75,5,6341.0,0.24430671665218012
|
||||
3,Lateral_movement,0.75,6,3130.0,0.14387169007556813
|
||||
3,Lateral_movement,0.75,7,4431.0,0.18672110322562685
|
||||
3,Lateral_movement,0.75,8,131.0,0.013312835445965593
|
||||
3,Lateral_movement,0.75,9,2199.0,0.11040430652782546
|
||||
3,Lateral_movement,1.0,0,3084.0,0.13944024958176968
|
||||
3,Lateral_movement,1.0,1,11.0,0.0004973549758104626
|
||||
3,Lateral_movement,1.0,2,2152.0,0.09730071890401049
|
||||
3,Lateral_movement,1.0,3,72.0,0.0032554143871230275
|
||||
3,Lateral_movement,1.0,4,566.0,0.0255911742098838
|
||||
3,Lateral_movement,1.0,5,6341.0,0.28670253651037664
|
||||
3,Lateral_movement,1.0,6,3130.0,0.14152009766243162
|
||||
3,Lateral_movement,1.0,7,4431.0,0.2003436270741963
|
||||
3,Lateral_movement,1.0,8,131.0,0.005923045621015508
|
||||
3,Lateral_movement,1.0,9,2199.0,0.09942578107338247
|
||||
4,Normal,0.0,0,864.0,0.1
|
||||
4,Normal,0.0,1,418.0,0.1
|
||||
4,Normal,0.0,2,79.0,0.1
|
||||
4,Normal,0.0,3,33716.0,0.1
|
||||
4,Normal,0.0,4,10789.0,0.1
|
||||
4,Normal,0.0,5,1541.0,0.1
|
||||
4,Normal,0.0,6,73646.0,0.1
|
||||
4,Normal,0.0,7,57785.0,0.1
|
||||
4,Normal,0.0,8,92031.0,0.1
|
||||
4,Normal,0.0,9,21908.0,0.1
|
||||
4,Normal,0.25,0,864.0,0.05188460739258251
|
||||
4,Normal,0.25,1,418.0,0.04327172113503886
|
||||
4,Normal,0.25,2,79.0,0.028530992000834074
|
||||
4,Normal,0.25,3,33716.0,0.1296788380467858
|
||||
4,Normal,0.25,4,10789.0,0.09753385669084398
|
||||
4,Normal,0.25,5,1541.0,0.059959880532585996
|
||||
4,Normal,0.25,6,73646.0,0.15765123965898994
|
||||
4,Normal,0.25,7,57785.0,0.14837607503612354
|
||||
4,Normal,0.25,8,92031.0,0.16668389274771683
|
||||
4,Normal,0.25,9,21908.0,0.11642889675849852
|
||||
4,Normal,0.5,0,864.0,0.021795608067872552
|
||||
4,Normal,0.5,1,418.0,0.015160036935382743
|
||||
4,Normal,0.5,2,79.0,0.006590610776578489
|
||||
4,Normal,0.5,3,33716.0,0.13615391733129972
|
||||
4,Normal,0.5,4,10789.0,0.07701985826089296
|
||||
4,Normal,0.5,5,1541.0,0.029108071823932803
|
||||
4,Normal,0.5,6,73646.0,0.2012272074670356
|
||||
4,Normal,0.5,7,57785.0,0.17824595265334742
|
||||
4,Normal,0.5,8,92031.0,0.22494647347085892
|
||||
4,Normal,0.5,9,21908.0,0.10975226321279885
|
||||
4,Normal,0.75,0,864.0,0.008234413752433757
|
||||
4,Normal,0.75,1,418.0,0.0047767178535437
|
||||
4,Normal,0.75,2,79.0,0.0013692025418151716
|
||||
4,Normal,0.75,3,33716.0,0.12856549080612478
|
||||
4,Normal,0.75,4,10789.0,0.05469948948624618
|
||||
4,Normal,0.75,5,1541.0,0.01270864818588827
|
||||
4,Normal,0.75,6,73646.0,0.23099854847865048
|
||||
4,Normal,0.75,7,57785.0,0.19257890653203613
|
||||
4,Normal,0.75,8,92031.0,0.2730222139329396
|
||||
4,Normal,0.75,9,21908.0,0.093046368430322
|
||||
4,Normal,1.0,0,864.0,0.0029510514828692143
|
||||
4,Normal,1.0,1,418.0,0.0014277077775918191
|
||||
4,Normal,1.0,2,79.0,0.000269829938827162
|
||||
4,Normal,1.0,3,33716.0,0.11515931920881763
|
||||
4,Normal,1.0,4,10789.0,0.03685057227856013
|
||||
4,Normal,1.0,5,1541.0,0.005263391591552615
|
||||
4,Normal,1.0,6,73646.0,0.2515429832261414
|
||||
4,Normal,1.0,7,57785.0,0.1973686457611083
|
||||
4,Normal,1.0,8,92031.0,0.31433821645826004
|
||||
4,Normal,1.0,9,21908.0,0.0748282822762717
|
||||
5,RDOS,0.0,0,17951.0,0.1
|
||||
5,RDOS,0.0,1,7758.0,0.1
|
||||
5,RDOS,0.0,2,48860.0,0.1
|
||||
5,RDOS,0.0,3,94.0,0.1
|
||||
5,RDOS,0.0,4,1624.0,0.1
|
||||
5,RDOS,0.0,5,1142.0,0.1
|
||||
5,RDOS,0.0,6,4.0,0.1
|
||||
5,RDOS,0.0,7,10129.0,0.1
|
||||
5,RDOS,0.0,8,4382.0,0.1
|
||||
5,RDOS,0.0,9,6939.0,0.1
|
||||
5,RDOS,0.25,0,17951.0,0.14502857687379875
|
||||
5,RDOS,0.25,1,7758.0,0.117589587658176
|
||||
5,RDOS,0.25,2,48860.0,0.18628152700149078
|
||||
5,RDOS,0.25,3,94.0,0.039013362856557
|
||||
5,RDOS,0.25,4,1624.0,0.07953859769224278
|
||||
5,RDOS,0.25,5,1142.0,0.0728363123438262
|
||||
5,RDOS,0.25,6,4.0,0.017719294303109735
|
||||
5,RDOS,0.25,7,10129.0,0.12569646568603307
|
||||
5,RDOS,0.25,8,4382.0,0.10194114727488954
|
||||
5,RDOS,0.25,9,6939.0,0.11435512830987608
|
||||
5,RDOS,0.5,0,17951.0,0.17198442418661095
|
||||
5,RDOS,0.5,1,7758.0,0.11306278550208099
|
||||
5,RDOS,0.5,2,48860.0,0.2837406273058024
|
||||
5,RDOS,0.5,3,94.0,0.01244539600169817
|
||||
5,RDOS,0.5,4,1624.0,0.05172944342275919
|
||||
5,RDOS,0.5,5,1142.0,0.043378831099658635
|
||||
5,RDOS,0.5,6,4.0,0.002567289170851146
|
||||
5,RDOS,0.5,7,10129.0,0.12918975624376042
|
||||
5,RDOS,0.5,8,4382.0,0.08497300555974267
|
||||
5,RDOS,0.5,9,6939.0,0.10692844150703547
|
||||
5,RDOS,0.75,0,17951.0,0.18336531141459259
|
||||
5,RDOS,0.75,1,7758.0,0.09773790520016168
|
||||
5,RDOS,0.75,2,48860.0,0.38856689071177486
|
||||
5,RDOS,0.75,3,94.0,0.0035694112275838435
|
||||
5,RDOS,0.75,4,1624.0,0.03024755073569198
|
||||
5,RDOS,0.75,5,1142.0,0.02322738257067633
|
||||
5,RDOS,0.75,6,4.0,0.00033442265364325624
|
||||
5,RDOS,0.75,7,10129.0,0.11937836417894782
|
||||
5,RDOS,0.75,8,4382.0,0.06368028158230388
|
||||
5,RDOS,0.75,9,6939.0,0.08989247972462366
|
||||
5,RDOS,1.0,0,17951.0,0.18153777696874082
|
||||
5,RDOS,1.0,1,7758.0,0.07845635751342496
|
||||
5,RDOS,1.0,2,48860.0,0.4941193127231172
|
||||
5,RDOS,1.0,3,94.0,0.0009506184076130376
|
||||
5,RDOS,1.0,4,1624.0,0.016423449935782694
|
||||
5,RDOS,1.0,5,1142.0,0.011549002356320096
|
||||
5,RDOS,1.0,6,4.0,4.0451847132469685e-05
|
||||
5,RDOS,1.0,7,10129.0,0.10243418990119636
|
||||
5,RDOS,1.0,8,4382.0,0.044314998533620545
|
||||
5,RDOS,1.0,9,6939.0,0.0701738418130518
|
||||
6,Reconnaissance,0.0,0,1651.0,0.1
|
||||
6,Reconnaissance,0.0,1,208.0,0.1
|
||||
6,Reconnaissance,0.0,2,12985.0,0.1
|
||||
6,Reconnaissance,0.0,3,17154.0,0.1
|
||||
6,Reconnaissance,0.0,4,22975.0,0.1
|
||||
6,Reconnaissance,0.0,5,16770.0,0.1
|
||||
6,Reconnaissance,0.0,6,5123.0,0.1
|
||||
6,Reconnaissance,0.0,7,8929.0,0.1
|
||||
6,Reconnaissance,0.0,8,2708.0,0.1
|
||||
6,Reconnaissance,0.0,9,658.0,0.1
|
||||
6,Reconnaissance,0.25,0,1651.0,0.07374148107073869
|
||||
6,Reconnaissance,0.25,1,208.0,0.04393300617126853
|
||||
6,Reconnaissance,0.25,2,12985.0,0.1234910764123451
|
||||
6,Reconnaissance,0.25,3,17154.0,0.13239343160661676
|
||||
6,Reconnaissance,0.25,4,22975.0,0.14242589843429998
|
||||
6,Reconnaissance,0.25,5,16770.0,0.1316462078006376
|
||||
6,Reconnaissance,0.25,6,5123.0,0.09787148471969809
|
||||
6,Reconnaissance,0.25,7,8929.0,0.11245422928318183
|
||||
6,Reconnaissance,0.25,8,2708.0,0.08345209671284375
|
||||
6,Reconnaissance,0.25,9,658.0,0.05859108778836971
|
||||
6,Reconnaissance,0.5,0,1651.0,0.04926272195084457
|
||||
6,Reconnaissance,0.5,1,208.0,0.0174854387978552
|
||||
6,Reconnaissance,0.5,2,12985.0,0.13815475751236914
|
||||
6,Reconnaissance,0.5,3,17154.0,0.15879161684940787
|
||||
6,Reconnaissance,0.5,4,22975.0,0.18376915905719687
|
||||
6,Reconnaissance,0.5,5,16770.0,0.15700424660659393
|
||||
6,Reconnaissance,0.5,6,5123.0,0.08677748234243855
|
||||
6,Reconnaissance,0.5,7,8929.0,0.11456350164519018
|
||||
6,Reconnaissance,0.5,8,2708.0,0.06309125958277643
|
||||
6,Reconnaissance,0.5,9,658.0,0.03109981565532717
|
||||
6,Reconnaissance,0.75,0,1651.0,0.030840346291550146
|
||||
6,Reconnaissance,0.75,1,208.0,0.006521634300518785
|
||||
6,Reconnaissance,0.75,2,12985.0,0.14484063041988704
|
||||
6,Reconnaissance,0.75,3,17154.0,0.17847730325539155
|
||||
6,Reconnaissance,0.75,4,22975.0,0.22220333914755816
|
||||
6,Reconnaissance,0.75,5,16770.0,0.17547236839271976
|
||||
6,Reconnaissance,0.75,6,5123.0,0.07210281289252812
|
||||
6,Reconnaissance,0.75,7,8929.0,0.10937323511754056
|
||||
6,Reconnaissance,0.75,8,2708.0,0.044698755937912026
|
||||
6,Reconnaissance,0.75,9,658.0,0.015469574244393894
|
||||
6,Reconnaissance,1.0,0,1651.0,0.0185170646358834
|
||||
6,Reconnaissance,1.0,1,208.0,0.002332858536804208
|
||||
6,Reconnaissance,1.0,2,12985.0,0.14563542355962808
|
||||
6,Reconnaissance,1.0,3,17154.0,0.1923935352900932
|
||||
6,Reconnaissance,1.0,4,22975.0,0.25767992732248407
|
||||
6,Reconnaissance,1.0,5,16770.0,0.18808671952983927
|
||||
6,Reconnaissance,1.0,6,5123.0,0.057457857134845955
|
||||
6,Reconnaissance,1.0,7,8929.0,0.10014468209194603
|
||||
6,Reconnaissance,1.0,8,2708.0,0.030372023642624018
|
||||
6,Reconnaissance,1.0,9,658.0,0.007379908255851774
|
||||
7,Tampering,0.0,0,118.0,0.1
|
||||
7,Tampering,0.0,1,371.0,0.1
|
||||
7,Tampering,0.0,2,135.0,0.1
|
||||
7,Tampering,0.0,3,2437.0,0.1
|
||||
7,Tampering,0.0,4,76.0,0.1
|
||||
7,Tampering,0.0,5,132.0,0.1
|
||||
7,Tampering,0.0,6,146.0,0.1
|
||||
7,Tampering,0.0,7,26.0,0.1
|
||||
7,Tampering,0.0,8,64.0,0.1
|
||||
7,Tampering,0.0,9,81.0,0.1
|
||||
7,Tampering,0.25,0,118.0,0.0914906614903368
|
||||
7,Tampering,0.25,1,371.0,0.12182873736373746
|
||||
7,Tampering,0.25,2,135.0,0.09462147373200593
|
||||
7,Tampering,0.25,3,2437.0,0.19503841012134226
|
||||
7,Tampering,0.25,4,76.0,0.08196145582272057
|
||||
7,Tampering,0.25,5,132.0,0.09409136058601118
|
||||
7,Tampering,0.25,6,146.0,0.09649270466577241
|
||||
7,Tampering,0.25,7,26.0,0.06268297534976354
|
||||
7,Tampering,0.25,8,64.0,0.07851475128408372
|
||||
7,Tampering,0.25,9,81.0,0.08327746958422619
|
||||
7,Tampering,0.5,0,118.0,0.07465928078918507
|
||||
7,Tampering,0.5,1,371.0,0.13238224845001956
|
||||
7,Tampering,0.5,2,135.0,0.07985639165881428
|
||||
7,Tampering,0.5,3,2437.0,0.33928961183869
|
||||
7,Tampering,0.5,4,76.0,0.05991693570925297
|
||||
7,Tampering,0.5,5,132.0,0.07896411346733563
|
||||
7,Tampering,0.5,6,146.0,0.08304609697876314
|
||||
7,Tampering,0.5,7,26.0,0.035045275003324286
|
||||
7,Tampering,0.5,8,64.0,0.05498355110805415
|
||||
7,Tampering,0.5,9,81.0,0.06185649499656092
|
||||
7,Tampering,0.75,0,118.0,0.05307060131423309
|
||||
7,Tampering,0.75,1,371.0,0.12530630827033065
|
||||
7,Tampering,0.75,2,135.0,0.05870739760575514
|
||||
7,Tampering,0.75,3,2437.0,0.514143263422916
|
||||
7,Tampering,0.75,4,76.0,0.038155108874919706
|
||||
7,Tampering,0.75,5,132.0,0.05772619755866028
|
||||
7,Tampering,0.75,6,146.0,0.062259717357723954
|
||||
7,Tampering,0.75,7,26.0,0.017067602272501124
|
||||
7,Tampering,0.75,8,64.0,0.03354111405422001
|
||||
7,Tampering,0.75,9,81.0,0.04002268926874002
|
||||
7,Tampering,1.0,0,118.0,0.03290574456218628
|
||||
7,Tampering,1.0,1,371.0,0.10345789180145008
|
||||
7,Tampering,1.0,2,135.0,0.03764640267707752
|
||||
7,Tampering,1.0,3,2437.0,0.6795872838817624
|
||||
7,Tampering,1.0,4,76.0,0.021193530395984383
|
||||
7,Tampering,1.0,5,132.0,0.03680981595092025
|
||||
7,Tampering,1.0,6,146.0,0.04071388733965421
|
||||
7,Tampering,1.0,7,26.0,0.007250418293363078
|
||||
7,Tampering,1.0,8,64.0,0.01784718349135527
|
||||
7,Tampering,1.0,9,81.0,0.022587841606246516
|
||||
8,Weaponization,0.0,0,895.0,0.1
|
||||
8,Weaponization,0.0,1,1014.0,0.1
|
||||
8,Weaponization,0.0,2,23228.0,0.1
|
||||
8,Weaponization,0.0,3,10869.0,0.1
|
||||
8,Weaponization,0.0,4,218.0,0.1
|
||||
8,Weaponization,0.0,5,1570.0,0.1
|
||||
8,Weaponization,0.0,6,17.0,0.1
|
||||
8,Weaponization,0.0,7,2670.0,0.1
|
||||
8,Weaponization,0.0,8,38.0,0.1
|
||||
8,Weaponization,0.0,9,4575.0,0.1
|
||||
8,Weaponization,0.25,0,895.0,0.08582243760083762
|
||||
8,Weaponization,0.25,1,1014.0,0.08854306990947461
|
||||
8,Weaponization,0.25,2,23228.0,0.19370821585254064
|
||||
8,Weaponization,0.25,3,10869.0,0.16021105634310917
|
||||
8,Weaponization,0.25,4,218.0,0.06029193331621394
|
||||
8,Weaponization,0.25,5,1570.0,0.09876885061052997
|
||||
8,Weaponization,0.25,6,17.0,0.03186084700198994
|
||||
8,Weaponization,0.25,7,2670.0,0.11279059873978263
|
||||
8,Weaponization,0.25,8,38.0,0.03895749161753624
|
||||
8,Weaponization,0.25,9,4575.0,0.1290454990079852
|
||||
8,Weaponization,0.5,0,895.0,0.05954631191374364
|
||||
8,Weaponization,0.5,1,1014.0,0.06338147296578088
|
||||
8,Weaponization,0.5,2,23228.0,0.3033536738955772
|
||||
8,Weaponization,0.5,3,10869.0,0.2075095768145407
|
||||
8,Weaponization,0.5,4,218.0,0.02938810596696327
|
||||
8,Weaponization,0.5,5,1570.0,0.07886661054849604
|
||||
8,Weaponization,0.5,6,17.0,0.00820668588741752
|
||||
8,Weaponization,0.5,7,2670.0,0.10284874130321621
|
||||
8,Weaponization,0.5,8,38.0,0.0122697340296214
|
||||
8,Weaponization,0.5,9,4575.0,0.13462908667464316
|
||||
8,Weaponization,0.75,0,895.0,0.035987691170914056
|
||||
8,Weaponization,0.75,1,1014.0,0.039519839730892736
|
||||
8,Weaponization,0.75,2,23228.0,0.41380484518079425
|
||||
8,Weaponization,0.75,3,10869.0,0.2341148165504639
|
||||
8,Weaponization,0.75,4,218.0,0.012477543098555985
|
||||
8,Weaponization,0.75,5,1570.0,0.054854397303363285
|
||||
8,Weaponization,0.75,6,17.0,0.0018412951086240396
|
||||
8,Weaponization,0.75,7,2670.0,0.08169023670182742
|
||||
8,Weaponization,0.75,8,38.0,0.0033660797865626784
|
||||
8,Weaponization,0.75,9,4575.0,0.12234325536800156
|
||||
8,Weaponization,1.0,0,895.0,0.01984742981327893
|
||||
8,Weaponization,1.0,1,1014.0,0.02248636182197188
|
||||
8,Weaponization,1.0,2,23228.0,0.5151017873774781
|
||||
8,Weaponization,1.0,3,10869.0,0.24102984876036723
|
||||
8,Weaponization,1.0,4,218.0,0.004834346032731627
|
||||
8,Weaponization,1.0,5,1570.0,0.0348161617953608
|
||||
8,Weaponization,1.0,6,17.0,0.00037699028695613605
|
||||
8,Weaponization,1.0,7,2670.0,0.059209650951346074
|
||||
8,Weaponization,1.0,8,38.0,0.0008426841708431277
|
||||
8,Weaponization,1.0,9,4575.0,0.10145473898966603
|
||||
|
@@ -0,0 +1,335 @@
|
||||
# Post-hoc Statistical Audit of the PROTEAN Experiment Grid
|
||||
|
||||
Status: **read-only post-hoc audit of completed experiments.** No training, no partition
|
||||
regeneration, no modification of any existing result, manifest, dataset, configuration, source
|
||||
file, or test. All numbers below are recomputed from the audited artifacts by
|
||||
`scripts/posthoc_statistical_audit.py`, which writes new `posthoc__*.csv` tables under
|
||||
`artifacts/analysis/` and refuses to overwrite existing files.
|
||||
|
||||
Scope: determine what the completed 108-run grid can and cannot support statistically, and
|
||||
provide transparent, reusable wording for the thesis limitations.
|
||||
|
||||
---
|
||||
|
||||
## 1. Executive findings
|
||||
|
||||
1. **The comparison is valid and paired.** All 18 `(dataset, alpha, seed)` cells pair the
|
||||
`original_protean` run with the validation-selected `support_weighted` run on an identical
|
||||
`manifest_content_hash`, split, and seed (`posthoc__paired_global_per_seed.csv`,
|
||||
`pairing_verified = True` for every row).
|
||||
2. **The paired differences are small and mostly seed-inconsistent.** For test rare-class
|
||||
macro-F1, only **X-IIoTID α=0.50** has all three seed differences in the same direction
|
||||
(**+0.0230** mean; paired-difference sample std **0.0058**); the other five conditions have
|
||||
mixed-sign seed differences.
|
||||
3. **No condition can reach statistical significance with three seeds.** The smallest attainable
|
||||
two-sided exact sign-flip p-value with three nonzero differences is **0.25**, and only
|
||||
X-IIoTID α=0.50 attains it. This is a **power/design limit**, not evidence of no effect.
|
||||
4. **The paired-difference variability is not the group variability.** At X-IIoTID α=0.50 the
|
||||
baseline and proposed cross-seed std are **0.0287** and **0.0267**, while the paired-difference
|
||||
std is **0.0058** — shared seed-to-seed variation cancels in the paired difference.
|
||||
5. **Selected beta is weakly separated and unstable across seeds.** The best-vs-second margin is
|
||||
only **0.0024–0.0122**, and the pooled selection agrees with at most **2 of 3** seeds in every
|
||||
condition (`posthoc__selection_stability.csv`).
|
||||
6. **The `S_j` participant rule is materially exercised**: `|S_j| < M` in **53 of 162**
|
||||
class-condition cells (verified). This is an implementation/reproduction assumption, not an
|
||||
ordinary statistical limitation.
|
||||
7. **Publication metrics could not be verified** because the referenced `journal/main.tex`
|
||||
(and the paper's Table IV) is **not present in this repository**. Reproduced baseline
|
||||
accuracies are reported instead; the discrepancy figures remain unverified.
|
||||
|
||||
---
|
||||
|
||||
## 2. Data sources and pairing validation
|
||||
|
||||
Primary sources (all committed, read-only):
|
||||
|
||||
| Purpose | Artifact |
|
||||
|---|---|
|
||||
| Paired per-seed global metrics | `artifacts/analysis/deepcmp__global_per_seed.csv` |
|
||||
| Per-run source of truth | `results/<run>/test_global_metrics.json`, `metadata.json` |
|
||||
| Validation per-seed / pooled | `artifacts/analysis/<ds>__alpha*__multiseed__validation_per_seed.csv`, `..._validation_pooled.csv`, `..._selection_summary.json` |
|
||||
| Selection margins | `artifacts/analysis/final__selection_strength.csv`, `final__pooled_validation_curves.csv` |
|
||||
| Per-class | `results/<run>/test_per_class.csv` → `deepcmp__per_class.csv` |
|
||||
| Per-client | `results/<run>/per_client.csv` → `deepcmp__per_client.csv` |
|
||||
| `S_j` rule | `artifacts/analysis/sj_participant_rule_analysis.csv` |
|
||||
| Reproduced baseline metrics | `docs/baseline/BASELINE_VALIDATION.md` |
|
||||
|
||||
Pairing rule enforced in the script: same `dataset`, `alpha`, `seed`, identical
|
||||
`manifest_content_hash`, and recorded `beta == validation-selected beta`. Result: **18/18 cells
|
||||
verified**, written with `pairing_verified` and the manifest hash into
|
||||
`posthoc__paired_global_per_seed.csv`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Descriptive paired results by condition
|
||||
|
||||
Test split, `delta = proposed_selected_beta − original_protean`, three paired seeds per condition.
|
||||
Source: `posthoc__paired_global_by_condition.csv`. The standard deviations below are three distinct
|
||||
quantities: `base_std` and `prop_std` are the cross-seed spread within each group, `paired_std` is
|
||||
computed directly from the three matched differences.
|
||||
|
||||
### 3.1 Rare-class macro-F1 (primary comparison)
|
||||
|
||||
| Dataset | alpha | beta* | Δ seed 0 | Δ seed 1 | Δ seed 2 | mean Δ | paired std | base std | prop std | +/−/0 | same sign |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 0.00 | +0.0016 | −0.0020 | −0.0016 | −0.0007 | 0.0019 | 0.0472 | 0.0476 | 1/2/0 | no |
|
||||
| X-IIoTID | 0.50 | 0.75 | +0.0266 | +0.0163 | +0.0261 | **+0.0230** | 0.0058 | 0.0287 | 0.0267 | 3/0/0 | **yes** |
|
||||
| X-IIoTID | 0.75 | 0.50 | −0.0097 | +0.0101 | +0.0036 | +0.0014 | 0.0101 | 0.0778 | 0.0855 | 2/1/0 | no |
|
||||
| 5G-NIDD | 0.25 | 0.00 | −0.0050 | −0.0010 | +0.0029 | −0.0010 | 0.0039 | 0.0254 | 0.0242 | 1/2/0 | no |
|
||||
| 5G-NIDD | 0.50 | 1.00 | +0.0090 | −0.0811 | +0.0464 | −0.0086 | 0.0655 | 0.0371 | 0.0285 | 2/1/0 | no |
|
||||
| 5G-NIDD | 0.75 | 0.50 | −0.0011 | +0.0450 | +0.0067 | +0.0168 | 0.0247 | 0.0231 | 0.0145 | 2/1/0 | no |
|
||||
|
||||
**Verification of the reported `+0.0230`.** Source `final__six_condition_master.csv`
|
||||
(row X-IIoTID α=0.50): original rare-F1 mean 0.642520 (std **0.028683**), selected-beta mean
|
||||
0.665503 (std **0.026685**), difference **+0.022983**. The paired-difference std computed directly
|
||||
from the three matched seed differences is **0.0058**. The value is exact.
|
||||
|
||||
**What this does and does not establish.** X-IIoTID α=0.50 is the only condition where the
|
||||
improvement appears at every seed, and only there is the paired difference small relative to its own
|
||||
mean. It does **not** establish statistical significance (see §4), and it does not generalize: the
|
||||
same beta direction fails at 5G-NIDD α=0.50 and is mixed at four other conditions.
|
||||
|
||||
### 3.2 Other metrics
|
||||
|
||||
All five metrics (rare-class macro-F1, rare-class recall, macro-F1, weighted-F1, accuracy) with the
|
||||
same columns, including per-metric positive/negative/zero counts and sign agreement, are in
|
||||
`posthoc__paired_global_by_condition.csv`. Highlights:
|
||||
|
||||
- **Rare-class recall** is negative in **5 of 6** conditions (and in **all four** `beta>0`
|
||||
conditions): the rare-class F1 gains are precision-driven.
|
||||
- **Overall macro-F1** moves in the same direction as rare-class F1 in every condition, but its
|
||||
paired differences are equally seed-inconsistent.
|
||||
- **5G-NIDD α=0.50** has a paired-difference std of **0.0655**, larger than either group std,
|
||||
because the three seed differences have mixed sign (one large negative).
|
||||
|
||||
---
|
||||
|
||||
## 4. Inferential analysis and limitations
|
||||
|
||||
Sources: `posthoc__paired_test_results.csv`. Only three paired seeds exist per condition.
|
||||
|
||||
| Dataset | alpha | beta* | exact sign-flip p (2-sided) | exact Wilcoxon p (2-sided) | paired-t 95% CI | min attainable sign-flip p |
|
||||
|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 0.00 | 0.75 | 1.00 | [−0.0054, +0.0041] | 0.25 |
|
||||
| X-IIoTID | 0.50 | 0.75 | **0.25** | 0.50 | [+0.0085, +0.0374] | 0.25 |
|
||||
| X-IIoTID | 0.75 | 0.50 | 0.75 | 1.00 | [−0.0236, +0.0264] | 0.25 |
|
||||
| 5G-NIDD | 0.25 | 0.00 | 0.75 | 1.00 | [−0.0108, +0.0087] | 0.25 |
|
||||
| 5G-NIDD | 0.50 | 1.00 | 1.00 | 1.00 | [−0.1713, +0.1542] | 0.25 |
|
||||
| 5G-NIDD | 0.75 | 0.50 | 0.50 | 1.00 | [−0.0444, +0.0781] | 0.25 |
|
||||
|
||||
Test definitions and assumptions:
|
||||
|
||||
- **Exact sign-flip permutation test** on the mean of the paired differences. Assumption: under the
|
||||
null the sign of each paired difference is exchangeable. With `n = 3`, the null has `2^3 = 8`
|
||||
sign vectors, so the smallest attainable two-sided p-value is `2/8 = 0.25`; this was verified by
|
||||
enumeration, not assumed.
|
||||
- **Exact Wilcoxon signed-rank** (average ranks for ties, zero differences dropped, exact null by
|
||||
enumerating `2^n` sign assignments, two-sided min-rank convention). At `n = 3` the exact p-values
|
||||
are coarse (0.50 or 1.00), so this test adds little.
|
||||
- **Paired-t 95% CI** using `t = 4.3027` (df = 2). This is an explicitly assumption-dependent,
|
||||
underpowered summary; it is not a reliable interval estimate for a population.
|
||||
|
||||
Interpretation rules applied:
|
||||
|
||||
- **No condition is statistically significant**, and none can be with three seeds.
|
||||
- **Non-significance is not evidence of no effect**; it reflects a sample far too small to resolve
|
||||
effects of this size.
|
||||
- Tests are **not** pooled across the six conditions (they are not replicates of one condition) nor
|
||||
across clients (clients are nested in the same trained model and seed).
|
||||
- Bootstrap CIs are deliberately **not** used: three seeds cannot support a reliable resampling
|
||||
distribution.
|
||||
|
||||
Conclusion: **prioritize descriptive reporting** (values, direction, seed consistency) over
|
||||
inferential claims for this grid.
|
||||
|
||||
---
|
||||
|
||||
## 5. Beta-selection stability
|
||||
|
||||
Source: `posthoc__selection_stability.csv`; per-seed selections are computed from each condition's
|
||||
`validation_per_seed.csv` (per-client-mean validation rare-class macro-F1).
|
||||
|
||||
| Dataset | alpha | pooled beta | per-seed beta (s0/s1/s2) | agree with pooled | best score | 2nd-best (beta, score) | margin | candidate range |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 0.00 | 0.75 / 0.50 / 0.00 | 1/3 | 0.5917 | 0.25, 0.5887 | 0.0030 | 0.0596 |
|
||||
| X-IIoTID | 0.50 | 0.75 | 0.50 / 1.00 / 0.75 | 1/3 | 0.5794 | 1.00, 0.5735 | 0.0059 | 0.0350 |
|
||||
| X-IIoTID | 0.75 | 0.50 | 0.50 / 0.50 / 0.00 | 2/3 | 0.4767 | 0.25, 0.4744 | 0.0024 | 0.0081 |
|
||||
| 5G-NIDD | 0.25 | 0.00 | 0.25 / 0.00 / 0.00 | 2/3 | 0.7615 | 0.25, 0.7493 | 0.0122 | 0.0392 |
|
||||
| 5G-NIDD | 0.50 | 1.00 | 1.00 / 0.00 / 1.00 | 2/3 | 0.6924 | 0.75, 0.6884 | 0.0040 | 0.0473 |
|
||||
| 5G-NIDD | 0.75 | 0.50 | 0.50 / 1.00 / 1.00 | 1/3 | 0.8091 | 1.00, 0.8043 | 0.0048 | 0.0310 |
|
||||
|
||||
Verification: the reported selection margins of **0.0024–0.0122** are confirmed exactly.
|
||||
|
||||
**Interpretation.** A margin of 0.002–0.012 is small relative to both the candidate-score range
|
||||
(0.008–0.060) and the across-seed std of the selected beta (0.022–0.139 in earlier analyses), and the
|
||||
pooled selection agrees with only 1–2 of 3 seeds. This makes the selection **fragile / weakly
|
||||
separated**: it should be described as "best among the tested candidates under the pooled validation
|
||||
procedure", **not** as an optimal or robust choice. A small margin does **not** prove the selected
|
||||
beta is wrong; it means the data cannot discriminate among close candidates. All selections remain
|
||||
**validation-only** (`test_used_for_selection = False`); test metrics are never used to select beta.
|
||||
|
||||
---
|
||||
|
||||
## 6. `S_j` participant-rule audit
|
||||
|
||||
Source: `artifacts/analysis/sj_participant_rule_analysis.csv` (162 class-condition cells = 18
|
||||
condition-seeds × 9 classes). Verified: **`|S_j| < M` in 53 / 162 cells** (X-IIoTID 19/7/3 and
|
||||
5G-NIDD 16/4/4 for α = 0.25/0.50/0.75); all 53 are exactly the cells with ≥1 missing client
|
||||
(`missing_clients` non-empty). `S_j` is the set of clients with `N_ij > 0` and a valid prototype.
|
||||
|
||||
**Why this is an implementation/reproduction assumption, not an ordinary statistical limitation.**
|
||||
Eq. (2)/(5.5) of the source paper sums over `M` clients but is silent on `N_ij = 0`; no official
|
||||
implementation exists (see `docs/baseline/PROTEAN_BASELINE_INVESTIGATION.md`). The project therefore
|
||||
*froze* `S_j = {i : N_ij > 0 and C_ij exists}` and used equal `1/|S_j|` weights in the baseline. In
|
||||
`53/162` cells the literal `1/M` reading (e.g. substituting a zero vector for an absent prototype)
|
||||
would differ from the `S_j` reading, so all results are **conditional on this rule**. More seeds
|
||||
would not remove this; only the (unavailable) official code, or a documented threat/robustness
|
||||
analysis across alternative participant rules, could.
|
||||
|
||||
---
|
||||
|
||||
## 7. Baseline reproduction discrepancy audit
|
||||
|
||||
**Reproduced baseline metrics available in this repository**
|
||||
(`docs/baseline/BASELINE_VALIDATION.md`, and `final__six_condition_master.csv`):
|
||||
|
||||
| Evidence | Value |
|
||||
|---|---|
|
||||
| X-IIoTID α=0.50 seed 0 accuracy | 0.8904 (macro-F1 0.7077, rare-F1 0.6096) |
|
||||
| 5G-NIDD α=0.50 seed 0 accuracy | 0.7344 (macro-F1 0.8369, rare-F1 0.8720) |
|
||||
| X-IIoTID original accuracy means (α=0.25/0.50/0.75) | 0.8754 / 0.8995 / 0.9206 |
|
||||
| 5G-NIDD original accuracy means (α=0.25/0.50/0.75) | 0.7310 / 0.7440 / 0.7510 |
|
||||
|
||||
**Unresolved data gap.** The task's comparison uses the paper's reported metrics ("Table IV") in
|
||||
`journal/main.tex`. That file and directory **do not exist in this repository**, and no file records
|
||||
the paper's numeric accuracy/F1 table (the baseline investigation records only that the paper reports
|
||||
"macro accuracy, accuracy, F1, precision", without values). Therefore the claimed deviations
|
||||
(X-IIoTID ≈ 5.1–9.7 percentage points below the paper; 5G-NIDD ≈ 4.6–8.4 percentage points above the
|
||||
paper) **could not be verified** here. Per the constraints, this is documented as a gap rather than
|
||||
reconstructed.
|
||||
|
||||
**Classification.** Requires the external source (`journal/main.tex` / the published paper and
|
||||
thesis). Even once sourced, such deviations must be reported as **reproduction differences**, not as
|
||||
automatic evidence of an implementation bug: metric definitions, train/test split (the project uses
|
||||
70/10/20 stratified, the paper 80/20), class handling (one dropped class on X-IIoTID), seed
|
||||
initialization, and nearest-prototype inference all differ from the paper and can move accuracy
|
||||
independently of any defect. No defect is claimed.
|
||||
|
||||
---
|
||||
|
||||
## 8. Metric-selection versus headline-metric audit
|
||||
|
||||
Traced in code:
|
||||
|
||||
- **Selection metric** (`scripts/run_baseline.py`, line ~303): `val_rare_class_macro_f1_per_client_mean`
|
||||
— the mean over eligible clients of each client's own validation rare-class macro-F1. Pooled
|
||||
selection across seeds uses this (`scripts/analyze_multiseed_beta.py`).
|
||||
- **Headline test metric**: `test_rare_class_macro_f1` from `rare_class_metrics(test_labels, test_pred,
|
||||
num_classes, union_rare)` — a **global** rare-class macro-F1 pooled over the union of client rare
|
||||
classes.
|
||||
|
||||
**Consequence.** The method is *selected* on a **per-client-mean validation** rare-class macro-F1 but
|
||||
*reported* on a **global test** rare-class macro-F1. These are not the same estimator: the per-client
|
||||
mean weights every client equally (and can be dominated by small clients), whereas the global metric
|
||||
weights by evaluation support. A beta that is best under the selection criterion need not be best
|
||||
under the reported criterion. The two quantities are therefore reported separately in this thesis
|
||||
(validation selection tables vs test evaluation tables) and must never be conflated; neither uses the
|
||||
test set for selection.
|
||||
|
||||
---
|
||||
|
||||
## 9. What can and cannot be concluded
|
||||
|
||||
**Supported by the evidence.**
|
||||
|
||||
- The comparison is a valid paired design with verified common manifests and splits.
|
||||
- Test rare-class macro-F1 changes are **small (≤ ~0.023)** and **condition-dependent**; only
|
||||
X-IIoTID α=0.50 improves in all three seeds.
|
||||
- Rare-class recall tends to fall when rare-class F1 rises (precision–recall trade-off).
|
||||
- The validation-selected beta is **weakly separated** and its pooled choice agrees with at most 2/3
|
||||
seeds; it is "best among tested candidates", not robust or optimal.
|
||||
- The `S_j` rule is materially exercised (53/162 cells) and conditions all results.
|
||||
|
||||
**Not supported / cannot be concluded.**
|
||||
|
||||
- Statistical superiority or significance (min attainable sign-flip p = 0.25).
|
||||
- That the method has *no* effect (non-significance ≠ equivalence).
|
||||
- A universal or globally optimal beta.
|
||||
- That prototype displacement implies better detection.
|
||||
- Any cross-condition pooling as if the 18 cells were independent replicates.
|
||||
- The paper-comparison deviation magnitudes (source absent).
|
||||
- Any privacy claim from the `N_ij` flow without a threat model.
|
||||
|
||||
---
|
||||
|
||||
## 10. Recommended replacement wording for thesis section A.7 (Indonesian)
|
||||
|
||||
> **Keterbatasan bukti statistik.** Grid eksperimen yang telah diselesaikan terdiri atas enam
|
||||
> kondisi (dua dataset × tiga nilai `α`) dengan tiga seed per kondisi, sehingga setiap perbandingan
|
||||
> berpasangan antara `original_protean` dan metode usulan hanya memiliki tiga selisih. Dengan tiga
|
||||
> selisih tersebut, uji permutasi tanda dua sisi yang eksak sekalipun tidak dapat menghasilkan nilai
|
||||
> p di bawah 0,25; karena itu hasil eksperimen ini dilaporkan secara deskriptif (nilai tengah,
|
||||
> arah perubahan, dan konsistensi antar-seed), bukan sebagai klaim signifikansi. Ketidaksignifikanan
|
||||
> tidak boleh ditafsirkan sebagai bukti bahwa metode tidak berpengaruh, melainkan sebagai
|
||||
> keterbatasan daya uji akibat jumlah seed yang kecil. Perbedaan hasil yang teramati juga kecil dan
|
||||
> bergantung pada kondisi; hanya kondisi X-IIoTID `α`=0,50 yang konsisten membaik pada ketiga seed.
|
||||
> Pemilihan `β` dilakukan sepenuhnya pada data validasi, dengan margin antara kandidat terbaik dan
|
||||
> kedua terbaik yang kecil (0,0024–0,0122), sehingga nilai `β` terpilih dinyatakan sebagai yang
|
||||
> terbaik di antara kandidat yang diuji pada prosedur validasi, bukan sebagai nilai optimal global
|
||||
> maupun pilihan yang kokoh. Selain itu, aturan peserta agregasi `S_j` merupakan asumsi reproduksi
|
||||
> yang secara material berpengaruh pada 53 dari 162 sel kelas-kondisi; seluruh hasil bersifat
|
||||
> kondisional terhadap asumsi tersebut. Generalisasi juga terbatas pada dua dataset, sepuluh
|
||||
> client, satu arsitektur, tiga tingkat `α`, tiga seed, dan lima kandidat `β` diskret.
|
||||
|
||||
*(This is proposed wording for review; no existing thesis document was edited.)*
|
||||
|
||||
---
|
||||
|
||||
## 11. Reproducibility
|
||||
|
||||
```bash
|
||||
# post-hoc audit (read-only; no training; writes posthoc__*.csv only)
|
||||
.venv/bin/python scripts/posthoc_statistical_audit.py
|
||||
```
|
||||
|
||||
The script:
|
||||
1. re-verifies pairing (dataset, alpha, seed, `manifest_content_hash`, recorded beta) for all 18
|
||||
cells and aborts on any mismatch;
|
||||
2. writes `posthoc__paired_global_per_seed.csv`, `posthoc__paired_global_by_condition.csv`,
|
||||
`posthoc__paired_test_results.csv`, `posthoc__selection_stability.csv`,
|
||||
`posthoc__per_class_summary.csv`, `posthoc__per_client_summary.csv`;
|
||||
3. refuses to overwrite any existing artifact.
|
||||
|
||||
Inputs are the committed `deepcmp__*`, `final__*`, `*__multiseed__validation_per_seed.csv`, and
|
||||
`sj_participant_rule_analysis.csv` tables plus per-run `metadata.json`. No network, no training, no
|
||||
partition regeneration.
|
||||
|
||||
---
|
||||
|
||||
## 12. Unresolved data gaps
|
||||
|
||||
1. **`journal/main.tex` / paper Table IV absent** — the paper's reported accuracy/F1 values cannot
|
||||
be verified from the repository; the reproduction-deviation magnitudes remain unconfirmed.
|
||||
2. **No official PROTEAN code** — the `S_j` participant rule and missing-class handling remain a
|
||||
reproduction assumption (material in 53/162 cells).
|
||||
3. **Three seeds only** — inferential power is fundamentally limited; no significant result is
|
||||
attainable and bootstrap/interval estimates would be unreliable.
|
||||
4. **One missing validation artifact set** for `xiiotid__alpha0.5__seed0__original_protean`
|
||||
(documented in the README) — does not affect the six selections but limits that cell's
|
||||
validation-side diagnostics.
|
||||
5. **Metric mismatch** between the per-client validation selection criterion and the global test
|
||||
headline metric cannot be removed post-hoc; it is a design/protocol property.
|
||||
|
||||
---
|
||||
|
||||
## Appendix — Limitations classification (task Phase 5)
|
||||
|
||||
| Item | Classification |
|
||||
|---|---|
|
||||
| A. Limited seeds / statistical evidence | Computable (done: descriptive + exact tests) and a **design-scope limitation** for significance |
|
||||
| B. `S_j` participant rule (53/162) | **Requires source-code/protocol verification** (no official code) → documented reproduction assumption |
|
||||
| C. Baseline reproduction discrepancy | **Requires the external source** (`journal/main.tex` / paper table; absent here); report as reproduction difference, not a defect |
|
||||
| D. Selection margins and seed agreement | **Computable** (done; margins 0.0024–0.0122 verified) |
|
||||
| E. Selection metric vs headline metric | **Computable/traceable in code** (confirmed: validation per-client mean vs global test rare-F1) |
|
||||
| F. Experimental scope (2 datasets, 10 clients, 1 architecture, 3 α, 3 seeds, 5 betas) | **Design-scope limitation** — cannot be removed post-hoc; needs additional experiments |
|
||||
| G. Support information / privacy (`N_ij` at server) | **Requires protocol verification (done: `N_ij` is sent) + threat-model analysis** for any privacy claim; no vulnerability is claimed here |
|
||||
@@ -0,0 +1,454 @@
|
||||
# Audit Ketangguhan (Robustness) dan Persiapan Penguji Tesis PROTEAN
|
||||
|
||||
Dokumen ini merupakan audit berbasis bukti dan bahan persiapan sidang untuk penelitian
|
||||
**agregasi prototype terbobot pada PROTEAN**. Seluruh angka diambil dari artefak eksperimen yang
|
||||
telah selesai; tidak ada pelatihan ulang, tidak ada perubahan pada algoritma, model, loss,
|
||||
optimizer, dataset, partisi, maupun hasil eksperimen. Skrip pendukung berada di
|
||||
`kisikisithesis/robustness_checks.py` dan bersifat deterministik serta hanya-baca.
|
||||
|
||||
Status repositori saat audit: branch `main`, HEAD `34bab45`, tidak ada perubahan pada berkas
|
||||
terlacak. Direktori `kisikisithesis/` seluruhnya baru.
|
||||
|
||||
---
|
||||
|
||||
## 1. Ringkasan Eksekutif
|
||||
|
||||
1. **Mekanisme yang diubah hanya agregasi prototype di sisi server.** Bobot client
|
||||
`w_ij(β) = N_ij^β / Σ_{k∈S_j} N_kj^β` dengan `β ∈ {0, 0.25, 0.50, 0.75, 1.00}`; model lokal,
|
||||
loss, optimizer, inferensi, dataset, dan manifest partisi tetap.
|
||||
2. **Sifat yang dijamin oleh rumus** (diverifikasi sintetis, `robustness_checks.py`):
|
||||
`β=0` = rata-rata aritmetik atas `S_j`; `β=1` = rata-rata terbobot dukungan; satu penyumbang
|
||||
invarian terhadap `β`; dukungan seragam invarian terhadap `β`; bobot selalu berjumlah 1;
|
||||
`N_ij=0` dan prototype hilang dikeluarkan dari `S_j`; `β` tak valid ditolak.
|
||||
3. **Efek langsung vs efek akhir-ke-akhir berbeda.** Displacement prototype `‖C̄_j(β)−C̄_j(0)‖`
|
||||
naik monoton terhadap `β` (X-IIoTID 1.305–3.422; 5G-NIDD 1.385–2.919 pada `β=1`), tetapi
|
||||
displacement besar **bukan** bukti deteksi membaik.
|
||||
4. **Hasil akhir-ke-akhir kecil dan bergantung kondisi.** Perbedaan rare-class macro-F1 uji:
|
||||
−0,0007 / +0,0230 / +0,0014 (X-IIoTID) dan −0,0010 / −0,0086 / +0,0168 (5G-NIDD). Hanya
|
||||
X-IIoTID α=0,50 yang arahnya konsisten pada ketiga seed.
|
||||
5. **Daya uji statistik tidak memadai.** Dengan 3 seed, p terkecil yang mungkin untuk uji tanda
|
||||
eksak dua sisi adalah **0,25** (diverifikasi); hanya X-IIoTID α=0,50 mencapainya. Maka tidak
|
||||
ada klaim signifikansi.
|
||||
6. **Pemilihan β rapuh secara relatif.** Margin terbaik vs terbaik kedua hanya **0,0024–0,0122**,
|
||||
dan kesepakatan seleksi pooled dengan seleksi per-seed paling banyak **2 dari 3**.
|
||||
7. **Asumsi reproduksi `S_j` material.** `|S_j| < M` pada **53 dari 162** sel kelas-kondisi.
|
||||
8. **Reproduksi vs paper belum dapat diverifikasi** karena `journal/main.tex` (dan Table IV paper)
|
||||
tidak ada di repositori ini.
|
||||
|
||||
Kalimat klaim yang aman: *"Di antara kandidat β yang diuji, pembobotan dukungan sampel kelas
|
||||
memberi perubahan rare-class macro-F1 yang kecil dan bergantung kondisi; hanya sebagian kondisi
|
||||
menunjukkan perbaikan yang konsisten antar-seed, dan tidak ada klaim signifikansi statistik."*
|
||||
|
||||
---
|
||||
|
||||
## 2. Inventaris Repositori dan Bukti
|
||||
|
||||
Sumber utama (semua sudah tersimpan, hanya-baca):
|
||||
|
||||
| Keperluan | Artefak |
|
||||
|---|---|
|
||||
| Protokol beku | `docs/research/EXPERIMENT_PROTOCOL.md`, `configs/experiments/experiment_protocol.yaml` |
|
||||
| Implementasi agregasi | `src/protean_research/aggregation/prototype_aggregation.py` |
|
||||
| Efek agregasi langsung | `src/protean_research/analysis/direct_prototype.py`, `scripts/analyze_prototype_effect.py` |
|
||||
| Snapshot prototype | `artifacts/analysis/*prototype_snapshot.json` (108 berkas; 18 sel × 6 konfigurasi) |
|
||||
| Partisi beku | `artifacts/partitions/*.json` (18 manifest) |
|
||||
| Perbandingan berpasangan | `artifacts/analysis/deepcmp__*.csv` |
|
||||
| Ringkasan enam kondisi | `artifacts/analysis/final__six_condition_master.csv` |
|
||||
| Kurva seleksi | `final__pooled_validation_curves.csv`, `final__selection_strength.csv` |
|
||||
| Sensitivitas β (tabel) | `final__prototype_displacement_by_beta.csv`, `..._by_class.csv` |
|
||||
| Heterogenitas terealisasi | `final__realized_heterogeneity.csv` |
|
||||
| Aturan `S_j` | `artifacts/analysis/sj_participant_rule_analysis.csv` |
|
||||
| Ekuivalensi β=0 | `*__multiseed__beta0_equivalence.csv` (5 kondisi; X-IIoTID α=0,50 tidak ada) |
|
||||
| Audit statistik | `docs/research/POSTHOC_STATISTICAL_AUDIT.md`, `artifacts/analysis/posthoc__*.csv` |
|
||||
| Metrik per-run | `results/<run>/test_global_metrics.json`, `test_per_class.csv`, `per_client.csv` |
|
||||
| Validasi per-seed | `artifacts/analysis/<ds>__alpha*__multiseed__validation_per_seed.csv` |
|
||||
| Baseline tereproduksi | `docs/baseline/BASELINE_VALIDATION.md`, `PROTEAN_BASELINE_INVESTIGATION.md` |
|
||||
|
||||
**Batas bukti yang harus dicatat:**
|
||||
- `journal/main.tex` / Table IV paper **tidak ada** → perbandingan metrik paper tidak dapat
|
||||
diverifikasi.
|
||||
- `xiiotid__alpha0.5__multiseed__beta0_equivalence.csv` **tidak ada**, dan satu run
|
||||
`xiiotid__alpha0.5__seed0__original_protean` kehilangan artefak validasi.
|
||||
- Hanya **3 seed** per kondisi.
|
||||
- Pemilihan β memakai **rata-rata per-client (validasi)**, sedangkan headline memakai
|
||||
**rare-class macro-F1 global (uji)**.
|
||||
|
||||
Analisis yang **dapat** dihitung dari artefak: seluruh perbandingan berpasangan, displacement
|
||||
langsung, stabilitas seleksi, audit `S_j`, ringkasan per-kelas/per-client.
|
||||
Analisis yang **memerlukan pelatihan baru**: menambah seed, dataset, arsitektur, atau kandidat β
|
||||
kontinu.
|
||||
|
||||
---
|
||||
|
||||
## 3. Penjelasan Matematis β dan Bobot Client (terverifikasi)
|
||||
|
||||
Diberikan himpunan peserta `S_j = { i : N_ij > 0 dan prototype lokal C_ij valid }`. Bobot
|
||||
ternormalisasi dan prototype global:
|
||||
|
||||
```
|
||||
w_ij(β) = N_ij^β / Σ_{k∈S_j} N_kj^β
|
||||
C̄_j(β) = Σ_{i∈S_j} w_ij(β) · C_ij
|
||||
```
|
||||
|
||||
**β = 0.** `N_ij^0 = 1` sehingga `w_ij = 1/|S_j|` dan `C̄_j(0)` adalah rata-rata aritmetik atas
|
||||
`S_j`. Ini adalah **kontrol setara**, bukan otomatis sama dengan baseline PROTEAN: kesetaraan hanya
|
||||
berlaku bila aturan peserta/penanganan kelas-tak-tersedia juga sama.
|
||||
|
||||
**β = 1.** `w_ij ∝ N_ij`. Karena setiap `C_ij` adalah rata-rata embedding kelas lokal, `C̄_j(1)`
|
||||
setara dengan rata-rata gabungan seluruh embedding kelas `j` yang disertakan bila seluruh asumsi
|
||||
terpenuhi.
|
||||
|
||||
**0 < β < 1.** Bobot tumbuh sub-linear terhadap dukungan: client berdukungan besar lebih
|
||||
berpengaruh, tetapi tidak sepenuhnya proporsional.
|
||||
|
||||
**Mengapa β besar tidak menjamin deteksi lebih baik.** Agregasi hanya menghasilkan posisi
|
||||
`C̄_j(β)`; inferensi memilih prototype terdekat (jarak L2). Meningkatnya bobot client berdukungan
|
||||
besar dapat menggeser `C̄_j` ke arah cluster padat tetapi **kurang mewakili** variasi serangan
|
||||
langka, sehingga menurunkan recall kelas langka. Jadi peningkatan `β` mengubah geometri prototype
|
||||
secara deterministik, tetapi hubungannya dengan kinerja deteksi bersifat empiris dan bergantung
|
||||
data.
|
||||
|
||||
**N_ij = 0.** Client tidak menyumbang prototype untuk kelas tersebut dan tidak masuk `S_j`;
|
||||
implementasi tidak pernah membuat vektor nol (`aggregate_class_prototypes`, baris penyaring
|
||||
`support <= 0 atau proto is None`).
|
||||
|
||||
**Mengapa penyebut dan aturan peserta penting.** Bila Eq. (2) paper (`1/M` atas semua client)
|
||||
diterapkan harfiah dengan mengisi prototype hilang sebagai nol, hasilnya berbeda dari `1/|S_j|`.
|
||||
Pada **53/162** sel kelas-kondisi, `|S_j| < M`, sehingga aturan ini material dan seluruh hasil
|
||||
bersifat kondisional.
|
||||
|
||||
**Sifat yang dijamin rumus** (diverifikasi): normalisasi (Σw=1), endpoint β, invariansi satu
|
||||
penyumbang/dukungan seragam, pengecualian `N_ij=0`, penolakan β tak valid.
|
||||
**Sifat yang bergantung data/implementasi**: besar displacement per kelas, client mana yang
|
||||
dominan, dan apakah deteksi membaik.
|
||||
|
||||
---
|
||||
|
||||
## 4. Temuan Sensitivitas Parameter yang Ada (efek agregasi langsung)
|
||||
|
||||
Dihitung ulang dari snapshot round-0 yang tersimpan oleh `kisikisithesis/robustness_checks.py`
|
||||
(tanpa pelatihan, snapshot tidak diubah).
|
||||
|
||||
**Tabel kondisi (mean displacement `‖C̄_j(1)−C̄_j(0)‖`, dari
|
||||
`final__prototype_displacement_by_beta.csv`):**
|
||||
|
||||
| Dataset | α=0.25 | α=0.50 | α=0.75 |
|
||||
|---|---|---|---|
|
||||
| X-IIoTID | 3.422 (β=0.25: 1.476; β=0.50: 2.403; β=0.75: 3.003) | 2.099 | 1.305 |
|
||||
| 5G-NIDD | 2.919 (1.295 / 2.055 / 2.559) | 2.340 | 1.385 |
|
||||
|
||||
Displacement naik **monoton** terhadap β di setiap kondisi; makin kecil α (makin timpang),
|
||||
makin besar displacement.
|
||||
|
||||
**Contoh sensitivitas per kelas (5G-NIDD α=0.25 seed 0, snapshot β=0):**
|
||||
|
||||
| Kelas | \|S_j\| | N_min | N_max | disp@0.5 | disp@1.0 | bobot client terbesar @β=1 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| HTTPFlood | 9 | 1 | 48.469 | 3.895 | 4.727 | 0.492 |
|
||||
| TCPConnectScan | 9 | 2 | 10.870 | 4.040 | 6.173 | 0.774 |
|
||||
| SlowrateDoS | 9 | 1 | 16.517 | 3.606 | 4.216 | 0.323 |
|
||||
| SYNFlood | 7 | 144 | 4.762 | 1.151 | 2.726 | 0.700 |
|
||||
| Benign | 10 | 1 | 185.504 | 2.335 | 3.229 | 0.555 |
|
||||
|
||||
Pada snapshot ini, mean disp@1.0 untuk kelas `|S_j|<M` (6 kelas) = **3.619** vs `|S_j|=M`
|
||||
(3 kelas) = **3.537**. Pada X-IIoTID α=0.25 seed 0: **3.620** vs **2.726** (kelas dengan
|
||||
peserta tidak lengkap bergeser lebih jauh). Ini **deskriptif**, bukan kausal.
|
||||
|
||||
**Kasus prototype berubah tanpa bukti perbaikan deteksi (penting).** Contoh `TCPConnectScan`
|
||||
bergeser 6.173 pada β=1, tetapi perubahan F1 uji kelas ini kecil dan bergantung kondisi. Jadi
|
||||
displacement ≠ kinerja deteksi; ini dibahas juga di §5 dan §8.
|
||||
|
||||
**Catatan utility langsung.** `prototype_effect_rows` dan `client_weight_rows`
|
||||
(`src/protean_research/analysis/direct_prototype.py`) menghitung displacement dan bobot dari
|
||||
snapshot yang sama; tidak memodifikasi snapshot dan tidak memicu pelatihan.
|
||||
|
||||
---
|
||||
|
||||
## 5. Temuan Robustness End-to-End yang Ada
|
||||
|
||||
Berbasis audit post-hoc (`posthoc__*.csv`); angka terverifikasi.
|
||||
|
||||
**Perbedaan berpasangan rare-class macro-F1 uji (proposed − original), per seed:**
|
||||
|
||||
| Dataset | α | β* | Δ s0 | Δ s1 | Δ s2 | mean | std berpasangan | std base | std proposed | +/−/0 | arah sama |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 0.00 | +0.0016 | −0.0020 | −0.0016 | −0.0007 | 0.0019 | 0.0472 | 0.0476 | 1/2/0 | tidak |
|
||||
| X-IIoTID | 0.50 | 0.75 | +0.0266 | +0.0163 | +0.0261 | **+0.0230** | 0.0058 | 0.0287 | 0.0267 | 3/0/0 | **ya** |
|
||||
| X-IIoTID | 0.75 | 0.50 | −0.0097 | +0.0101 | +0.0036 | +0.0014 | 0.0101 | 0.0778 | 0.0855 | 2/1/0 | tidak |
|
||||
| 5G-NIDD | 0.25 | 0.00 | −0.0050 | −0.0010 | +0.0029 | −0.0010 | 0.0039 | 0.0254 | 0.0242 | 1/2/0 | tidak |
|
||||
| 5G-NIDD | 0.50 | 1.00 | +0.0090 | −0.0811 | +0.0464 | −0.0086 | 0.0655 | 0.0371 | 0.0285 | 2/1/0 | tidak |
|
||||
| 5G-NIDD | 0.75 | 0.50 | −0.0011 | +0.0450 | +0.0067 | +0.0168 | 0.0247 | 0.0231 | 0.0145 | 2/1/0 | tidak |
|
||||
|
||||
- **Recall kelas langka turun pada 5/6 kondisi** (dan pada **semua 4** kondisi β>0): perolehan F1
|
||||
cenderung berasal dari presisi.
|
||||
- **Uji eksak (n=3):** sign-flip p = 0.75 / **0.25** / 0.75 / 0.75 / 1.00 / 0.50; Wilcoxon p =
|
||||
1.00 / 0.50 / 1.00 / 1.00 / 1.00 / 1.00. p terkecil yang mungkin = **0.25**.
|
||||
- **Confusion matrix** (5G-NIDD α=0.75 seed0): error dominan `Benign → UDPFlood` nyaris tetap
|
||||
(56.899 → 57.054; **+155**, bukan perbaikan); error `HTTPFlood → SlowrateDoS` turun 547;
|
||||
`Benign → TCPConnectScan` turun 520; `SlowrateDoS → HTTPFlood` naik 229.
|
||||
- **Per kelas** (mean ΔF1 atas semua 9 observasi): Tampering +0.052, Reconnaissance +0.030,
|
||||
ICMPFlood +0.027; SYNFlood −0.019, C&C −0.018. Konsistensi antar-seed tidak merata.
|
||||
- **Per client:** redistribusi — sebagian client naik, sebagian turun; pada α=0.25 beta=0 semua
|
||||
client "tidak berubah". Analisis dukungan rendah bersifat deskriptif (Pearson Δ vs fraksi
|
||||
dukungan ≈ −0,09; n=120).
|
||||
|
||||
**Diagnostik seleksi:** margin 0,0024–0,0122; kesepakatan pooled vs per-seed ≤2/3; pooled
|
||||
per-seed: X 0.75/0.50/0.00; 0.50/1.00/0.75; 0.50/0.50/0.00 dan 5G 0.25/0.00/0.00; 1.00/0.00/1.00;
|
||||
0.50/1.00/1.00.
|
||||
|
||||
**Eksperimen yang dikecualikan:** kondisi X-IIoTID α=0,50 tidak memiliki artefak ekuivalensi β=0
|
||||
dan kehilangan artefak validasi untuk original seed 0; analisis langsung memakai sel yang lengkap.
|
||||
|
||||
---
|
||||
|
||||
## 6. Pertanyaan Penguji dan Jawaban (≥20)
|
||||
|
||||
Format: **Jawaban lisan (ID)** · *Alasan teknis* · *Bukti* · *Tindak lanjut* · *Peringatan*.
|
||||
|
||||
**Q1. Apa yang berubah jika β = 0, 0,5, atau 1?**
|
||||
Jawab: "β=0 berarti rata-rata sederhana atas client penyumbang; β=1 berarti kontribusi
|
||||
proporsional terhadap jumlah sampel kelas; β=0,5 adalah kompromi sub-linear." *Alasan:* rumus di
|
||||
§3. *Bukti:* `test_beta0_is_equal_arithmetic_mean_over_s_j`, `test_beta1_is_support_weighted_mean`;
|
||||
`robustness_checks.py` (β=0 → [1.333,1.667]; β=1 → [1.857,2.286]; β=0,5 → [1.586,2.000]).
|
||||
*Tindak lanjut:* "Apakah β=0 identik dengan PROTEAN asli?" → lihat Q13. *Peringatan:* perubahan
|
||||
prototype tidak otomatis mengubah kinerja deteksi.
|
||||
|
||||
**Q2. Mengapa memilih himpunan kandidat β ini?**
|
||||
Jawab: "Lima kandidat diskret menutupi dua ujung dan tiga titik antara agar eksperimen tetap
|
||||
terkendali dan terjangkau, sesuai spesifikasi riset." *Alasan:* protokol membekukan
|
||||
`{0,0.25,0.5,0.75,1.0}`. *Bukti:* `EXPERIMENT_PROTOCOL.md`, `test_beta_candidates_are_the_frozen_protocol_values`.
|
||||
*Tindak lanjut:* "Mengapa tidak β kontinu?" *Peringatan:* belum ada analisis β kontinu.
|
||||
|
||||
**Q3. Apa yang terjadi bila partisi makin/tidak makin non-IID?**
|
||||
Jawab: "Makin kecil α, dukungan kelas makin timpang dan displacement prototype makin besar;
|
||||
sebaliknya pada α besar hampir seragam." *Alasan & bukti:* `final__realized_heterogeneity.csv`,
|
||||
`final__prototype_displacement_by_beta.csv` (displacement β=1 turun saat α naik). *Tindak lanjut:*
|
||||
"Apakah α menyebabkannya?" *Peringatan:* hanya asosiasi deskriptif; α adalah knob, bukan
|
||||
heterogenitas terealisasi.
|
||||
|
||||
**Q4. Bagaimana bila kelas langka absen di sebagian besar client?**
|
||||
Jawab: "Kelas itu hanya diagregasi dari client yang memilikinya (`S_j`); bila tidak ada penyumbang,
|
||||
kelas dihilangkan dari hasil agregasi." *Bukti:* `test_class_with_only_missing_contributors_is_omitted`,
|
||||
`aggregate_class_prototypes`. *Tindak lanjut:* "Bagaimana pengaruhnya pada evaluasi rare-class?"
|
||||
*Peringatan:* kelangkaan ekstrem membatasi daya metrik.
|
||||
|
||||
**Q5. Apa yang terjadi ketika N_ij = 0?**
|
||||
Jawab: "Client tidak masuk `S_j` dan tidak ada vektor nol yang dibuat." *Bukti:*
|
||||
`aggregate_class_prototypes` (penyaring `support <= 0`), `test_missing_class_is_excluded_from_s_j`.
|
||||
*Tindak lanjut:* Q14.
|
||||
|
||||
**Q6. Bagaimana bila satu client jauh lebih banyak sampelnya?**
|
||||
Jawab: "Pada β tinggi bobotnya mendominasi; pada β=0 semua setara." *Bukti:* synthetic dominance
|
||||
[1,1000] → bobot 0,999 bagian; snapshot TCPConnectScan bobot client terbesar ≈0,774 pada β=1.
|
||||
*Tindak lanjut:* "Apakah ini selalu baik?" *Peringatan:* dominasi dapat menekan variasi kelas
|
||||
langka.
|
||||
|
||||
**Q7. Mengapa dukungan spesifik kelas, bukan total sampel client?**
|
||||
Jawab: "Bobot diterapkan per kelas, jadi relevan jumlah sampel kelas tersebut; total client tidak
|
||||
mewakili dukungan kelas tertentu." *Alasan:* `N_ij` didefinisikan per kelas. *Bukti:*
|
||||
`prototype_aggregation.py` menggunakan `N_ij` per kelas. *Tindak lanjut:* "Bisakah total sampel
|
||||
dipakai?" → mungkin sebagai pembanding, tetapi berbeda semantik.
|
||||
|
||||
**Q8. Apakah prototype yang berubah pasti memperbaiki klasifikasi?**
|
||||
Jawab: "Tidak." *Bukti:* displacement monoton naik terhadap β, sedangkan respons deteksi
|
||||
non-monoton (`final__prototype_displacement_by_beta.csv` vs `final__pooled_validation_curves.csv`).
|
||||
*Tindak lanjut:* Q9. *Peringatan:* jangan menyamakan displacement dengan kinerja.
|
||||
|
||||
**Q9. Mengapa macro-F1 bisa naik tetapi recall/presisi turun?**
|
||||
Jawab: "F1 adalah rata-rata harmonik; pergeseran prototype dapat mengubah trade-off
|
||||
presisi–recall." *Bukti:* rare-class recall turun di 5/6 kondisi; contoh Tampering (ΔF1 +0,144,
|
||||
Δpresisi +0,212, Δrecall −0,009 di X-IIoTID α=0,50). *Tindak lanjut:* "Apakah itu bermanfaat?"
|
||||
|
||||
**Q10. Bagaimana bila β terpilih tidak terbaik di uji?**
|
||||
Jawab: "Mungkin saja; seleksi memakai validasi, uji tidak boleh dipakai memilih β." *Bukti:*
|
||||
`test_used_for_selection = false`; `final__selection_strength.csv`. *Tindak lanjut:* Q12.
|
||||
|
||||
**Q11. Mengapa hanya tiga seed, dan apa yang tidak dapat disimpulkan?**
|
||||
Jawab: "Tiga seed sesuai protokol proyek; belum cukup untuk klaim signifikansi." *Bukti:* p
|
||||
terkecil yang mungkin = 0,25 (`posthoc__paired_test_results.csv`). *Tindak lanjut:* Q16.
|
||||
*Peringatan:* ketidaksignifikanan ≠ tidak ada efek.
|
||||
|
||||
**Q12. Apa implikasi margin seleksi kecil?**
|
||||
Jawab: "Seleksi rapuh; β terpilih adalah 'terbaik di antara kandidat yang diuji', bukan optimal."
|
||||
*Bukti:* margin 0,0024–0,0122; kesepakatan ≤2/3. *Tindak lanjut:* "Perlu seed tambahan?"
|
||||
|
||||
**Q13. Mengapa β=0 dianggap setara dengan agregasi bobot seragam asli, dan bagaimana diverifikasi?**
|
||||
Jawab: "β=0 secara matematis = rata-rata seragam atas `S_j`; kesetaraan dengan baseline asli hanya
|
||||
bila aturan peserta sama." *Bukti:* `test_beta0_matches_original_protean_on_identical_s_j`;
|
||||
`*__multiseed__beta0_equivalence.csv` (kelas B; `aggregation_max_abs_diff` ≈ 2e−16; snapshot
|
||||
round-0 byte-identik). *Tindak lanjut:* "Mengapa tidak bitwise identik?" *Peringatan:* bukan bukti
|
||||
reproduksi PROTEAN asli; masih ada divergensi floating-point end-to-end. X-IIoTID α=0,50 tidak
|
||||
punya artefak ekuivalensi ini (celah).
|
||||
|
||||
**Q14. Bagaimana perilaku bila |S_j| < jumlah client?**
|
||||
Jawab: "Hanya client di `S_j` yang diagregasi; penyebut memakai jumlah bobot kontributor."
|
||||
*Bukti:* `sj_participant_rule_analysis.csv` (53/162 sel), kode `aggregate_class_prototypes`.
|
||||
*Tindak lanjut:* "Apa alternatifnya?" *Peringatan:* aturan ini asumsi reproduksi.
|
||||
|
||||
**Q15. Bisakah pembobotan dukungan memperburuk kinerja? Mekanisme masuk akal?**
|
||||
Jawab: "Bisa, secara prinsip." *Alasan:* pada β tinggi, client berdukungan besar mendominasi;
|
||||
jika variasi kelas langka hanya ada di client berdukungan kecil, prototype global bergeser menjauh
|
||||
dari variasi itu → recall turun. *Bukti:* hal ini teramati pada **5G-NIDD α=0,50** (Δ = −0,0086)
|
||||
dan pada α=0,25 (β=1 paling merugikan di tingkat client, dari sintesis). *Peringatan:* contoh lain
|
||||
bersifat mekanisme hipotetis, bukan hasil teramati.
|
||||
|
||||
**Q16. Eksperimen tambahan apa yang paling langsung menguji reprodusibilitas efek?**
|
||||
Jawab: "Menambah seed pada kondisi terkuat (X-IIoTID α=0,50) dan kondisi terlemah (5G-NIDD α=0,50),
|
||||
lalu seleksi dan evaluasi berpasangan; idealnya juga beberapa arsitektur." *Alasan:* daya uji
|
||||
bergantung jumlah seed. *Bukti:* §5. *Tindak lanjut:* Q21.
|
||||
|
||||
**Q17. Asumsi apa yang berbeda dari paper PROTEAN?**
|
||||
Jawab: "Penanganan kelas tak tersedia (`S_j`), split 70/10/20 (paper 80/20), nilai λ tanpa rujukan,
|
||||
seed tak diberi paper, dan preprocessing yang tidak dispesifikasikan." *Bukti:*
|
||||
`PROTEAN_BASELINE_INVESTIGATION.md`, `EXPERIMENT_PROTOCOL.md`. *Peringatan:* perbedaan reproduksi,
|
||||
bukan otomatis bug.
|
||||
|
||||
**Q18. Apa implikasi privasi karena server menerima N_ij?**
|
||||
Jawab: "Server menerima hitungan dukungan per kelas selain prototype; ini memperluas informasi yang
|
||||
diungkap dibanding hanya prototype." *Alasan & bukti:* `client.py` menghasilkan `supports`;
|
||||
`server.py` meneruskan `client_supports` ke agregasi. *Peringatan:* **belum ada analisis
|
||||
threat-model/privasi**, jadi tidak boleh diklaim sebagai kerentanan; ini hanya deskripsi aliran data.
|
||||
|
||||
**Q19. Parameter mana yang tetap, mana yang disetel, mana bagian metode usulan?**
|
||||
Jawab: "Tetap: model, loss, optimizer, inferensi, dataset, partisi, arsitektur. Disetel: β.
|
||||
Bagian metode usulan: bobot `N_ij^β` pada agregasi prototype server." *Bukti:* §3,
|
||||
`prototype_aggregation.py`, protokol.
|
||||
|
||||
**Q20. Apa yang akan menggugurkan klaim empiris utama?**
|
||||
Jawab: "Bila pada seed tambahan perbaikan X-IIoTID α=0,50 tidak muncul, atau bila β terpilih tidak
|
||||
stabil, maka klaim 'perbaikan bergantung kondisi' harus diperlemah; klaim keunggulan universal
|
||||
sudah tidak diajukan." *Bukti:* §5. *Peringatan:* tanpa seed tambahan, ini belum diuji.
|
||||
|
||||
**Q21. Mengapa tidak menjalankan β kontinu?**
|
||||
Jawab: "Di luar lingkup dan biaya; grid diskret sudah dibekukan." *Bukti:* protokol. *Tindak
|
||||
lanjut:* sebagai pekerjaan lanjutan.
|
||||
|
||||
**Q22. Apa arti 'dukungan sampel' dan mengapa bukan ukuran kualitas?**
|
||||
Jawab: "`N_ij` adalah jumlah sampel training kelas j pada client i; bukan skor kualitas prototype."
|
||||
*Bukti:* `AGENTS.md` §9/§10, `prototype_aggregation.py`.
|
||||
|
||||
*(22 pertanyaan; mencakup seluruh daftar yang diminta.)*
|
||||
|
||||
---
|
||||
|
||||
## 7. Rencana Demonstrasi Perubahan Parameter Terkendali
|
||||
|
||||
Tidak dieksekusi otomatis oleh dokumen ini; `robustness_checks.py` telah menjalankan versi
|
||||
read-only untuk sebagian skenario. Semua demo memakai input sintetis kecil atau snapshot tersimpan.
|
||||
|
||||
| # | Skenario | Input terkendali | Hasil yang diharapkan (rumus) | Implementasi / tes yang ada | Perlu tes baru? | Penjelasan lisan |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 1 | Endpoint β | `p=[[1,0],[0,2],[3,3]]`, `N=[5,10,20]` | β=0 → rata-rata; β=1 → terbobot | `test_beta0_...`, `test_beta1_...` | tidak | "Dua ujung rumus terbukti langsung." |
|
||||
| 2 | Normalisasi bobot | support apa pun > 0 | Σw = 1 | `test_normalized_weights_sum_to_one` | tidak | "Bobot selalu terdefinisi." |
|
||||
| 3 | Client dukungan nol | `N=[5,0,7]` | client 0 di luar `S_j` | `test_missing_class_is_excluded_from_s_j` | tidak | "Tanpa fabrikasi prototype." |
|
||||
| 4 | Dukungan seragam | `N=[4,4,4]` | invarian β | `test_equal_support_is_invariant_to_beta` | tidak | "Tanpa ketimpangan, β tak berpengaruh." |
|
||||
| 5 | Sangat timpang | `N=[1,1000]` | bobot ≈ [0,001; 0,999] | `robustness_checks.py` (dominance) | disarankan (unit test formal) | "Dominasi client besar." |
|
||||
| 6 | Ekuivalen β=0 | `S_j` identik | sama dengan rata-rata seragam | `test_beta0_matches_original_protean_on_identical_s_j`; `beta0_equivalence.csv` | tidak | "Kontrol, bukan asli." |
|
||||
| 7 | Stabilitas numerik / β tak valid | β ∈ {−0.1,1.1,None,"x",∞} | ValueError | `test_invalid_beta_is_rejected`, `test_output_is_finite_...` | tidak | "Validasi ketat." |
|
||||
| 8 | Sensitivitas satu client dominan terhadap prototype | snapshot nyata + client dominan | pergeseran prototype | `client_weight_rows`, `prototype_effect_rows` | tidak | "Pergeseran terukur, bukan kinerja." |
|
||||
|
||||
---
|
||||
|
||||
## 8. Audit Integritas Riset dan Batasan
|
||||
|
||||
**Klaim yang didukung eksperimen.** Desain berpasangan tervalidasi (manifest/split/seed sama);
|
||||
perbedaan rare-F1 uji kecil dan bergantung kondisi; hanya X-IIoTID α=0,50 konsisten positif;
|
||||
recall kelas langka cenderung turun; seleksi β rapuh; `S_j` material (53/162).
|
||||
|
||||
**Klaim yang hanya deskriptif.** Asosiasi timpang–displacement; redistribusi manfaat antar-client;
|
||||
dukungan rendah tidak dirugikan (asosiasi lemah, tanpa kausalitas).
|
||||
|
||||
**Asumsi reproduksi.** `S_j` dan penanganan kelas tak tersedia; split 70/10/20; λ=1,0; preprocessing;
|
||||
seed proyek; inisialisasi prototype round-0.
|
||||
|
||||
**Klaim yang harus dilemahkan/dihapus.** Superioritas/signifikansi; β optimal universal; bahwa
|
||||
displacement = deteksi lebih baik; keunggulan umum lintas dataset.
|
||||
|
||||
**Eksperimen tambahan yang memperkuat.** Penambahan seed (terutama X-IIoTID α=0,50 dan 5G α=0,50);
|
||||
dataset ketiga; arsitektur kedua; analisis β kontinu; analisis threat-model privasi `N_ij`.
|
||||
|
||||
**Analisis yang tidak layak dijalankan.** Uji inferensial dengan 3 seed (daya nihil); bootstrap
|
||||
interval dari 3 seed; menjadikan client sebagai replikasi independen (nested).
|
||||
|
||||
**Perhatian khusus:**
|
||||
- **Tiga seed** → p minimum 0,25.
|
||||
- **Celah reproduksi vs paper** → `journal/main.tex`/Table IV tidak ada; belum terverifikasi.
|
||||
- **`S_j`** → 53/162 sel; asumsi, bukan statistik.
|
||||
- **Metrik seleksi vs headline** → per-client mean (validasi) vs global (uji).
|
||||
- **Displacement vs deteksi** → dipisahkan.
|
||||
- **Privasi** → deskripsi aliran `N_ij`; belum ada threat-model.
|
||||
|
||||
---
|
||||
|
||||
## 9. Eksperimen Lanjutan Prioritas (dengan estimasi komputasi)
|
||||
|
||||
Estimasi berdasar runtime terukur (`artifacts/analysis/*__multiseed__runtime.csv`): 5G-NIDD
|
||||
≈ 62–65 menit/run, X-IIoTID ≈ 40–45 menit/run; 108 run ≈ 94,5 jam CPU (8 core).
|
||||
|
||||
| Prioritas | Eksperimen | Rasional | Estimasi biaya |
|
||||
|---|---|---|---|
|
||||
| 1 | Tambah seed (3→10) pada X-IIoTID α=0,50 dan 5G α=0,50 (kondisi terkuat & terlemah) | menguji reprodusibilitas efek terbesar & arah negatif | 2 kondisi × 7 seed × 6 config ≈ 84 run ≈ 70–75 jam |
|
||||
| 2 | Tambah seed penuh (3→10) semua 6 kondisi | daya uji dan stabilitas seleksi | ≈ 252 run ≈ 210–220 jam |
|
||||
| 3 | Dataset IDS ketiga | validitas eksternal | bergantung ukuran; ≈ 40–70 jam |
|
||||
| 4 | Arsitektur alternatif (mis. MLP/1D-CNN berbeda) | sensitivitas arsitektur | ≈ 90–200 jam |
|
||||
| 5 | Grid β kontinu (mis. 0.0–1.0 langkah 0.1) | resolusi kandidat | 6 kondisi × 3 seed × 6 β tambahan ≈ 108 run ≈ 90 jam |
|
||||
| 6 | Analisis threat-model privasi `N_ij` | klaim privasi | tanpa pelatihan (analistik) |
|
||||
|
||||
Tidak disarankan: uji signifikansi atau bootstrap dari 3 seed (informasi rendah, berisiko
|
||||
menyesatkan).
|
||||
|
||||
---
|
||||
|
||||
## 10. Cheat Sheet Pertahanan Lisan (satu halaman)
|
||||
|
||||
- **Mekanisme:** hanya agregasi prototype server; `w_ij(β)=N_ij^β/ΣN_kj^β`; `β∈{0,.25,.5,.75,1}`.
|
||||
- **β=0:** rata-rata seragam atas `S_j` (kontrol; setara baseline hanya bila aturan peserta sama).
|
||||
- **β=1:** terbobot dukungan; **β antara:** sub-linear.
|
||||
- **Hasil utama:** Δrare-F1 uji = X {−0,0007; +0,0230; +0,0014}, 5G {−0,0010; −0,0086; +0,0168}.
|
||||
- **Konsisten antar-seed:** hanya X-IIoTID α=0,50.
|
||||
- **Statistik:** n=3; p minimum 0,25; **tanpa klaim signifikansi**.
|
||||
- **Seleksi:** margin 0,0024–0,0122; rapuh; validasi-saja (`test_used_for_selection=false`).
|
||||
- **Direct vs end-to-end:** displacement naik monoton; deteksi non-monoton; dipisahkan.
|
||||
- **`S_j`:** 53/162 sel; asumsi reproduksi.
|
||||
- **Confusion dominan:** `Benign→UDPFlood` tetap (+155), bukan perbaikan.
|
||||
- **Jangan klaim:** superioritas universal, β optimal, displacement = deteksi lebih baik,
|
||||
signifikansi, kerentanan privasi tanpa threat-model.
|
||||
- **Celah:** paper Table IV tidak tersedia; X-IIoTID α=0,50 tanpa ekuivalensi β=0;
|
||||
artefak validasi seed0 original hilang.
|
||||
|
||||
---
|
||||
|
||||
## 11. Path Artefak dan Perintah Reproduksi
|
||||
|
||||
```bash
|
||||
# Pemeriksaan formula + efek agregasi langsung (read-only, tanpa pelatihan)
|
||||
.venv/bin/python kisikisithesis/robustness_checks.py
|
||||
|
||||
# Audit statistik post-hoc (read-only; hanya menulis posthoc__*.csv)
|
||||
.venv/bin/python scripts/posthoc_statistical_audit.py
|
||||
|
||||
# Analisis efek prototype langsung (read-only) - contoh
|
||||
.venv/bin/python scripts/analyze_prototype_effect.py \
|
||||
--snapshot artifacts/analysis/5gnidd__alpha0.25__seed0__support_weighted__beta0__prototype_snapshot.json \
|
||||
--class-names "Benign,HTTPFlood,ICMPFlood,SYNFlood,SYNScan,SlowrateDoS,TCPConnectScan,UDPFlood,UDPScan" \
|
||||
--output-dir /tmp/opencode/proto_effect # output di luar repo bila ingin menghindari file baru
|
||||
|
||||
# Uji unit agregasi & efek (read-only)
|
||||
.venv/bin/python -m pytest tests/test_support_weighted_aggregation.py tests/test_baseline_aggregation.py tests/test_prototype_effect.py
|
||||
```
|
||||
|
||||
Artefak kunci: `artifacts/analysis/deepcmp__*.csv`, `final__*.csv`,
|
||||
`posthoc__*.csv`, `sj_participant_rule_analysis.csv`, `*__multiseed__validation_per_seed.csv`,
|
||||
`*__multiseed__beta0_equivalence.csv`, `artifacts/partitions/*.json`.
|
||||
|
||||
---
|
||||
|
||||
## 12. Pertanyaan Tak Terjawab dan Bukti yang Hilang
|
||||
|
||||
1. **Metrik paper PROTEAN (Table IV)** tidak dapat diverifikasi: `journal/main.tex` tidak ada.
|
||||
2. **Ekuivalensi β=0 untuk X-IIoTID α=0,50** tidak tersedia (berkas tidak ada).
|
||||
3. **Artefak validasi seed 0 original X-IIoTID α=0,50** tidak lengkap.
|
||||
4. **Daya statistik** tidak memadai (3 seed); signifikansi tidak dapat diklaim.
|
||||
5. **Privasi `N_ij`** belum dianalisis dengan threat-model.
|
||||
6. **β kontinu** dan **generalisasi lintas dataset/arsitektur** belum diuji.
|
||||
7. **Ketidakpastian seleksi β** tidak dapat dipisahkan sepenuhnya dari variasi seed dengan n=3.
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic, read-only robustness checks for the PROTEAN thesis.
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
.venv/bin/python kisikisithesis/robustness_checks.py
|
||||
|
||||
Nothing is written and no experimental artifact is modified. The script:
|
||||
* exercises the *actual* aggregation implementation on tiny synthetic inputs
|
||||
(formula invariants);
|
||||
* recomputes direct aggregation effects on persisted round-0 prototype
|
||||
snapshots (no training, no snapshot mutation).
|
||||
|
||||
All values are printed deterministically so the thesis report can quote them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
from protean_research.aggregation.prototype_aggregation import ( # noqa: E402
|
||||
AggregationMode,
|
||||
aggregate_prototypes,
|
||||
aggregate_class_prototypes,
|
||||
normalized_support_weights,
|
||||
validate_beta,
|
||||
BETA_CANDIDATES,
|
||||
)
|
||||
from protean_research.analysis.direct_prototype import ( # noqa: E402
|
||||
client_weight_rows,
|
||||
prototype_effect_rows,
|
||||
)
|
||||
|
||||
ANALYSIS = ROOT / "artifacts" / "analysis"
|
||||
|
||||
|
||||
def section(title: str) -> None:
|
||||
print("\n" + "=" * 74)
|
||||
print(title)
|
||||
print("=" * 74)
|
||||
|
||||
|
||||
def check(label: str, ok: bool) -> None:
|
||||
print(f" [{'PASS' if ok else 'FAIL'}] {label}")
|
||||
|
||||
|
||||
def synthetic_invariants() -> None:
|
||||
section("A. Synthetic formula invariants (actual implementation)")
|
||||
p = [[1.0, 0.0], [0.0, 2.0], [3.0, 3.0]]
|
||||
s = [5.0, 10.0, 20.0]
|
||||
|
||||
eq = [sum(v[d] for v in p) / len(p) for d in range(2)]
|
||||
b0 = aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED, supports=s, beta=0.0)
|
||||
print(f" beta=0 aggregate = {b0}")
|
||||
print(f" arithmetic mean = {eq}")
|
||||
check("beta=0 equals the arithmetic mean over S_j",
|
||||
all(abs(a - b) < 1e-12 for a, b in zip(b0, eq)))
|
||||
|
||||
w1 = normalized_support_weights(s, 1.0)
|
||||
b1 = aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED, supports=s, beta=1.0)
|
||||
closed = [sum(w * v[d] for w, v in zip(w1, p)) for d in range(2)]
|
||||
print(f" beta=1 weights = {[round(w, 6) for w in w1]}")
|
||||
print(f" beta=1 aggregate = {b1}")
|
||||
check("beta=1 equals the support-weighted closed form",
|
||||
all(abs(a - b) < 1e-12 for a, b in zip(b1, closed)))
|
||||
|
||||
w05 = normalized_support_weights(s, 0.5)
|
||||
b05 = aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED, supports=s, beta=0.5)
|
||||
print(f" beta=0.5 weights = {[round(w, 6) for w in w05]}")
|
||||
print(f" beta=0.5 aggregate = {b05}")
|
||||
|
||||
single = aggregate_prototypes([[7.0, 9.0]], AggregationMode.SUPPORT_WEIGHTED,
|
||||
supports=[3.0], beta=1.0)
|
||||
check("single contributor is beta-invariant", single == [7.0, 9.0])
|
||||
|
||||
equal = [aggregate_prototypes(p, AggregationMode.SUPPORT_WEIGHTED,
|
||||
supports=[4.0, 4.0, 4.0], beta=b) for b in BETA_CANDIDATES]
|
||||
check("equal support is beta-invariant", all(g == equal[0] for g in equal))
|
||||
|
||||
for b in BETA_CANDIDATES:
|
||||
tot = sum(normalized_support_weights(s, b))
|
||||
if abs(tot - 1.0) > 1e-12:
|
||||
check(f"weights sum to 1 at beta={b}", False)
|
||||
break
|
||||
else:
|
||||
check("normalized weights sum to 1 for every beta candidate", True)
|
||||
|
||||
# zero-support exclusion and missing-class handling through the S_j builder
|
||||
contributions = {0: [([1.0], 5.0), ([2.0], 0.0), (None, 7.0)], 1: [(None, 0.0)]}
|
||||
agg = aggregate_class_prototypes(contributions, AggregationMode.SUPPORT_WEIGHTED, beta=1.0)
|
||||
print(f" S_j builder output = {agg}")
|
||||
check("N_ij=0 and missing prototypes are dropped from S_j", agg.get(0) == [1.0])
|
||||
check("a class with no valid contributor is omitted", 1 not in agg)
|
||||
|
||||
dom = normalized_support_weights([1.0, 1000.0], 1.0)
|
||||
print(f" dominance weights [1,1000] at beta=1 = {[round(w, 6) for w in dom]}")
|
||||
check("a single dominant client can dominate the aggregate at beta=1", dom[1] > 0.999)
|
||||
|
||||
rejected = 0
|
||||
for bad in (-0.1, 1.1, None, "x", float("inf")):
|
||||
try:
|
||||
validate_beta(bad)
|
||||
except (ValueError, TypeError):
|
||||
rejected += 1
|
||||
check("invalid beta values are rejected", rejected == 5)
|
||||
|
||||
|
||||
def snapshot_sensitivity(tag: str, path: Path, names: dict) -> None:
|
||||
section(f"B. Direct aggregation effect on a persisted snapshot: {tag}")
|
||||
snap = json.loads(path.read_text())
|
||||
print(f" snapshot: {path.relative_to(ROOT)} (round {snap['round']})")
|
||||
rows = prototype_effect_rows(snap, betas=[0.0, 0.5, 1.0], class_names=names)
|
||||
weights = client_weight_rows(snap, betas=[0.0, 0.25, 0.5, 0.75, 1.0], class_names=names)
|
||||
|
||||
by_class = {}
|
||||
for r in rows:
|
||||
by_class.setdefault(r["class_id"], {})[r["beta"]] = r
|
||||
maxw = {}
|
||||
for w in weights:
|
||||
if w["beta"] == 1.0:
|
||||
cur = maxw.get(w["class_id"])
|
||||
if cur is None or w["weight"] > cur["weight"]:
|
||||
maxw[w["class_id"]] = w
|
||||
|
||||
print(f" {'class':16} {'|S_j|':>5} {'sup_min':>8} {'sup_max':>9} "
|
||||
f"{'disp@0.5':>9} {'disp@1.0':>9} {'maxW@1.0':>9}")
|
||||
for cid in sorted(by_class):
|
||||
r = by_class[cid]
|
||||
n = r[0.0]["n_contributors"]
|
||||
d05 = r[0.5]["displacement_from_beta0"]
|
||||
d10 = r[1.0]["displacement_from_beta0"]
|
||||
mw = maxw.get(cid, {}).get("weight", float("nan"))
|
||||
name = (r[0.0]["class_name"] or cid)
|
||||
print(f" {str(name):16} {n:>5} {r[0.0]['support_min']:>8.1f} "
|
||||
f"{r[0.0]['support_max']:>9.1f} {d05:>9.4f} {d10:>9.4f} {mw:>9.4f}")
|
||||
partial = [cid for cid in by_class if by_class[cid][0.0]["n_contributors"] < 10]
|
||||
full = [cid for cid in by_class if by_class[cid][0.0]["n_contributors"] == 10]
|
||||
if partial:
|
||||
md = sum(by_class[c][1.0]["displacement_from_beta0"] for c in partial) / len(partial)
|
||||
print(f" mean disp@1.0, |S_j|<M classes ({len(partial)}): {md:.4f}")
|
||||
if full:
|
||||
md = sum(by_class[c][1.0]["displacement_from_beta0"] for c in full) / len(full)
|
||||
print(f" mean disp@1.0, |S_j|=M classes ({len(full)}): {md:.4f}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
synthetic_invariants()
|
||||
snapshot_sensitivity(
|
||||
"5G-NIDD, alpha=0.25, seed=0 (beta=0 snapshot)",
|
||||
ANALYSIS / "5gnidd__alpha0.25__seed0__support_weighted__beta0__prototype_snapshot.json",
|
||||
{i: n for i, n in enumerate(
|
||||
["Benign", "HTTPFlood", "ICMPFlood", "SYNFlood", "SYNScan",
|
||||
"SlowrateDoS", "TCPConnectScan", "UDPFlood", "UDPScan"])})
|
||||
snapshot_sensitivity(
|
||||
"X-IIoTID, alpha=0.25, seed=0 (beta=0 snapshot)",
|
||||
ANALYSIS / "xiiotid__alpha0.25__seed0__support_weighted__beta0__prototype_snapshot.json",
|
||||
{i: n for i, n in enumerate(
|
||||
["C&C", "Exfiltration", "Exploitation", "Lateral_movement", "Normal",
|
||||
"RDOS", "Reconnaissance", "Tampering", "Weaponization"])})
|
||||
print("\nREAD-ONLY CHECKS COMPLETE (no files written, no snapshots modified).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Post-hoc statistical audit of the completed PROTEAN experiment grid.
|
||||
|
||||
Read-only post-hoc analysis: no training, no partition regeneration, no
|
||||
modification of any existing artifact. Reads the audited comparison tables and
|
||||
per-run metadata under ``results/`` and writes NEW derived tables with unique
|
||||
``posthoc__`` names under ``artifacts/analysis/``.
|
||||
|
||||
Computes:
|
||||
* paired global per-seed differences (original_protean vs selected-beta
|
||||
support_weighted) and condition-level descriptive summaries;
|
||||
* exploratory paired tests (exact sign-flip permutation, exact Wilcoxon
|
||||
signed-rank, paired t 95% CI) with explicit small-n limitations;
|
||||
* validation beta-selection stability (pooled vs per-seed);
|
||||
* per-class and per-client descriptive summaries.
|
||||
|
||||
Run:
|
||||
.venv/bin/python scripts/posthoc_statistical_audit.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ANALYSIS = ROOT / "artifacts" / "analysis"
|
||||
RESULTS = ROOT / "results"
|
||||
|
||||
DATASETS = ["xiiotid", "5gnidd"]
|
||||
ALPHAS = [0.25, 0.50, 0.75]
|
||||
SEEDS = [0, 1, 2]
|
||||
METRICS = ["rare_class_macro_f1", "rare_class_recall", "macro_f1",
|
||||
"weighted_f1", "accuracy"]
|
||||
SELECTED = {("xiiotid", 0.25): 0.0, ("xiiotid", 0.5): 0.75, ("xiiotid", 0.75): 0.5,
|
||||
("5gnidd", 0.25): 0.0, ("5gnidd", 0.5): 1.0, ("5gnidd", 0.75): 0.5}
|
||||
T_ZERO = 1e-6 # numerical "effectively zero" for sign counts
|
||||
T_PRACTICAL = 0.02 # practical negligible band (matches deep comparison)
|
||||
T975_DF2 = 4.302652729 # t_{0.975, df=2}
|
||||
|
||||
|
||||
def read_csv(p: Path) -> list[dict]:
|
||||
with p.open() as fh:
|
||||
return list(csv.DictReader(fh))
|
||||
|
||||
|
||||
def write_csv(name: str, rows: list[dict]) -> None:
|
||||
# Guard: this script may only ever write its own posthoc__ outputs, and may
|
||||
# overwrite them so the analysis is exactly reproducible on rerun. It can
|
||||
# never touch any pre-existing analysis artifact.
|
||||
if not name.startswith("posthoc__"):
|
||||
raise SystemExit(f"refusing to write non-posthoc artifact: {name}")
|
||||
path = ANALYSIS / name
|
||||
with path.open("w", newline="") as fh:
|
||||
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
print("wrote", path)
|
||||
|
||||
|
||||
def mean(v):
|
||||
return statistics.mean(v)
|
||||
|
||||
|
||||
def std_sample(v):
|
||||
return statistics.stdev(v) if len(v) > 1 else 0.0
|
||||
|
||||
|
||||
def median(v):
|
||||
return statistics.median(v)
|
||||
|
||||
|
||||
# --- exploratory exact tests ------------------------------------------------
|
||||
def signflip_p(deltas):
|
||||
"""Exact two-sided sign-flip permutation test on the mean (n<=~20)."""
|
||||
d = list(deltas)
|
||||
obs = abs(sum(d))
|
||||
if not d or all(x == 0 for x in d):
|
||||
return 1.0
|
||||
total = 0
|
||||
ge = 0
|
||||
for signs in itertools.product((1, -1), repeat=len(d)):
|
||||
s = abs(sum(sign * x for sign, x in zip(signs, d)))
|
||||
total += 1
|
||||
if s >= obs - 1e-12:
|
||||
ge += 1
|
||||
return ge / total
|
||||
|
||||
|
||||
def wilcoxon_exact_p(deltas):
|
||||
"""Exact two-sided Wilcoxon signed-rank (min-rank convention), zeros dropped."""
|
||||
d = [x for x in deltas if abs(x) > T_ZERO]
|
||||
n = len(d)
|
||||
if n == 0:
|
||||
return 1.0
|
||||
order = sorted(range(n), key=lambda i: abs(d[i]))
|
||||
ranks = [0.0] * n
|
||||
i = 0
|
||||
while i < n:
|
||||
j = i
|
||||
while j + 1 < n and abs(d[order[j + 1]]) == abs(d[order[i]]):
|
||||
j += 1
|
||||
avg = (i + j) / 2 + 1
|
||||
for k in range(i, j + 1):
|
||||
ranks[order[k]] = avg
|
||||
i = j + 1
|
||||
w_pos = sum(ranks[i] for i in range(n) if d[i] > 0)
|
||||
w_min_obs = min(w_pos, sum(ranks) - w_pos)
|
||||
dist = defaultdict(int)
|
||||
for signs in itertools.product((1, -1), repeat=n):
|
||||
wp = sum(ranks[i] for i in range(n) if signs[i] > 0)
|
||||
dist[min(wp, sum(ranks) - wp)] += 1
|
||||
le = sum(c for w, c in dist.items() if w <= w_min_obs + 1e-12)
|
||||
return min(1.0, 2 * le / (2 ** n))
|
||||
|
||||
|
||||
def paired_t_ci(deltas):
|
||||
d = list(deltas)
|
||||
n = len(d)
|
||||
m = mean(d)
|
||||
s = std_sample(d)
|
||||
half = T975_DF2 * s / math.sqrt(n) if n > 1 else 0.0
|
||||
return m - half, m + half
|
||||
|
||||
|
||||
# --- pairing verification ---------------------------------------------------
|
||||
def manifests(ds, alpha, seed, beta):
|
||||
o = json.loads((RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__original_protean"
|
||||
/ "metadata.json").read_text())
|
||||
s = json.loads((RESULTS / f"{ds}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta:g}"
|
||||
/ "metadata.json").read_text())
|
||||
return o, s
|
||||
|
||||
|
||||
def main() -> None:
|
||||
per_seed = read_csv(ANALYSIS / "deepcmp__global_per_seed.csv")
|
||||
|
||||
# 1. Verify pairing and selected beta, rebuild a per-seed table. ----------
|
||||
rebuilt = []
|
||||
pair_ok = True
|
||||
for ds in DATASETS:
|
||||
for alpha in ALPHAS:
|
||||
beta = SELECTED[(ds, alpha)]
|
||||
for seed in SEEDS:
|
||||
o, s = manifests(ds, alpha, seed, beta)
|
||||
ok = (o["dataset"] == s["dataset"] == ds
|
||||
and abs(float(o["alpha"]) - alpha) < 1e-9
|
||||
and int(o["seed"]) == int(s["seed"]) == seed
|
||||
and o["manifest_content_hash"] == s["manifest_content_hash"]
|
||||
and abs(float(s["beta"]) - beta) < 1e-9)
|
||||
pair_ok = pair_ok and ok
|
||||
for m in METRICS:
|
||||
row = next(r for r in per_seed
|
||||
if r["dataset"] == ds and abs(float(r["alpha"]) - alpha) < 1e-9
|
||||
and int(r["seed"]) == seed and r["metric"] == m)
|
||||
rebuilt.append({
|
||||
"dataset": ds, "alpha": alpha, "seed": seed,
|
||||
"selected_beta": beta, "metric": m,
|
||||
"original": row["original"], "proposed": row["proposed"],
|
||||
"delta": row["delta"],
|
||||
"manifest_content_hash": o["manifest_content_hash"],
|
||||
"pairing_verified": ok})
|
||||
if not pair_ok:
|
||||
raise SystemExit("pairing verification FAILED")
|
||||
print("pairing verified for all 18 condition-seed cells (manifest hash match)")
|
||||
write_csv("posthoc__paired_global_per_seed.csv", rebuilt)
|
||||
|
||||
# 2. Condition-level descriptive summary. -------------------------------
|
||||
by_cond = []
|
||||
tests = []
|
||||
for ds in DATASETS:
|
||||
for alpha in ALPHAS:
|
||||
beta = SELECTED[(ds, alpha)]
|
||||
for m in METRICS:
|
||||
rows = [r for r in rebuilt if r["dataset"] == ds
|
||||
and abs(r["alpha"] - alpha) < 1e-9 and r["metric"] == m]
|
||||
base = [float(r["original"]) for r in rows]
|
||||
prop = [float(r["proposed"]) for r in rows]
|
||||
dlt = [float(r["delta"]) for r in rows]
|
||||
npos = sum(1 for d in dlt if d > T_ZERO)
|
||||
nneg = sum(1 for d in dlt if d < -T_ZERO)
|
||||
nzero = sum(1 for d in dlt if abs(d) <= T_ZERO)
|
||||
by_cond.append({
|
||||
"dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m,
|
||||
"original_seed0": base[0], "original_seed1": base[1], "original_seed2": base[2],
|
||||
"proposed_seed0": prop[0], "proposed_seed1": prop[1], "proposed_seed2": prop[2],
|
||||
"delta_seed0": dlt[0], "delta_seed1": dlt[1], "delta_seed2": dlt[2],
|
||||
"delta_mean": mean(dlt), "delta_std_sample": std_sample(dlt),
|
||||
"delta_median": median(dlt), "delta_min": min(dlt), "delta_max": max(dlt),
|
||||
"baseline_std_sample": std_sample(base),
|
||||
"proposed_std_sample": std_sample(prop),
|
||||
"n_pos": npos, "n_neg": nneg, "n_zero": nzero,
|
||||
"same_sign_all_seeds": (npos == 3 or nneg == 3)})
|
||||
lo, hi = paired_t_ci(dlt)
|
||||
tests.append({
|
||||
"dataset": ds, "alpha": alpha, "selected_beta": beta, "metric": m,
|
||||
"n_pairs": len(dlt),
|
||||
"deltas": "|".join(f"{d:.6f}" for d in dlt),
|
||||
"signflip_exact_p_two_sided": signflip_p(dlt),
|
||||
"wilcoxon_exact_p_two_sided": wilcoxon_exact_p(dlt),
|
||||
"paired_t_ci95_low": lo, "paired_t_ci95_high": hi,
|
||||
"min_attainable_signflip_p": 2 / (2 ** len(dlt)),
|
||||
"note": ("exploratory; 3 seeds; no inferential power; "
|
||||
"beta=0 conditions are numerical, not a treatment contrast")})
|
||||
write_csv("posthoc__paired_global_by_condition.csv", by_cond)
|
||||
write_csv("posthoc__paired_test_results.csv", tests)
|
||||
|
||||
# 3. Beta-selection stability. -----------------------------------------
|
||||
stability = []
|
||||
for ds in DATASETS:
|
||||
for alpha in ALPHAS:
|
||||
vs = read_csv(ANALYSIS / f"{ds}__alpha{alpha:g}__multiseed__validation_per_seed.csv")
|
||||
scores = defaultdict(dict) # seed -> beta -> score
|
||||
for r in vs:
|
||||
scores[int(r["seed"])][float(r["beta"])] = float(r["val_rare_macro_f1_per_client_mean"])
|
||||
per_seed_beta = {s: max(scores[s], key=lambda b: (scores[s][b], -b)) for s in scores}
|
||||
pooled = {b: mean([scores[s][b] for s in SEEDS]) for b in sorted(next(iter(scores.values())))}
|
||||
pooled_beta = max(pooled, key=lambda b: (pooled[b], -b))
|
||||
ordered = sorted(pooled.values(), reverse=True)
|
||||
agree = sum(1 for s in SEEDS if abs(per_seed_beta[s] - pooled_beta) < 1e-9)
|
||||
second_beta = sorted((b for b in pooled if abs(b - pooled_beta) > 1e-9),
|
||||
key=lambda b: -pooled[b])[0]
|
||||
stability.append({
|
||||
"dataset": ds, "alpha": alpha,
|
||||
"pooled_selected_beta": pooled_beta,
|
||||
"per_seed_beta_s0": per_seed_beta[0],
|
||||
"per_seed_beta_s1": per_seed_beta[1],
|
||||
"per_seed_beta_s2": per_seed_beta[2],
|
||||
"n_seeds_agree_pooled": agree,
|
||||
"pooled_score_selected": pooled[pooled_beta],
|
||||
"second_best_beta": second_beta,
|
||||
"second_best_score": pooled[second_beta],
|
||||
"margin_best_minus_second": pooled[pooled_beta] - pooled[second_beta],
|
||||
"candidate_score_range": ordered[0] - ordered[-1],
|
||||
"test_used_for_selection": False})
|
||||
write_csv("posthoc__selection_stability.csv", stability)
|
||||
|
||||
# 4. Per-class summary. --------------------------------------------------
|
||||
pc = read_csv(ANALYSIS / "deepcmp__per_class.csv")
|
||||
grp = defaultdict(list)
|
||||
for r in pc:
|
||||
grp[(r["dataset"], r["class_name"])].append(float(r["delta_f1"]))
|
||||
per_class = []
|
||||
for (ds, name), vals in sorted(grp.items(), key=lambda kv: -mean(kv[1])):
|
||||
per_class.append({
|
||||
"dataset": ds, "class_name": name, "n_obs": len(vals),
|
||||
"mean_delta_f1": mean(vals), "median_delta_f1": median(vals),
|
||||
"min_delta_f1": min(vals), "max_delta_f1": max(vals),
|
||||
"n_positive": sum(1 for v in vals if v > 0),
|
||||
"n_negative": sum(1 for v in vals if v < 0)})
|
||||
write_csv("posthoc__per_class_summary.csv", per_class)
|
||||
|
||||
# 5. Per-client summary (descriptive only). ------------------------------
|
||||
cl = read_csv(ANALYSIS / "deepcmp__per_client.csv")
|
||||
by_condition = defaultdict(list)
|
||||
for r in cl:
|
||||
by_condition[(r["dataset"], float(r["alpha"]))].append(r)
|
||||
per_client = []
|
||||
for (ds, alpha), rows in sorted(by_condition.items()):
|
||||
d = [float(r["delta_rare_f1"]) for r in rows]
|
||||
per_client.append({
|
||||
"dataset": ds, "alpha": alpha, "selected_beta": SELECTED[(ds, alpha)],
|
||||
"n_client_seed_obs": len(d),
|
||||
"n_improved_gt_0.02": sum(1 for x in d if x > T_PRACTICAL),
|
||||
"n_degraded_lt_-0.02": sum(1 for x in d if x < -T_PRACTICAL),
|
||||
"n_unchanged": sum(1 for x in d if abs(x) <= T_PRACTICAL),
|
||||
"mean_delta": mean(d), "median_delta": median(d),
|
||||
"min_delta": min(d), "max_delta": max(d),
|
||||
"std_sample_within_condition": std_sample(d)})
|
||||
write_csv("posthoc__per_client_summary.csv", per_client)
|
||||
|
||||
# --- console summary ----------------------------------------------------
|
||||
print("\n=== paired rare-class macro-F1 (delta = proposed - original) ===")
|
||||
for r in by_cond:
|
||||
if r["metric"] != "rare_class_macro_f1":
|
||||
continue
|
||||
print(f" {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} "
|
||||
f"mean={float(r['delta_mean']):+.4f} std={float(r['delta_std_sample']):.4f} "
|
||||
f"base_std={float(r['baseline_std_sample']):.4f} prop_std={float(r['proposed_std_sample']):.4f} "
|
||||
f"pos/neg/zero={r['n_pos']}/{r['n_neg']}/{r['n_zero']} same_sign={r['same_sign_all_seeds']}")
|
||||
print("\n=== exact tests (rare-class macro-F1) ===")
|
||||
for r in tests:
|
||||
if r["metric"] != "rare_class_macro_f1":
|
||||
continue
|
||||
print(f" {r['dataset']:8} a{r['alpha']:<4} beta={r['selected_beta']:<4} "
|
||||
f"signflip_p={r['signflip_exact_p_two_sided']:.4f} "
|
||||
f"wilcoxon_p={r['wilcoxon_exact_p_two_sided']:.4f} "
|
||||
f"tCI=[{r['paired_t_ci95_low']:+.4f},{r['paired_t_ci95_high']:+.4f}] "
|
||||
f"(min signflip p={r['min_attainable_signflip_p']:.4f})")
|
||||
print("\n=== selection stability ===")
|
||||
for r in stability:
|
||||
print(f" {r['dataset']:8} a{r['alpha']:<4} pooled={r['pooled_selected_beta']:<4} "
|
||||
f"per-seed={r['per_seed_beta_s0']}/{r['per_seed_beta_s1']}/{r['per_seed_beta_s2']} "
|
||||
f"agree={r['n_seeds_agree_pooled']}/3 margin={float(r['margin_best_minus_second']):.4f} "
|
||||
f"range={float(r['candidate_score_range']):.4f}")
|
||||
print("\n=== S_j audit ===")
|
||||
sj = read_csv(ANALYSIS / "sj_participant_rule_analysis.csv")
|
||||
lt = sum(1 for r in sj if int(r["S_j"]) < int(r["M"]))
|
||||
print(f" |S_j| < M cells: {lt} / {len(sj)}")
|
||||
print("\n=== per-class extremes (mean delta F1) ===")
|
||||
print(" top +:", ", ".join(f"{r['dataset']}/{r['class_name']} {float(r['mean_delta_f1']):+.4f}"
|
||||
for r in per_class[:4]))
|
||||
print(" top -:", ", ".join(f"{r['dataset']}/{r['class_name']} {float(r['mean_delta_f1']):+.4f}"
|
||||
for r in per_class[-4:]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user