research: finalize experimental synthesis
No files matched your search
@@ -0,0 +1,91 @@
|
||||
dataset,alpha,seed,beta,n_eligible,n_improve_vs_beta0,n_degrade_vs_beta0,n_unchanged,mean_delta_rare_macro_f1,median_delta_rare_macro_f1,min_delta_rare_macro_f1,max_delta_rare_macro_f1
|
||||
xiiotid,0.25,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.25,0,0.25,10,2,5,3,-0.0064673967169698865,-0.00222622943520559,-0.04293419597212661,0.01876993547229222
|
||||
xiiotid,0.25,0,0.5,10,2,7,1,-0.021132734174687993,-0.029027854121098817,-0.07692307692307698,0.05741836974395109
|
||||
xiiotid,0.25,0,0.75,10,5,4,1,0.037652802750874605,0.0007104290991758952,-0.11166156370194247,0.3758987633016968
|
||||
xiiotid,0.25,0,1.0,10,4,5,1,-0.037995631629473454,-0.022968705139247758,-0.25698055252395446,0.14104421200350115
|
||||
xiiotid,0.25,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.25,1,0.25,10,7,1,2,0.02181236632914685,0.022496227330692564,-0.006350012700025401,0.07175976877899254
|
||||
xiiotid,0.25,1,0.5,10,7,2,1,0.04461512377469677,0.02108028499658518,-0.015820270172650433,0.23562664832014674
|
||||
xiiotid,0.25,1,0.75,10,5,4,1,0.04324763034707273,0.0052932124515639445,-0.018266942973367145,0.316706904876871
|
||||
xiiotid,0.25,1,1.0,10,5,4,1,0.0385170238078155,0.0052932124515639445,-0.038742745973951775,0.37740563530037213
|
||||
xiiotid,0.25,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.25,2,0.25,10,5,3,2,-0.024384536677342937,0.013768617160351215,-0.3028515240904621,0.08185020136870569
|
||||
xiiotid,0.25,2,0.5,10,4,4,2,-0.04103939310459288,0.0,-0.3397932816537467,0.11851572403806943
|
||||
xiiotid,0.25,2,0.75,10,3,5,2,-0.1176216808834886,-0.03792837801605936,-0.6011131725417439,0.12413263414375775
|
||||
xiiotid,0.25,2,1.0,10,3,6,1,-0.1793367342416782,-0.1787074870427493,-0.6775352307267201,0.07541312385625187
|
||||
xiiotid,0.5,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.5,0,0.25,10,9,1,0,0.014447779488805679,0.008862381334935221,-0.0048289738430582485,0.059259259259259345
|
||||
xiiotid,0.5,0,0.5,10,8,2,0,0.02842211408129357,0.01786607568659304,-0.02864627632429495,0.09259259259259256
|
||||
xiiotid,0.5,0,0.75,10,7,3,0,0.020663517144586436,0.011529026224022465,-0.03579795256891627,0.09259259259259256
|
||||
xiiotid,0.5,0,1.0,10,4,6,0,-0.005896451782293366,-0.0015897910791723424,-0.07604807732022623,0.04139430366632779
|
||||
xiiotid,0.5,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.5,1,0.25,10,6,2,2,0.007580540867216318,0.00691255565746568,-0.004456876015858091,0.023990860624524046
|
||||
xiiotid,0.5,1,0.5,10,7,2,1,0.016564399764498333,0.016189304599793786,-0.005423903365265936,0.04237609975314893
|
||||
xiiotid,0.5,1,0.75,10,8,1,1,0.026535489229920235,0.02453891247906187,-0.002998819235144723,0.0871872501828449
|
||||
xiiotid,0.5,1,1.0,10,9,0,1,0.054492862707554934,0.03303785159404127,0.0,0.19273339273339285
|
||||
xiiotid,0.5,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.5,2,0.25,10,7,1,2,0.011191488545697032,0.009832777111244473,-0.0007113366760957973,0.029454054607266078
|
||||
xiiotid,0.5,2,0.5,10,8,0,2,0.02389697185121845,0.019768783103869897,0.0,0.06966444292595181
|
||||
xiiotid,0.5,2,0.75,10,8,1,1,0.05781929956312084,0.0358882506416448,-0.0018572153733706909,0.1770966368667518
|
||||
xiiotid,0.5,2,1.0,10,6,3,1,0.038758776558190745,0.0271534489476824,-0.4028071047074897,0.2675649974447385
|
||||
xiiotid,0.75,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.75,0,0.25,10,5,5,0,8.859496571733683e-05,-0.001404275185996784,-0.019237420306723707,0.026871905512682204
|
||||
xiiotid,0.75,0,0.5,10,6,4,0,0.007584665753678871,0.013188342258148383,-0.0361435910590483,0.048012173012172976
|
||||
xiiotid,0.75,0,0.75,10,3,7,0,-0.009935209564988324,-0.010284597102047655,-0.04332234195639206,0.02528490028490027
|
||||
xiiotid,0.75,0,1.0,10,1,9,0,-0.017657004095942678,-0.013236301949758722,-0.0650297324550243,0.02220072031392789
|
||||
xiiotid,0.75,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.75,1,0.25,10,9,0,1,0.01311050414693552,0.010627333455872107,0.0,0.03177895616667353
|
||||
xiiotid,0.75,1,0.5,10,9,0,1,0.014643364516424288,0.010097783138646538,0.0,0.04017223028543784
|
||||
xiiotid,0.75,1,0.75,10,9,0,1,0.0127774748399905,0.007494938523124495,0.0,0.03752345215759845
|
||||
xiiotid,0.75,1,1.0,10,7,2,1,0.011258579628480089,0.005364141224669883,-0.011620820732722359,0.05776636713735561
|
||||
xiiotid,0.75,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
xiiotid,0.75,2,0.25,10,7,3,0,-0.005211888340901932,0.0029998983543945423,-0.12121212121212133,0.03817567567567559
|
||||
xiiotid,0.75,2,0.5,10,6,4,0,-0.007096659882482407,0.0024387861756007545,-0.1523809523809524,0.047814685314685246
|
||||
xiiotid,0.75,2,0.75,10,5,5,0,-0.00033649270926734557,-0.0004809578413735027,-0.08602150537634423,0.05314748201438846
|
||||
xiiotid,0.75,2,1.0,10,6,4,0,-0.002902966134960444,0.0017994539960968226,-0.12121212121212133,0.05314748201438846
|
||||
5gnidd,0.25,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.25,0,0.25,10,3,3,4,0.010938546012782963,0.0,-0.07653222666610299,0.18940707383214095
|
||||
5gnidd,0.25,0,0.5,10,3,4,3,-0.01890975297353068,0.0,-0.10737418541550403,0.023843748084083516
|
||||
5gnidd,0.25,0,0.75,10,3,4,3,-0.023097869142088355,0.0,-0.14782686599098172,0.08120187997137751
|
||||
5gnidd,0.25,0,1.0,10,3,4,3,-0.0019048472317972909,0.0,-0.1484580166381656,0.2777777777777778
|
||||
5gnidd,0.25,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.25,1,0.25,10,3,3,4,-0.003968051685100719,0.0,-0.035080645161290336,0.006988021376510556
|
||||
5gnidd,0.25,1,0.5,10,2,4,4,-0.03304496144720663,0.0,-0.31650978982621863,0.0007581492441447457
|
||||
5gnidd,0.25,1,0.75,10,2,5,3,-0.04279009081982739,-1.5429040849390674e-06,-0.4057158717743099,0.05971472095277662
|
||||
5gnidd,0.25,1,1.0,10,3,4,3,-0.0319707599550853,0.0,-0.3007384916695693,0.05547720837266114
|
||||
5gnidd,0.25,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.25,2,0.25,10,0,4,6,-0.04347321695249963,0.0,-0.20521276595744675,0.0
|
||||
5gnidd,0.25,2,0.5,10,1,3,6,-0.05526190135183806,0.0,-0.2831161695447409,0.002564102564102555
|
||||
5gnidd,0.25,2,0.75,10,1,4,5,-0.05176638308929169,0.0,-0.26831675979891445,0.09614699026463747
|
||||
5gnidd,0.25,2,1.0,10,4,3,3,-0.03757538403361462,0.0,-0.2684826405938592,0.1510489510489511
|
||||
5gnidd,0.5,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.5,0,0.25,10,2,7,1,-0.07713608929072921,-0.06868699969879549,-0.20363636363636362,0.030406580954845275
|
||||
5gnidd,0.5,0,0.5,10,7,2,1,0.025889785020554917,0.037598391946217946,-0.03433523137896821,0.06985074626865678
|
||||
5gnidd,0.5,0,0.75,10,5,4,1,0.012063078830364903,0.005480145076796034,-0.06785615288783342,0.07125000000000004
|
||||
5gnidd,0.5,0,1.0,10,6,3,1,0.030279508202206325,0.038493190742204336,-0.0884443881819511,0.10341347704408088
|
||||
5gnidd,0.5,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.5,1,0.25,10,3,3,4,-0.05952678669505902,0.0,-0.32307692307692304,0.01253088893341342
|
||||
5gnidd,0.5,1,0.5,10,3,5,2,-0.06985503383490366,-0.0002538055290239316,-0.423076923076923,0.018147843026199484
|
||||
5gnidd,0.5,1,0.75,10,3,5,2,-0.08992415594170661,-0.010869565217391242,-0.548076923076923,0.023966835082264493
|
||||
5gnidd,0.5,1,1.0,10,3,5,2,-0.10147959420243684,-0.010869565217391242,-0.6564102564102563,0.06308143239454578
|
||||
5gnidd,0.5,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.5,2,0.25,10,5,4,1,0.01020569435994877,0.00047281323877068626,-0.026210680207824577,0.07260640841702526
|
||||
5gnidd,0.5,2,0.5,10,5,4,1,0.006281201274024195,0.0021822149481723896,-0.04423375396450502,0.07378740970072228
|
||||
5gnidd,0.5,2,0.75,10,7,2,1,0.08127477908163376,0.08683194022289764,-0.030503434250412576,0.19744802710549003
|
||||
5gnidd,0.5,2,1.0,10,7,2,1,0.08661837030164057,0.10629718599862731,-0.033724722765818727,0.2015644281090505
|
||||
5gnidd,0.75,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.75,0,0.25,10,5,4,1,-0.011293149808252767,0.0008702225660137675,-0.11428571428571432,0.02246687492367061
|
||||
5gnidd,0.75,0,0.5,10,4,6,0,0.002110296281339108,-0.002470904928741302,-0.04342273307790534,0.07435002353424269
|
||||
5gnidd,0.75,0,0.75,10,4,5,1,-0.010014602019574038,-0.0009496838684330267,-0.1261904761904763,0.08104016437931305
|
||||
5gnidd,0.75,0,1.0,10,4,5,1,-0.018907127202940695,-0.008343874739802304,-0.1470238095238096,0.08792669889742655
|
||||
5gnidd,0.75,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.75,1,0.25,10,8,0,2,0.057402269299170985,0.034854877143638385,0.0,0.15555555555555545
|
||||
5gnidd,0.75,1,0.5,10,6,3,1,0.08504839334967931,0.036259163913778814,-0.01717171717171706,0.2666666666666666
|
||||
5gnidd,0.75,1,0.75,10,6,3,1,0.06946744905696867,0.036259163913778814,-0.02220504036522919,0.23333333333333328
|
||||
5gnidd,0.75,1,1.0,10,6,3,1,0.08868113938603633,0.034854877143638385,-0.03122327240008249,0.32110579479000534
|
||||
5gnidd,0.75,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0
|
||||
5gnidd,0.75,2,0.25,10,0,6,4,-0.008941010132134792,-0.007204243679679367,-0.022597337621727998,0.0
|
||||
5gnidd,0.75,2,0.5,10,4,2,4,0.005804717069634502,0.0,-0.005379760304999981,0.02370370370370367
|
||||
5gnidd,0.75,2,0.75,10,1,6,3,-0.019143806586356792,-0.0066193960137049535,-0.07992013632257533,0.0032679738562091387
|
||||
5gnidd,0.75,2,1.0,10,4,2,4,0.008864477513365766,0.0,-0.007116151507512436,0.03763472261159395
|
||||
|
@@ -0,0 +1,31 @@
|
||||
dataset,alpha,beta,pooled_validation_mean,sample_std,pooled_rare_recall,selected
|
||||
xiiotid,0.25,0.0,0.5917038146634539,0.08952883653922933,0.7594821244948339,True
|
||||
xiiotid,0.25,0.25,0.5886906256417319,0.06643525635398827,0.7807713039840465,False
|
||||
xiiotid,0.25,0.5,0.5858514801619258,0.044857849968223555,0.7725355473131942,False
|
||||
xiiotid,0.25,0.75,0.5794633987349401,0.0649060573642385,0.7607912573123681,False
|
||||
xiiotid,0.25,1.0,0.5320987006423418,0.045718338534042856,0.704574711153059,False
|
||||
xiiotid,0.5,0.0,0.5443606875936857,0.09144660841267144,0.8390875794416744,False
|
||||
xiiotid,0.5,0.25,0.5554339572275921,0.09006816161098347,0.8412777237951431,False
|
||||
xiiotid,0.5,0.5,0.5673218494926892,0.08853898624879539,0.8388718664026588,False
|
||||
xiiotid,0.5,0.75,0.5793667895728949,0.07512057331236716,0.8290881018460031,True
|
||||
xiiotid,0.5,1.0,0.5734790834215032,0.1004496749103663,0.8207630276313397,False
|
||||
xiiotid,0.75,0.0,0.471705692507431,0.055329314644425844,0.7872893964865675,False
|
||||
xiiotid,0.75,0.25,0.47436809609801456,0.04605501919031881,0.7842747460964024,False
|
||||
xiiotid,0.75,0.5,0.47674948263663786,0.04662425884469754,0.7865311249627326,True
|
||||
xiiotid,0.75,0.75,0.47254095002934254,0.04602831582869046,0.7807290076221336,False
|
||||
xiiotid,0.75,1.0,0.46860522897329,0.044741855341153436,0.7813087218498457,False
|
||||
5gnidd,0.25,0.0,0.7614534210908004,0.021902537932118035,0.9085770977445251,True
|
||||
5gnidd,0.25,0.25,0.7492858468825278,0.017106975874172658,0.9071367963077813,False
|
||||
5gnidd,0.25,0.5,0.7257145491666086,0.01101731431676257,0.8949473995819205,False
|
||||
5gnidd,0.25,0.75,0.7222353067403979,0.007966771094060871,0.8797107780157537,False
|
||||
5gnidd,0.25,1.0,0.7376364240173013,0.002755842556750987,0.8889910199323897,False
|
||||
5gnidd,0.5,0.0,0.6872301309573783,0.052725844344546816,0.9477370387940001,False
|
||||
5gnidd,0.5,0.25,0.6450777370820986,0.06474313739822393,0.9468616498164765,False
|
||||
5gnidd,0.5,0.5,0.6746687817772702,0.10225286523806497,0.9436587867425653,False
|
||||
5gnidd,0.5,0.75,0.6883680316141424,0.12603192659396695,0.9419154392409256,False
|
||||
5gnidd,0.5,1.0,0.6923695590578484,0.13914040832229815,0.9423005037244669,True
|
||||
5gnidd,0.75,0.0,0.7780936320958092,0.05096339060419691,0.9598578766446005,False
|
||||
5gnidd,0.75,0.25,0.7904830018820702,0.05510032325554253,0.9596139786642895,False
|
||||
5gnidd,0.75,0.5,0.8090814343293601,0.05873543815277214,0.9592678831536903,True
|
||||
5gnidd,0.75,0.75,0.7915299789128217,0.06497400095064636,0.9596244940396749,False
|
||||
5gnidd,0.75,1.0,0.8043064619946296,0.05480323650098284,0.9600406871348003,False
|
||||
|
@@ -0,0 +1,31 @@
|
||||
dataset,alpha,beta,mean_displacement,std_sample_across_classes,n_classes
|
||||
xiiotid,0.25,0.0,0.0,0.0,9
|
||||
xiiotid,0.25,0.25,1.4759443927974063,0.3915876677126786,9
|
||||
xiiotid,0.25,0.5,2.403216360439701,0.5578785827790653,9
|
||||
xiiotid,0.25,0.75,3.0026175335496204,0.6643181322495177,9
|
||||
xiiotid,0.25,1.0,3.42212533168931,0.7740752123481485,9
|
||||
xiiotid,0.5,0.0,0.0,0.0,9
|
||||
xiiotid,0.5,0.25,0.8255823617460618,0.31831330857243423,9
|
||||
xiiotid,0.5,0.5,1.3742233752967101,0.43187373764463177,9
|
||||
xiiotid,0.5,0.75,1.7801302991334842,0.5043074746507549,9
|
||||
xiiotid,0.5,1.0,2.0985368999756338,0.578855443340215,9
|
||||
xiiotid,0.75,0.0,0.0,0.0,9
|
||||
xiiotid,0.75,0.25,0.47722483146317857,0.12025761466601799,9
|
||||
xiiotid,0.75,0.5,0.8294921426085301,0.21286956479369043,9
|
||||
xiiotid,0.75,0.75,1.0965820930128614,0.2902562856464354,9
|
||||
xiiotid,0.75,1.0,1.305110487896406,0.3553065348087712,9
|
||||
5gnidd,0.25,0.0,0.0,0.0,9
|
||||
5gnidd,0.25,0.25,1.2954097330224899,0.4912234848812893,9
|
||||
5gnidd,0.25,0.5,2.055413931792982,0.6597523296778056,9
|
||||
5gnidd,0.25,0.75,2.558521399711773,0.7355506793948603,9
|
||||
5gnidd,0.25,1.0,2.91943348262683,0.7911632168313667,9
|
||||
5gnidd,0.5,0.0,0.0,0.0,9
|
||||
5gnidd,0.5,0.25,0.9534086318837013,0.3722328495814873,9
|
||||
5gnidd,0.5,0.5,1.5672291368417097,0.5712537132762766,9
|
||||
5gnidd,0.5,0.75,2.0032214267954767,0.6904879863519336,9
|
||||
5gnidd,0.5,1.0,2.3399083587182736,0.770453813804466,9
|
||||
5gnidd,0.75,0.0,0.0,0.0,9
|
||||
5gnidd,0.75,0.25,0.496912846928573,0.2311875190119548,9
|
||||
5gnidd,0.75,0.5,0.8749997159830957,0.3878012596482038,9
|
||||
5gnidd,0.75,0.75,1.1629465210044903,0.4928020443540843,9
|
||||
5gnidd,0.75,1.0,1.3846165631331335,0.5620843843793245,9
|
||||
|
@@ -0,0 +1,55 @@
|
||||
dataset,alpha,class_id,class_name,mean_displacement,std_sample_displacement,mean_relative_displacement
|
||||
xiiotid,0.25,0,C&C,4.01952572799183,0.29030488023104745,0.8472326753379348
|
||||
xiiotid,0.25,1,Exfiltration,4.316178984369498,0.8975835034935007,0.6638254826227857
|
||||
xiiotid,0.25,2,Exploitation,3.464004570499769,1.112779630373774,0.7020410983796918
|
||||
xiiotid,0.25,3,Lateral_movement,3.7263813611395125,1.3833916615643342,0.7895377723300525
|
||||
xiiotid,0.25,4,Normal,1.9966341269597232,0.5327206047251788,0.3701992118201333
|
||||
xiiotid,0.25,5,RDOS,2.5658707936642617,1.2193483512552363,0.3281078404629119
|
||||
xiiotid,0.25,6,Reconnaissance,3.3391889586431325,1.1073644700689869,0.6402396428113026
|
||||
xiiotid,0.25,7,Tampering,4.247813103636777,0.24113229030491104,0.9653304277937593
|
||||
xiiotid,0.25,8,Weaponization,3.1235303582992926,0.585141395295389,0.5634271266802224
|
||||
xiiotid,0.5,0,C&C,1.6935576092145832,0.5048307458168824,0.3187155176777392
|
||||
xiiotid,0.5,1,Exfiltration,2.0204769683934525,0.5020218016853628,0.2973123924755737
|
||||
xiiotid,0.5,2,Exploitation,2.343885289699028,0.4121833629248462,0.4231523311819784
|
||||
xiiotid,0.5,3,Lateral_movement,2.7240136758504594,1.0350790616091956,0.40257578575923714
|
||||
xiiotid,0.5,4,Normal,1.557331074720638,0.5480258187545208,0.2920883603702228
|
||||
xiiotid,0.5,5,RDOS,2.6312389613743776,1.1651317404804817,0.3523347102777825
|
||||
xiiotid,0.5,6,Reconnaissance,1.00151995809175,0.26099624924542353,0.16884924727680906
|
||||
xiiotid,0.5,7,Tampering,2.354018138564271,1.165407725687578,0.48267240310142356
|
||||
xiiotid,0.5,8,Weaponization,2.560790423872144,1.2282783431433644,0.4334295567239335
|
||||
xiiotid,0.75,0,C&C,1.2839465030740465,0.5431989970423492,0.2617816727650299
|
||||
xiiotid,0.75,1,Exfiltration,1.4475761921343262,0.08598433748869257,0.20679504552799255
|
||||
xiiotid,0.75,2,Exploitation,1.5392602070293495,0.25171201225922846,0.2839417733897609
|
||||
xiiotid,0.75,3,Lateral_movement,1.3693216274264168,0.5527158531960167,0.2618053240393338
|
||||
xiiotid,0.75,4,Normal,0.9031246281714095,0.298425213658255,0.1619501084487235
|
||||
xiiotid,0.75,5,RDOS,0.8141011244781585,0.20007357231332454,0.0957646213985643
|
||||
xiiotid,0.75,6,Reconnaissance,0.9825580428619212,0.4118545897797079,0.17068982082838355
|
||||
xiiotid,0.75,7,Tampering,1.934147031540858,0.4266139988379522,0.397088566305455
|
||||
xiiotid,0.75,8,Weaponization,1.4719590343511655,0.17030548300304388,0.23542365279592728
|
||||
5gnidd,0.25,0,Benign,2.648344737773736,0.5188602564457683,0.4821737572132332
|
||||
5gnidd,0.25,1,HTTPFlood,3.7602841668463944,1.5390234836296783,0.5584139964274445
|
||||
5gnidd,0.25,2,ICMPFlood,1.2149615886933787,0.1507085895098706,0.34685106134597526
|
||||
5gnidd,0.25,3,SYNFlood,3.1763695442454467,1.0069739670231537,0.9167291408106705
|
||||
5gnidd,0.25,4,SYNScan,3.814579780422797,1.4018369744978139,0.5442267347415422
|
||||
5gnidd,0.25,5,SlowrateDoS,3.178870743072719,1.1570108600313702,0.5619090743749164
|
||||
5gnidd,0.25,6,TCPConnectScan,3.257943080113345,2.5741100653263422,0.8873673593640223
|
||||
5gnidd,0.25,7,UDPFlood,2.4077369723424735,0.5783305176966848,0.47667894621080914
|
||||
5gnidd,0.25,8,UDPScan,2.815810730131176,1.6041886744624576,0.37142192996652196
|
||||
5gnidd,0.5,0,Benign,1.6898663850795066,0.6987889577561244,0.29591075596118094
|
||||
5gnidd,0.5,1,HTTPFlood,3.120458869697078,1.2894156117591726,0.43348480339644263
|
||||
5gnidd,0.5,2,ICMPFlood,1.1588454233322945,0.09271748323377921,0.32092174099674725
|
||||
5gnidd,0.5,3,SYNFlood,2.2617693416611573,0.7483497200280906,0.6584039968772434
|
||||
5gnidd,0.5,4,SYNScan,2.8752781124355127,0.9588449812962653,0.3958823033995225
|
||||
5gnidd,0.5,5,SlowrateDoS,2.763430109051773,1.4542368289687693,0.4536612328161466
|
||||
5gnidd,0.5,6,TCPConnectScan,2.130611279721881,0.21282292441201225,0.5753306009264239
|
||||
5gnidd,0.5,7,UDPFlood,1.5982465815194804,0.42810184263788215,0.30005165770255965
|
||||
5gnidd,0.5,8,UDPScan,3.460669125965779,2.0113615605321216,0.4243985120323746
|
||||
5gnidd,0.75,0,Benign,0.8242192109522325,0.2718678603634198,0.13820856413319157
|
||||
5gnidd,0.75,1,HTTPFlood,1.3664020721980037,0.5435449017657998,0.17507931126042764
|
||||
5gnidd,0.75,2,ICMPFlood,0.6733630855518974,0.33119344952410124,0.1786782977125397
|
||||
5gnidd,0.75,3,SYNFlood,1.8966313690365364,0.1707004572307703,0.5423454423708408
|
||||
5gnidd,0.75,4,SYNScan,1.5627721380862394,0.6395395815733285,0.19898769631915747
|
||||
5gnidd,0.75,5,SlowrateDoS,1.5649187261569397,0.654138365253272,0.2522141838283814
|
||||
5gnidd,0.75,6,TCPConnectScan,1.2260809195578348,0.4757573735353394,0.32576611133643296
|
||||
5gnidd,0.75,7,UDPFlood,0.9026363032967967,0.10316225038256528,0.16267861998433658
|
||||
5gnidd,0.75,8,UDPScan,2.444525243361723,0.48441013395931937,0.2850436623074378
|
||||
|
@@ -0,0 +1,7 @@
|
||||
dataset,alpha,cells_sj_lt_M_total,cells_total,missing_client_class_cells_total,client_size_ratio_mean,client_size_ratio_max,support_cv_mean,support_cv_max_where_available,support_max_min_ratio_max,union_rare_mean,union_unseen_mean,max_unseen_on_client,eligible_clients_per_seed,union_rare_per_seed,union_unseen_per_seed
|
||||
xiiotid,0.25,19,27,38,14.745270966890343,20.837033843500755,1.4423832412721973,2.1341217158156836,122800.0,7.666666666666667,5.666666666666667,4,10|10|10,8|7|8,6|6|5
|
||||
xiiotid,0.5,7,27,9,7.405360903230798,9.890694678426321,1.1833344484214374,2.0951833233286665,12215.0,6.666666666666667,2.3333333333333335,2,10|10|10,6|7|7,3|3|1
|
||||
xiiotid,0.75,3,27,4,4.668973713188908,5.07283910650696,0.8806570470947873,1.3221152695060867,1903.0,5.0,1.0,1,10|10|10,4|5|6,2|1|0
|
||||
5gnidd,0.25,16,27,30,17.600940070872678,27.82808487633464,1.3021194271854393,2.138942553879626,185504.0,6.666666666666667,5.333333333333333,3,10|10|10,6|6|8,6|5|5
|
||||
5gnidd,0.5,4,27,4,9.594308791178781,12.717827450589933,1.2251708481787265,1.9640881124634801,88943.0,5.666666666666667,1.3333333333333333,1,10|10|10,6|5|6,2|0|2
|
||||
5gnidd,0.75,4,27,5,6.308651536045827,11.355971110240189,0.9162099053779219,1.2651419073123291,7642.5,6.0,1.3333333333333333,1,10|10|10,6|6|6,2|2|0
|
||||
|
@@ -0,0 +1,7 @@
|
||||
dataset,alpha,selected_beta,selected_minus_original_rare_macro_f1,selected_minus_original_rare_recall,selected_minus_original_macro_f1,selected_minus_original_accuracy,selected_minus_original_weighted_f1,descriptive_interpretation
|
||||
xiiotid,0.25,0.0,-0.000654707802382859,-0.0007083831515385386,-0.0006741622353423349,-0.0014166016708530016,-0.0007698862919814919,numerical/descriptive difference only; not a test of statistical superiority
|
||||
xiiotid,0.5,0.75,0.022982586416122386,-0.014374000954948563,0.02220443400015426,0.006937663385145676,0.00889883310041717,numerical/descriptive difference only; not a test of statistical superiority
|
||||
xiiotid,0.75,0.5,0.0013560185452363704,-0.007739443483596897,0.005786987337823679,0.007533373047310832,0.00608024689536979,numerical/descriptive difference only; not a test of statistical superiority
|
||||
5gnidd,0.25,0.0,-0.0010388235847773597,0.0006605396210892511,-0.0027650156824441385,-0.0007429430689033234,-0.0009720113916064088,numerical/descriptive difference only; not a test of statistical superiority
|
||||
5gnidd,0.5,1.0,-0.008557880645695959,-0.005809066141944985,-0.003799042805718411,-0.0007182696828510471,-0.0005346795091106804,numerical/descriptive difference only; not a test of statistical superiority
|
||||
5gnidd,0.75,0.5,0.01683683306978989,-0.0015126750275882284,0.011744866973314672,0.0013186554101197734,0.0014199972720251308,numerical/descriptive difference only; not a test of statistical superiority
|
||||
|
@@ -0,0 +1,7 @@
|
||||
dataset,alpha,selected_beta,best_pooled_score,second_best_pooled_score,second_best_beta,absolute_gap,relative_gap,selected_beta_sample_std,min_candidate_sample_std,max_candidate_sample_std,top2_gap_over_selected_std,per_seed_diagnostic_betas,n_seeds_agree_with_pooled,separation_classification
|
||||
xiiotid,0.25,0.0,0.5917038146634539,0.5886906256417319,0.25,0.0030131890217219848,0.005092394111800361,0.08952883653922933,0.044857849968223555,0.08952883653922933,0.033656072592897816,0.75|0.5|0,1,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std)
|
||||
xiiotid,0.5,0.75,0.5793667895728949,0.5734790834215032,1.0,0.0058877061513916695,0.010162312126540849,0.07512057331236716,0.07512057331236716,0.1004496749103663,0.07837674676562102,0.5|1|0.75,1,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std)
|
||||
xiiotid,0.75,0.5,0.47674948263663786,0.47436809609801456,0.25,0.002381386538623298,0.004995047976671449,0.04662425884469754,0.044741855341153436,0.055329314644425844,0.05107612641212262,0.5|0.5|0,2,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std)
|
||||
5gnidd,0.25,0.0,0.7614534210908004,0.7492858468825278,0.25,0.012167574208272636,0.01597940710653357,0.021902537932118035,0.002755842556750987,0.021902537932118035,0.5555326166302408,0.25|0|0,2,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std)
|
||||
5gnidd,0.5,1.0,0.6923695590578484,0.6883680316141424,0.75,0.004001527443705966,0.005779467614305719,0.13914040832229815,0.052725844344546816,0.13914040832229815,0.02875891692395368,1|0|1,2,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std)
|
||||
5gnidd,0.75,0.5,0.8090814343293601,0.8043064619946296,1.0,0.004774972334730454,0.005901720311612864,0.05873543815277214,0.05096339060419691,0.06497400095064636,0.081296275041154,0.5|1|1,1,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std)
|
||||
|
@@ -0,0 +1,7 @@
|
||||
dataset,alpha,selected_beta,pooled_validation_mean,pooled_validation_std_sample,selected_rare_macro_f1_mean,selected_rare_macro_f1_std,selected_rare_recall_mean,selected_rare_recall_std,selected_macro_f1_mean,selected_macro_f1_std,selected_accuracy_mean,selected_accuracy_std,selected_weighted_f1_mean,selected_weighted_f1_std,original_rare_macro_f1_mean,original_rare_macro_f1_std,original_rare_recall_mean,original_rare_recall_std,original_macro_f1_mean,original_macro_f1_std,original_accuracy_mean,original_accuracy_std,original_weighted_f1_mean,original_weighted_f1_std,cells_sj_lt_M,missing_client_class_cells,client_size_ratio_mean,support_cv_mean,mean_displacement_beta1,beta0_equivalence_class,test_used_for_selection,audit_verdict
|
||||
xiiotid,0.25,0.0,0.5917038146634539,0.08952883653922933,0.6321355974773142,0.047641778563026725,0.8212568016859243,0.012283281629867224,0.6813433722144538,0.020555132742947534,0.8740145713449322,0.010266545051804108,0.9068823111564912,0.007408425095578143,0.632790305279697,0.04715228886220545,0.8219651848374628,0.010810973971484781,0.6820175344497962,0.02096686007526524,0.8754311730157852,0.009066523301774463,0.9076521974484727,0.0069343939753111275,19,38,14.745270966890343,1.4423832412721973,3.42212533168931,B,False,PASS WITH MINOR CORRECTIONS (Session 7)
|
||||
xiiotid,0.5,0.75,0.5793667895728949,0.07512057331236716,0.6655025083742895,0.026684712591240056,0.86258272574027,0.013920609948849614,0.7311691156241865,0.006233755140268,0.9063916985504398,0.018450460685614026,0.9336316455838293,0.011230108565482132,0.6425199219581671,0.028683282930463034,0.8769567266952185,0.010770970951613777,0.7089646816240323,0.0011722251041387445,0.8994540351652941,0.008398091032446977,0.9247328124834121,0.006244765764216192,7,9,7.405360903230798,1.1833344484214374,2.0985368999756338,B,False,PASS WITH MINOR CORRECTIONS (Session 6)
|
||||
xiiotid,0.75,0.5,0.47674948263663786,0.04662425884469754,0.6052456782307091,0.08548886486265879,0.8487646652689419,0.039255535276369505,0.7576359407646712,0.014060416901925105,0.9281095123164508,0.012136684529934048,0.9452163089135119,0.008528481213975155,0.6038896596854727,0.07776445613264914,0.8565041087525388,0.03533794219558259,0.7518489534268475,0.012228464472937675,0.92057613926914,0.012128815595858657,0.9391360620181421,0.008628878095570825,3,4,4.668973713188908,0.8806570470947873,1.305110487896406,B,False,PASS (Session 8)
|
||||
5gnidd,0.25,0.0,0.7614534210908004,0.021902537932118035,0.8412355814155769,0.024201888695409735,0.9612162422875251,0.03134699156678076,0.8166404182626198,0.01335696520619808,0.730225309136966,0.019524710435296355,0.7044634053674654,0.018175165847202895,0.8422744050003542,0.025370068519978167,0.9605557026664359,0.032423277290702865,0.8194054339450639,0.013918675694370832,0.7309682522058694,0.02143091560670625,0.7054354167590718,0.02036694411653514,16,30,17.600940070872678,1.3021194271854393,2.91943348262683,B,False,PASS (Session 11)
|
||||
5gnidd,0.5,1.0,0.6923695590578484,0.13914040832229815,0.8719837976555308,0.028500884123071776,0.9671039796439406,0.007942238914573898,0.8385961780131425,0.019627422368850796,0.743256969203502,0.0006668791219600918,0.7181952009373496,0.0016799838366213199,0.8805416783012268,0.0370578200850738,0.9729130457858856,0.0038428107281798614,0.8423952208188609,0.02083962881650294,0.7439752388863531,0.008392660852506539,0.7187298804464602,0.00867754781098361,4,4,9.594308791178781,1.2251708481787265,2.3399083587182736,B,False,PASS (Session 9)
|
||||
5gnidd,0.75,0.5,0.8090814343293601,0.05873543815277214,0.8967376245747419,0.014470318925337908,0.9597014107775274,0.0010245851497612311,0.8531495688589309,0.009572296127013734,0.7522737210618877,0.0025025090963709863,0.7274674299268966,0.0032168140791904,0.879900791504952,0.023120839413686988,0.9612140858051156,0.002143727670860224,0.8414047018856162,0.014091078328488028,0.750955065651768,0.0035777922288362054,0.7260474326548715,0.004543897028843018,4,5,6.308651536045827,0.9162099053779219,1.3846165631331335,B,False,PASS (Session 10)
|
||||
|
@@ -0,0 +1,217 @@
|
||||
{
|
||||
"description": "Authoritative six-condition master table (validation-selected beta; test metrics are post-selection descriptive summaries).",
|
||||
"source_artifacts": "artifacts/analysis/*__multiseed__*, **partition_heterogeneity**",
|
||||
"n_conditions": 6,
|
||||
"rows": [
|
||||
{
|
||||
"dataset": "xiiotid",
|
||||
"alpha": 0.25,
|
||||
"selected_beta": 0.0,
|
||||
"pooled_validation_mean": 0.5917038146634539,
|
||||
"pooled_validation_std_sample": 0.08952883653922933,
|
||||
"selected_rare_macro_f1_mean": 0.6321355974773142,
|
||||
"selected_rare_macro_f1_std": 0.047641778563026725,
|
||||
"selected_rare_recall_mean": 0.8212568016859243,
|
||||
"selected_rare_recall_std": 0.012283281629867224,
|
||||
"selected_macro_f1_mean": 0.6813433722144538,
|
||||
"selected_macro_f1_std": 0.020555132742947534,
|
||||
"selected_accuracy_mean": 0.8740145713449322,
|
||||
"selected_accuracy_std": 0.010266545051804108,
|
||||
"selected_weighted_f1_mean": 0.9068823111564912,
|
||||
"selected_weighted_f1_std": 0.007408425095578143,
|
||||
"original_rare_macro_f1_mean": 0.632790305279697,
|
||||
"original_rare_macro_f1_std": 0.04715228886220545,
|
||||
"original_rare_recall_mean": 0.8219651848374628,
|
||||
"original_rare_recall_std": 0.010810973971484781,
|
||||
"original_macro_f1_mean": 0.6820175344497962,
|
||||
"original_macro_f1_std": 0.02096686007526524,
|
||||
"original_accuracy_mean": 0.8754311730157852,
|
||||
"original_accuracy_std": 0.009066523301774463,
|
||||
"original_weighted_f1_mean": 0.9076521974484727,
|
||||
"original_weighted_f1_std": 0.0069343939753111275,
|
||||
"cells_sj_lt_M": 19,
|
||||
"missing_client_class_cells": 38,
|
||||
"client_size_ratio_mean": 14.745270966890343,
|
||||
"support_cv_mean": 1.4423832412721973,
|
||||
"mean_displacement_beta1": 3.42212533168931,
|
||||
"beta0_equivalence_class": "B",
|
||||
"test_used_for_selection": false,
|
||||
"audit_verdict": "PASS WITH MINOR CORRECTIONS (Session 7)"
|
||||
},
|
||||
{
|
||||
"dataset": "xiiotid",
|
||||
"alpha": 0.5,
|
||||
"selected_beta": 0.75,
|
||||
"pooled_validation_mean": 0.5793667895728949,
|
||||
"pooled_validation_std_sample": 0.07512057331236716,
|
||||
"selected_rare_macro_f1_mean": 0.6655025083742895,
|
||||
"selected_rare_macro_f1_std": 0.026684712591240056,
|
||||
"selected_rare_recall_mean": 0.86258272574027,
|
||||
"selected_rare_recall_std": 0.013920609948849614,
|
||||
"selected_macro_f1_mean": 0.7311691156241865,
|
||||
"selected_macro_f1_std": 0.006233755140268,
|
||||
"selected_accuracy_mean": 0.9063916985504398,
|
||||
"selected_accuracy_std": 0.018450460685614026,
|
||||
"selected_weighted_f1_mean": 0.9336316455838293,
|
||||
"selected_weighted_f1_std": 0.011230108565482132,
|
||||
"original_rare_macro_f1_mean": 0.6425199219581671,
|
||||
"original_rare_macro_f1_std": 0.028683282930463034,
|
||||
"original_rare_recall_mean": 0.8769567266952185,
|
||||
"original_rare_recall_std": 0.010770970951613777,
|
||||
"original_macro_f1_mean": 0.7089646816240323,
|
||||
"original_macro_f1_std": 0.0011722251041387445,
|
||||
"original_accuracy_mean": 0.8994540351652941,
|
||||
"original_accuracy_std": 0.008398091032446977,
|
||||
"original_weighted_f1_mean": 0.9247328124834121,
|
||||
"original_weighted_f1_std": 0.006244765764216192,
|
||||
"cells_sj_lt_M": 7,
|
||||
"missing_client_class_cells": 9,
|
||||
"client_size_ratio_mean": 7.405360903230798,
|
||||
"support_cv_mean": 1.1833344484214374,
|
||||
"mean_displacement_beta1": 2.0985368999756338,
|
||||
"beta0_equivalence_class": "B",
|
||||
"test_used_for_selection": false,
|
||||
"audit_verdict": "PASS WITH MINOR CORRECTIONS (Session 6)"
|
||||
},
|
||||
{
|
||||
"dataset": "xiiotid",
|
||||
"alpha": 0.75,
|
||||
"selected_beta": 0.5,
|
||||
"pooled_validation_mean": 0.47674948263663786,
|
||||
"pooled_validation_std_sample": 0.04662425884469754,
|
||||
"selected_rare_macro_f1_mean": 0.6052456782307091,
|
||||
"selected_rare_macro_f1_std": 0.08548886486265879,
|
||||
"selected_rare_recall_mean": 0.8487646652689419,
|
||||
"selected_rare_recall_std": 0.039255535276369505,
|
||||
"selected_macro_f1_mean": 0.7576359407646712,
|
||||
"selected_macro_f1_std": 0.014060416901925105,
|
||||
"selected_accuracy_mean": 0.9281095123164508,
|
||||
"selected_accuracy_std": 0.012136684529934048,
|
||||
"selected_weighted_f1_mean": 0.9452163089135119,
|
||||
"selected_weighted_f1_std": 0.008528481213975155,
|
||||
"original_rare_macro_f1_mean": 0.6038896596854727,
|
||||
"original_rare_macro_f1_std": 0.07776445613264914,
|
||||
"original_rare_recall_mean": 0.8565041087525388,
|
||||
"original_rare_recall_std": 0.03533794219558259,
|
||||
"original_macro_f1_mean": 0.7518489534268475,
|
||||
"original_macro_f1_std": 0.012228464472937675,
|
||||
"original_accuracy_mean": 0.92057613926914,
|
||||
"original_accuracy_std": 0.012128815595858657,
|
||||
"original_weighted_f1_mean": 0.9391360620181421,
|
||||
"original_weighted_f1_std": 0.008628878095570825,
|
||||
"cells_sj_lt_M": 3,
|
||||
"missing_client_class_cells": 4,
|
||||
"client_size_ratio_mean": 4.668973713188908,
|
||||
"support_cv_mean": 0.8806570470947873,
|
||||
"mean_displacement_beta1": 1.305110487896406,
|
||||
"beta0_equivalence_class": "B",
|
||||
"test_used_for_selection": false,
|
||||
"audit_verdict": "PASS (Session 8)"
|
||||
},
|
||||
{
|
||||
"dataset": "5gnidd",
|
||||
"alpha": 0.25,
|
||||
"selected_beta": 0.0,
|
||||
"pooled_validation_mean": 0.7614534210908004,
|
||||
"pooled_validation_std_sample": 0.021902537932118035,
|
||||
"selected_rare_macro_f1_mean": 0.8412355814155769,
|
||||
"selected_rare_macro_f1_std": 0.024201888695409735,
|
||||
"selected_rare_recall_mean": 0.9612162422875251,
|
||||
"selected_rare_recall_std": 0.03134699156678076,
|
||||
"selected_macro_f1_mean": 0.8166404182626198,
|
||||
"selected_macro_f1_std": 0.01335696520619808,
|
||||
"selected_accuracy_mean": 0.730225309136966,
|
||||
"selected_accuracy_std": 0.019524710435296355,
|
||||
"selected_weighted_f1_mean": 0.7044634053674654,
|
||||
"selected_weighted_f1_std": 0.018175165847202895,
|
||||
"original_rare_macro_f1_mean": 0.8422744050003542,
|
||||
"original_rare_macro_f1_std": 0.025370068519978167,
|
||||
"original_rare_recall_mean": 0.9605557026664359,
|
||||
"original_rare_recall_std": 0.032423277290702865,
|
||||
"original_macro_f1_mean": 0.8194054339450639,
|
||||
"original_macro_f1_std": 0.013918675694370832,
|
||||
"original_accuracy_mean": 0.7309682522058694,
|
||||
"original_accuracy_std": 0.02143091560670625,
|
||||
"original_weighted_f1_mean": 0.7054354167590718,
|
||||
"original_weighted_f1_std": 0.02036694411653514,
|
||||
"cells_sj_lt_M": 16,
|
||||
"missing_client_class_cells": 30,
|
||||
"client_size_ratio_mean": 17.600940070872678,
|
||||
"support_cv_mean": 1.3021194271854393,
|
||||
"mean_displacement_beta1": 2.91943348262683,
|
||||
"beta0_equivalence_class": "B",
|
||||
"test_used_for_selection": false,
|
||||
"audit_verdict": "PASS (Session 11)"
|
||||
},
|
||||
{
|
||||
"dataset": "5gnidd",
|
||||
"alpha": 0.5,
|
||||
"selected_beta": 1.0,
|
||||
"pooled_validation_mean": 0.6923695590578484,
|
||||
"pooled_validation_std_sample": 0.13914040832229815,
|
||||
"selected_rare_macro_f1_mean": 0.8719837976555308,
|
||||
"selected_rare_macro_f1_std": 0.028500884123071776,
|
||||
"selected_rare_recall_mean": 0.9671039796439406,
|
||||
"selected_rare_recall_std": 0.007942238914573898,
|
||||
"selected_macro_f1_mean": 0.8385961780131425,
|
||||
"selected_macro_f1_std": 0.019627422368850796,
|
||||
"selected_accuracy_mean": 0.743256969203502,
|
||||
"selected_accuracy_std": 0.0006668791219600918,
|
||||
"selected_weighted_f1_mean": 0.7181952009373496,
|
||||
"selected_weighted_f1_std": 0.0016799838366213199,
|
||||
"original_rare_macro_f1_mean": 0.8805416783012268,
|
||||
"original_rare_macro_f1_std": 0.0370578200850738,
|
||||
"original_rare_recall_mean": 0.9729130457858856,
|
||||
"original_rare_recall_std": 0.0038428107281798614,
|
||||
"original_macro_f1_mean": 0.8423952208188609,
|
||||
"original_macro_f1_std": 0.02083962881650294,
|
||||
"original_accuracy_mean": 0.7439752388863531,
|
||||
"original_accuracy_std": 0.008392660852506539,
|
||||
"original_weighted_f1_mean": 0.7187298804464602,
|
||||
"original_weighted_f1_std": 0.00867754781098361,
|
||||
"cells_sj_lt_M": 4,
|
||||
"missing_client_class_cells": 4,
|
||||
"client_size_ratio_mean": 9.594308791178781,
|
||||
"support_cv_mean": 1.2251708481787265,
|
||||
"mean_displacement_beta1": 2.3399083587182736,
|
||||
"beta0_equivalence_class": "B",
|
||||
"test_used_for_selection": false,
|
||||
"audit_verdict": "PASS (Session 9)"
|
||||
},
|
||||
{
|
||||
"dataset": "5gnidd",
|
||||
"alpha": 0.75,
|
||||
"selected_beta": 0.5,
|
||||
"pooled_validation_mean": 0.8090814343293601,
|
||||
"pooled_validation_std_sample": 0.05873543815277214,
|
||||
"selected_rare_macro_f1_mean": 0.8967376245747419,
|
||||
"selected_rare_macro_f1_std": 0.014470318925337908,
|
||||
"selected_rare_recall_mean": 0.9597014107775274,
|
||||
"selected_rare_recall_std": 0.0010245851497612311,
|
||||
"selected_macro_f1_mean": 0.8531495688589309,
|
||||
"selected_macro_f1_std": 0.009572296127013734,
|
||||
"selected_accuracy_mean": 0.7522737210618877,
|
||||
"selected_accuracy_std": 0.0025025090963709863,
|
||||
"selected_weighted_f1_mean": 0.7274674299268966,
|
||||
"selected_weighted_f1_std": 0.0032168140791904,
|
||||
"original_rare_macro_f1_mean": 0.879900791504952,
|
||||
"original_rare_macro_f1_std": 0.023120839413686988,
|
||||
"original_rare_recall_mean": 0.9612140858051156,
|
||||
"original_rare_recall_std": 0.002143727670860224,
|
||||
"original_macro_f1_mean": 0.8414047018856162,
|
||||
"original_macro_f1_std": 0.014091078328488028,
|
||||
"original_accuracy_mean": 0.750955065651768,
|
||||
"original_accuracy_std": 0.0035777922288362054,
|
||||
"original_weighted_f1_mean": 0.7260474326548715,
|
||||
"original_weighted_f1_std": 0.004543897028843018,
|
||||
"cells_sj_lt_M": 4,
|
||||
"missing_client_class_cells": 5,
|
||||
"client_size_ratio_mean": 6.308651536045827,
|
||||
"support_cv_mean": 0.9162099053779219,
|
||||
"mean_displacement_beta1": 1.3846165631331335,
|
||||
"beta0_equivalence_class": "B",
|
||||
"test_used_for_selection": false,
|
||||
"audit_verdict": "PASS (Session 10)"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 45 KiB |
|
After Width: | Height: | Size: 48 KiB |
|
After Width: | Height: | Size: 48 KiB |
|
After Width: | Height: | Size: 68 KiB |
|
After Width: | Height: | Size: 48 KiB |
|
After Width: | Height: | Size: 55 KiB |
|
After Width: | Height: | Size: 45 KiB |
@@ -3,30 +3,36 @@
|
||||
This is the canonical continuation point for the next agent session. It is written to be
|
||||
readable without prior chat history, terminal scrollback, or git history.
|
||||
|
||||
Last updated: 2026-09-30 (Session 11). Session 11 type: **final 5G-NIDD beta-validation condition**
|
||||
(`alpha=0.25`), completing the planned experiment grid. Sessions 6–8 completed the X-IIoTID
|
||||
three-alpha study (`alpha=0.25/0.50/0.75`, seeds `{0,1,2}`); Session 9 ran 5G-NIDD `alpha=0.50`;
|
||||
Session 10 ran 5G-NIDD `alpha=0.75`; Session 11 ran **only** 5G-NIDD `alpha=0.25` for seeds
|
||||
`{0,1,2}` across the five frozen beta candidates plus `original_protean`, using the same
|
||||
pooled-across-seed validation-only selection rule. **No condition was run after alpha=0.25.** No
|
||||
commit or push was made.
|
||||
Last updated: 2026-09-30 (Session 12). Session 12 type: **thesis synthesis of the complete audited
|
||||
grid** — no training, no partition regeneration, no source/model/aggregation change. Session 11 ran
|
||||
the final 5G-NIDD condition (`alpha=0.25`), completing the planned grid (108 full runs + 1 smoke).
|
||||
|
||||
> **Audit status (Session 11).** Independent audit verdict: **PASS**
|
||||
> (`docs/research/SESSION11_AUDIT.md`). `beta_star(5gnidd, alpha=0.25) = 0.00` confirmed from
|
||||
> validation only; run inventory 18/18; pairing hash-identical and reproduced from raw data;
|
||||
> rare/unseen 0 mismatches over 1,620 cells; leakage audit PASS; `beta=0` class **B**; source/data
|
||||
> integrity PASS; pytest 75 passed. **Session 11 is safe to checkpoint.**
|
||||
> **Session 12 scope and status.** Consolidated the 108-run grid into authoritative six-condition
|
||||
> tables, publication-ready figures, a final RQ1/RQ2/RQ3 synthesis, a Chapter-4 blueprint and a
|
||||
> claims register, then independently re-verified every number against the audited artifacts.
|
||||
> **No material scientific gap remains for the defined thesis scope (classification A).** No
|
||||
> training was executed, no tracked file changed, `planningawal/` and both raw datasets are
|
||||
> unchanged, `results/` is unchanged (108 non-smoke run dirs), and pytest is **75 passed**.
|
||||
> Deliverables: `docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md`,
|
||||
> `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md`, `docs/thesis/CLAIMS_REGISTER.md`,
|
||||
> `artifacts/analysis/final__*.{csv,json}`, `artifacts/figures/final/figA–G*.png`.
|
||||
|
||||
> **Headline (Session 11).** PRIMARY pooled-validation selected beta for 5G-NIDD `alpha=0.25` =
|
||||
> **0.00** (validation only; per-seed diagnostics 0.25 / 0.00 / 0.00). The pooled validation curve
|
||||
> declines from 0.761 (beta=0) to 0.722 (beta=0.75) and rises slightly to 0.738 (beta=1.0), with
|
||||
> the between-beta spread (0.039) exceeding the across-seed std (0.003–0.022). This is the first
|
||||
> condition where the selected beta is 0, i.e. the equal-`S_j` mean; the selected mode is therefore
|
||||
> numerically the baseline and shows no support-weighting benefit. `beta=0` is class **B** for all
|
||||
> three seeds. Realized heterogeneity is the highest of the three 5G-NIDD conditions
|
||||
> (16/27 `|S_j|<M` cells, 30 missing cells, client ratio mean 17.60, mean CV 1.302), and the
|
||||
> rare-class union differs across seeds (6/6/8), so this condition's test rare support is not
|
||||
> constant.
|
||||
> **Final RQ findings (Session 12).** (1) Changing beta changes contribution weights and prototype
|
||||
> positions; for a fixed snapshot displacement is monotone increasing in beta in all six
|
||||
> conditions, and realized support imbalance and displacement co-vary within each dataset. (2) The
|
||||
> pooled validation rare-class response is **not monotone** in beta and is weak relative to seed
|
||||
> variance; the direct aggregation effect and the end-to-end training effect remain distinct.
|
||||
> (3) Selected betas are condition-dependent and non-monotone in alpha:
|
||||
> X-IIoTID `{0.00, 0.75, 0.50}` and 5G-NIDD `{0.00, 1.00, 0.50}` for alpha `{0.25, 0.50, 0.75}`;
|
||||
> both datasets select beta=0 at alpha=0.25 (no weighting benefit) and beta=0.50 at alpha=0.75;
|
||||
> no universal/global optimum. All six top-two selection gaps are smaller than the selected beta's
|
||||
> across-seed std. No inferential test was added (3 seeds).
|
||||
|
||||
> **Session 11 audit (preserved).** Verdict **PASS** (`docs/research/SESSION11_AUDIT.md`);
|
||||
> `beta_star(5gnidd, alpha=0.25) = 0.00` validation-only (per-seed 0.25/0.00/0.00); run inventory
|
||||
> 18/18; pairing hash-identical; rare/unseen 0 mismatches; leakage PASS; `beta=0` class **B**;
|
||||
> source/data integrity PASS; pytest 75 passed. Realized heterogeneity highest of the three 5G-NIDD
|
||||
> conditions (16/27 cells, 30 missing, client ratio 17.60, mean CV 1.302).
|
||||
|
||||
---
|
||||
|
||||
@@ -304,11 +310,13 @@ Six-condition, validation-only pooled selections:
|
||||
|
||||
## 15. Git status
|
||||
|
||||
No commit or push was made. `HEAD = bbd35085063194db58e4472e70b204d4be4a3607`, branch `main`, in
|
||||
sync with `origin/main`. Only untracked docs/analysis artifacts exist (no tracked modification);
|
||||
`results/` is gitignored. `planningawal/` and both raw datasets are unchanged. `equal_available`
|
||||
absent. No test-based selection. The 18 runs are the only `results/5gnidd__alpha0.25__*`
|
||||
directories.
|
||||
No commit or push was made in Session 12. `HEAD = d30e9831e6c17fc6b28383ae4592070ccd83657e`,
|
||||
branch `main`, in sync with `origin/main`. `git diff` / `git diff --cached` are empty (no tracked
|
||||
modification); only untracked docs/analysis/figures artifacts exist. `planningawal/` and both raw
|
||||
datasets are unchanged (SHA-256 match). `src/`, `tests/`, `configs/`, `scripts/`, `pyproject.toml`
|
||||
are unchanged. `results/` is gitignored and holds exactly 108 non-smoke run directories plus 1
|
||||
`smoke`. `equal_available` absent. No test-based selection. `matplotlib 3.11.2` was installed into
|
||||
the git-ignored `.venv` only; no dependency file was modified.
|
||||
|
||||
## 16. Session 11 audit result
|
||||
|
||||
@@ -321,45 +329,51 @@ directories.
|
||||
- Direct-effect, client-level, cross-alpha, cross-dataset labels: PASS.
|
||||
- Source/data integrity PASS. pytest: **75 passed, 0 failed, 0 skipped**.
|
||||
|
||||
## 17. Files created / modified this session
|
||||
## 17. Files created / modified (Session 12)
|
||||
|
||||
Created:
|
||||
|
||||
- `docs/research/5GNIDD_ALPHA025_BETA_VALIDATION.md`
|
||||
- `docs/research/5GNIDD_CROSS_ALPHA_ANALYSIS.md`
|
||||
- `docs/research/CROSS_DATASET_BETA_ANALYSIS.md`
|
||||
- `docs/research/SESSION11_AUDIT.md`
|
||||
- `artifacts/analysis/5gnidd__alpha0.25__*` — 18 prototype snapshots + 3 per-seed
|
||||
effect/weights CSVs + `5gnidd__alpha0.25__multiseed__*` (validation per-seed/pooled,
|
||||
client detail/summary/deltas, test descriptive, prototype effect pooled, beta0 equivalence,
|
||||
runtime, selection summary, audit recomputation) + `5gnidd__alpha0.25__partition_characterization.csv`
|
||||
- `artifacts/analysis/5gnidd__cross_alpha__*` (regenerated to include alpha=0.25)
|
||||
- `artifacts/analysis/cross_dataset__matched_alpha_summary.{csv,json}`,
|
||||
`artifacts/analysis/cross_dataset__beta_curve.csv`
|
||||
- `results/5gnidd__alpha0.25__seed{0,1,2}__*` (18 run dirs, gitignored)
|
||||
- `docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md` — final six-condition synthesis, RQ1/RQ2/RQ3
|
||||
answers, threats to validity, contribution, unsupported claims, gap assessment.
|
||||
- `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md` — precise Chapter-4 blueprint (purpose / artifact /
|
||||
key numbers / allowed / avoided per subsection).
|
||||
- `docs/thesis/CLAIMS_REGISTER.md` — claim / evidence / status / limitation / thesis-safe wording.
|
||||
- `artifacts/analysis/final__six_condition_master.{csv,json}`
|
||||
- `artifacts/analysis/final__pooled_validation_curves.csv` (30 rows)
|
||||
- `artifacts/analysis/final__realized_heterogeneity.csv`
|
||||
- `artifacts/analysis/final__prototype_displacement_by_beta.csv`
|
||||
- `artifacts/analysis/final__prototype_displacement_by_class.csv`
|
||||
- `artifacts/analysis/final__client_effect_summary.csv` (90 rows)
|
||||
- `artifacts/analysis/final__selected_vs_original.csv`
|
||||
- `artifacts/analysis/final__selection_strength.csv`
|
||||
- `artifacts/figures/final/figA–G*.png` — 8 thesis-friendly figures (matplotlib only).
|
||||
|
||||
Modified:
|
||||
|
||||
- `docs/handoff/SESSION_HANDOFF.md` (this file)
|
||||
- `artifacts/analysis/5gnidd__cross_alpha__*` (extended from 2 to 3 alphas; previous 2-alpha
|
||||
content was superseded, not silently lost — it is reproducible from the frozen manifests)
|
||||
- `docs/handoff/SESSION_HANDOFF.md` (this file).
|
||||
|
||||
No source or test code was changed.
|
||||
No source, test, config, model or aggregation file was changed. The synthesis generator is kept
|
||||
outside the repository at `/tmp/opencode/s12/synthesize_final_analysis.py` (Session-11 convention
|
||||
for ad-hoc analysis helpers). `matplotlib 3.11.2` was installed into the git-ignored `.venv`.
|
||||
|
||||
## 18. Exact recommended next phase
|
||||
|
||||
1. **Session 11 is audited and safe to checkpoint** (verdict PASS). Before checkpointing, stage
|
||||
only the intended docs/analysis artifacts; do not commit `results/` or the raw datasets.
|
||||
2. **The planned experiment grid is COMPLETE** (two datasets × three alphas × three seeds × five
|
||||
betas + baseline = 108 runs). Do **not** run any new condition.
|
||||
3. Remaining work is analysis/writing, not new training:
|
||||
- optionally add an inferential comparison (e.g. paired sign/permutation test over seeds) to
|
||||
formalise the weak selection effects — clearly labelled as post-hoc if added;
|
||||
- freeze the final thesis tables/report from the six-condition artifacts;
|
||||
- decide and document the `beta=0` vs `original_protean` reporting convention.
|
||||
4. Do not infer a monotone alpha→beta law; do not claim dataset causality or superiority.
|
||||
1. **The experiment grid is COMPLETE** (2 datasets × 3 alphas × 3 seeds × (5 betas + baseline) =
|
||||
108 runs). Do **not** run any new condition; no new training is required (gap classification A).
|
||||
2. **Synthesis is complete and verified.** All six selections, selected/original test summaries,
|
||||
heterogeneity values, displacement summaries and client counts were independently recomputed
|
||||
from raw artifacts with zero mismatches.
|
||||
3. Remaining work is **writing and formatting**, not experiments:
|
||||
- draft Chapter 4 from `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md` using the `final__*` tables
|
||||
and `figA–G` figures, respecting `docs/thesis/CLAIMS_REGISTER.md`;
|
||||
- optionally add appendices with the full per-seed and per-class tables;
|
||||
- if the examiners require inferential support, add a clearly post-hoc, explicitly
|
||||
underpowered analysis and label it as such (not done by default).
|
||||
4. Do not infer a monotone alpha→beta law; do not claim dataset causality, superiority,
|
||||
generalization, or a global optimum.
|
||||
5. Update `EXPERIMENT_PROTOCOL.md` only if a real ambiguity appears; update this handoff.
|
||||
6. Do not commit or push unless asked.
|
||||
6. Do not commit or push unless asked. When checkpointing, stage only intended docs/analysis/
|
||||
figure artifacts; never commit `results/` or raw datasets.
|
||||
|
||||
## 19. Important commands
|
||||
|
||||
@@ -394,16 +408,15 @@ No source or test code was changed.
|
||||
## 20. Recommended next-session prompt
|
||||
|
||||
```
|
||||
You are continuing the perdalam-protean research repository (Session 12).
|
||||
You are continuing the perdalam-protean research repository (Session 13).
|
||||
|
||||
FIRST, read completely:
|
||||
1. docs/handoff/SESSION_HANDOFF.md
|
||||
2. docs/research/SESSION11_AUDIT.md
|
||||
3. docs/research/5GNIDD_ALPHA025_BETA_VALIDATION.md
|
||||
4. docs/research/5GNIDD_CROSS_ALPHA_ANALYSIS.md
|
||||
2. docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md
|
||||
3. docs/thesis/CHAPTER4_RESULTS_OUTLINE.md
|
||||
4. docs/thesis/CLAIMS_REGISTER.md
|
||||
5. docs/research/CROSS_DATASET_BETA_ANALYSIS.md
|
||||
6. docs/research/XIIOTID_CROSS_ALPHA_ANALYSIS.md
|
||||
7. docs/research/EXPERIMENT_PROTOCOL.md
|
||||
6. docs/research/EXPERIMENT_PROTOCOL.md
|
||||
|
||||
Critical rules:
|
||||
- planningawal/ is immutable.
|
||||
@@ -412,14 +425,16 @@ Critical rules:
|
||||
- Do NOT add an executable equal_available mode.
|
||||
- Beta selection uses validation only; PRIMARY is pooled across clients and seeds.
|
||||
- The planned 2-dataset x 3-alpha experiment grid is COMPLETE. Do NOT run new conditions.
|
||||
- Scientific gap classification is A (NO MATERIAL GAP); no new experiment is required.
|
||||
|
||||
Context: Session 11 ran the final condition 5G-NIDD alpha=0.25 (18 runs, ~19.11 h), audited PASS,
|
||||
and selected beta_star=0.00 (validation only; per-seed 0.25/0.00/0.00). The six-condition pooled
|
||||
selections are 5G-NIDD 0.25/0.50/0.75 = 0.00/1.00/0.50 and X-IIoTID = 0.00/0.75/0.50; no universal
|
||||
or monotone beta. beta=0 equivalence class B. pytest 75 passed.
|
||||
Context: Session 12 completed the thesis synthesis with no training. Six pooled selections:
|
||||
X-IIoTID 0.25/0.50/0.75 = 0.00/0.75/0.50 and 5G-NIDD = 0.00/1.00/0.50; no universal/monotone beta;
|
||||
all top-two gaps below selected-beta across-seed std; beta=0 equivalence class B everywhere.
|
||||
Authoritative artifacts: artifacts/analysis/final__*.{csv,json}; figures artifacts/figures/final/.
|
||||
All synthesis numbers independently verified; pytest 75 passed; no tracked file changed.
|
||||
|
||||
Task: no new training. Consolidate the final thesis-ready six-condition tables/report from the
|
||||
existing audited artifacts, optionally add a clearly post-hoc inferential comparison of the weak
|
||||
pooled selections, and keep all limitations (S_j assumption, three seeds, no global optimum)
|
||||
explicit. Use .venv/bin/python. Do not commit or push unless asked.
|
||||
Task: write/format thesis chapters from the existing synthesis. Use the final__* tables and
|
||||
figA-G figures, obey CLAIMS_REGISTER.md wording, keep all limitations explicit, and do not run
|
||||
training or add inferential tests by default. Use .venv/bin/python. Do not commit or push unless
|
||||
asked.
|
||||
```
|
||||
@@ -0,0 +1,491 @@
|
||||
# Final Experimental Synthesis
|
||||
|
||||
Session 12 synthesis of the complete PROTEAN support-weighted prototype aggregation study.
|
||||
This document is **read-only with respect to experiments**: every number is loaded from the
|
||||
audited machine-readable artifacts under `artifacts/analysis/`. No training was run, no
|
||||
partition was regenerated, no source/model/aggregation code was modified, and no beta was
|
||||
re-selected from test metrics.
|
||||
|
||||
- Repository HEAD at synthesis: `d30e9831e6c17fc6b28383ae4592070ccd83657e` (branch `main`, in sync with `origin/main`).
|
||||
- Experimental grid: **2 datasets × 3 alpha levels × 3 seeds × (5 betas + original baseline) = 108 full runs** (plus one unrelated `smoke` directory, excluded).
|
||||
- Tests: `75 passed, 0 failed, 0 skipped`.
|
||||
- Beta candidates (frozen): `{0.00, 0.25, 0.50, 0.75, 1.00}`.
|
||||
- Selection rule (frozen): maximize **pooled validation per-client-mean rare-class macro-F1**, validation only; per-seed selection is diagnostic only; test is never used for selection.
|
||||
- `planningawal/` and both raw datasets unchanged (re-verified).
|
||||
|
||||
Terminology used throughout:
|
||||
|
||||
- **Measured**: directly read or recomputed from an audited artifact.
|
||||
- **Descriptive interpretation**: a neutral reading of measured values.
|
||||
- **Unsupported / not established**: a claim that the design cannot defend (causal, universal, superiority).
|
||||
|
||||
---
|
||||
|
||||
## 1. Scope and protocol
|
||||
|
||||
The thesis modifies **only** the server-side prototype aggregation of PROTEAN (Chennoufi et al.,
|
||||
2025). Each contributing client `i` provides a class-`j` local prototype `C_ij` (local class
|
||||
embedding mean) and a class-specific training support `N_ij`. The candidate aggregation is
|
||||
|
||||
```
|
||||
Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) / ( sum_{i in S_j} N_ij^beta )
|
||||
S_j = { i | N_ij > 0 and a valid local prototype C_ij exists }
|
||||
```
|
||||
|
||||
`S_j` is an explicit **reproduction assumption** (no verified official PROTEAN code is
|
||||
available); `N_ij` is class-specific **sample support, not prototype quality**. The frozen
|
||||
`original_protean` mode uses the equal-mean reference. `beta=0` is arithmetically the equal mean
|
||||
over the shared `S_j` but is **not bitwise identical** to the baseline (floating-point
|
||||
divergence; equivalence class **B** in every condition).
|
||||
|
||||
Protocol (frozen, `docs/research/EXPERIMENT_PROTOCOL.md`): datasets X-IIoTID (`class2`,
|
||||
benign `Normal`, K=9) and 5G-NIDD (`Attack Type`, benign `Benign`, K=9); 10 clients; symmetric
|
||||
Dirichlet non-IID with `alpha ∈ {0.25, 0.50, 0.75}`; seeds `{0,1,2}`; stratified train/validation/test
|
||||
= 70/10/20 (validation is a project decision; the paper uses 80/20); 10 rounds; 3 local epochs;
|
||||
batch 256; Adam; `lr=1e-3`, `mu=0.1`, `lambda=1.0` (project decisions). All compared configurations
|
||||
within a `(dataset, alpha, seed)` reuse exactly the same manifest, split and initialization
|
||||
(verified by content and partition hashes).
|
||||
|
||||
---
|
||||
|
||||
## 2. Completed experimental grid
|
||||
|
||||
| dataset | alpha | seeds | betas + baseline | runs | audit |
|
||||
|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 0,1,2 | 5 + 1 | 18 | PASS WITH MINOR CORRECTIONS (Session 7) |
|
||||
| X-IIoTID | 0.50 | 0,1,2 | 5 + 1 | 18 | PASS WITH MINOR CORRECTIONS (Session 6) |
|
||||
| X-IIoTID | 0.75 | 0,1,2 | 5 + 1 | 18 | PASS (Session 8) |
|
||||
| 5G-NIDD | 0.25 | 0,1,2 | 5 + 1 | 18 | PASS (Session 11) |
|
||||
| 5G-NIDD | 0.50 | 0,1,2 | 5 + 1 | 18 | PASS (Session 9) |
|
||||
| 5G-NIDD | 0.75 | 0,1,2 | 5 + 1 | 18 | PASS (Session 10) |
|
||||
| **total** | | | | **108** | |
|
||||
|
||||
---
|
||||
|
||||
## 3. Six-condition master result
|
||||
|
||||
Authoritative table: `artifacts/analysis/final__six_condition_master.csv` / `.json`.
|
||||
|
||||
| dataset | alpha | beta* | pooled val (std) | sel test rare-F1 | orig test rare-F1 | sel rare-recall | orig rare-recall | sel macro-F1 | sel acc | sel wF1 |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | **0.00** | 0.5917 (0.0895) | 0.6321 | 0.6328 | 0.8213 | 0.8220 | 0.6813 | 0.8740 | 0.9069 |
|
||||
| X-IIoTID | 0.50 | **0.75** | 0.5794 (0.0751) | 0.6655 | 0.6425 | 0.8626 | 0.8770 | 0.7312 | 0.9064 | 0.9336 |
|
||||
| X-IIoTID | 0.75 | **0.50** | 0.4767 (0.0466) | 0.6052 | 0.6039 | 0.8488 | 0.8565 | 0.7576 | 0.9281 | 0.9452 |
|
||||
| 5G-NIDD | 0.25 | **0.00** | 0.7615 (0.0219) | 0.8412 | 0.8423 | 0.9612 | 0.9606 | 0.8166 | 0.7302 | 0.7045 |
|
||||
| 5G-NIDD | 0.50 | **1.00** | 0.6924 (0.1391) | 0.8720 | 0.8805 | 0.9671 | 0.9729 | 0.8386 | 0.7433 | 0.7182 |
|
||||
| 5G-NIDD | 0.75 | **0.50** | 0.8091 (0.0587) | 0.8967 | 0.8799 | 0.9597 | 0.9612 | 0.8531 | 0.7523 | 0.7275 |
|
||||
|
||||
All selected betas are validation-only; `test_used_for_selection = false` in every run;
|
||||
`beta0_equivalence_class = B` in every condition. The selected beta takes values
|
||||
`{0.00, 0.50, 1.00}` across the six conditions: **not universal, not monotone, not a function of
|
||||
alpha alone**.
|
||||
|
||||
Selected-beta test metrics are **post-selection descriptive summaries**, not selection inputs.
|
||||
|
||||
---
|
||||
|
||||
## 4. Realized non-IID characteristics
|
||||
|
||||
Dirichlet `alpha` is *not* a realized-heterogeneity measure. Authoritative realized table:
|
||||
`artifacts/analysis/final__realized_heterogeneity.csv`.
|
||||
|
||||
| dataset | alpha | `|S_j|<M` / 27 | missing client-class cells | client-size ratio (mean / max) | mean support CV | max support CV | max per-class max/min |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 19 | 38 | 14.75 / 20.84 | 1.442 | 2.134 | 122,800 |
|
||||
| X-IIoTID | 0.50 | 7 | 9 | 7.41 / 9.89 | 1.183 | 2.095 | 12,215 |
|
||||
| X-IIoTID | 0.75 | 3 | 4 | 4.67 / 5.07 | 0.881 | 1.322 | 1,903 |
|
||||
| 5G-NIDD | 0.25 | 16 | 30 | 17.60 / 27.83 | 1.302 | 2.139 | 185,504 |
|
||||
| 5G-NIDD | 0.50 | 4 | 4 | 9.59 / 12.72 | 1.225 | 1.964 | 88,943 |
|
||||
| 5G-NIDD | 0.75 | 4 | 5 | 6.31 / 11.36 | 0.916 | 1.265 | 7,642.5 |
|
||||
|
||||
Measured facts:
|
||||
|
||||
- Realized support imbalance, missing-cell frequency and client-size ratio **decrease as alpha
|
||||
rises** within each dataset (alpha orders heterogeneity only on average).
|
||||
- Per-seed realized imbalance is **not monotone** even at fixed alpha: for 5G-NIDD alpha=0.50,
|
||||
seed 0 has client-size ratio 11.63 while alpha=0.25 seed 0 has 9.93.
|
||||
- All 10 clients are eligible in every condition (exactly two rare attack classes each).
|
||||
- Rare-class **union differs across seeds** in several conditions (e.g. 5G-NIDD alpha=0.25:
|
||||
union rare 6/6/8, union unseen 6/5/5), so test rare support is not constant across seeds and
|
||||
absolute cross-seed metric levels are not directly comparable.
|
||||
- The `S_j` reproduction assumption is materially exercised in 19/27 (X-IIoTID alpha=0.25),
|
||||
16/27 (5G-NIDD alpha=0.25) and ≤7/27 cells elsewhere.
|
||||
|
||||
**Interpretation**: `alpha` is a design knob; the realized partition (support CV, missing cells,
|
||||
client-size ratio) is the variable that actually differs across conditions.
|
||||
|
||||
---
|
||||
|
||||
## 5. Pooled beta-selection behavior
|
||||
|
||||
Authoritative curve table: `artifacts/analysis/final__pooled_validation_curves.csv` (30 rows).
|
||||
Figure: `artifacts/figures/final/figA_*_pooled_validation_beta_curve.png`.
|
||||
|
||||
Pooled validation rare-class macro-F1 (selected in bold):
|
||||
|
||||
| beta | X-IIoTID a0.25 | X-IIoTID a0.50 | X-IIoTID a0.75 | 5G-NIDD a0.25 | 5G-NIDD a0.50 | 5G-NIDD a0.75 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 0.00 | **0.5917** | 0.5444 | 0.4717 | **0.7615** | 0.6872 | 0.7781 |
|
||||
| 0.25 | 0.5887 | 0.5554 | 0.4744 | 0.7493 | 0.6451 | 0.7905 |
|
||||
| 0.50 | 0.5859 | 0.5673 | **0.4767** | 0.7257 | 0.6747 | **0.8091** |
|
||||
| 0.75 | 0.5795 | **0.5794** | 0.4725 | 0.7222 | 0.6884 | 0.7915 |
|
||||
| 1.00 | 0.5321 | 0.5735 | 0.4686 | 0.7376 | **0.6924** | 0.8043 |
|
||||
|
||||
Measured facts:
|
||||
|
||||
- Response is **not monotone** in beta in any condition:
|
||||
X-IIoTID alpha=0.25 decreases monotonically (0.5917→0.5321);
|
||||
X-IIoTID alpha=0.50 peaks at 0.75; X-IIoTID alpha=0.75 peaks at 0.50;
|
||||
5G-NIDD alpha=0.25 falls to a minimum at 0.75 then rises at 1.00;
|
||||
5G-NIDD alpha=0.50 peaks at 1.00; 5G-NIDD alpha=0.75 peaks at 0.50.
|
||||
- Between-beta spreads (0.008 – 0.060) are of the same order as, or smaller than, across-seed
|
||||
sample std (0.003 – 0.139). See section 13.
|
||||
|
||||
---
|
||||
|
||||
## 6. Selected-beta test results (descriptive only)
|
||||
|
||||
Table: `artifacts/analysis/final__selected_vs_original.csv`.
|
||||
Figure: `artifacts/figures/final/figF_selected_vs_original_test_rare_macro_f1.png`.
|
||||
|
||||
Descriptive difference `selected beta − original_protean` (mean over 3 seeds), **not** a test of
|
||||
statistical superiority:
|
||||
|
||||
| dataset | alpha | beta* | Δ rare-F1 | Δ rare-recall | Δ macro-F1 | Δ accuracy | Δ wF1 |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 0.00 | −0.0007 | −0.0007 | −0.0007 | −0.0014 | −0.0008 |
|
||||
| X-IIoTID | 0.50 | 0.75 | **+0.0230** | −0.0144 | +0.0222 | +0.0069 | +0.0089 |
|
||||
| X-IIoTID | 0.75 | 0.50 | +0.0014 | −0.0077 | +0.0058 | +0.0075 | +0.0061 |
|
||||
| 5G-NIDD | 0.25 | 0.00 | −0.0010 | +0.0007 | −0.0028 | −0.0007 | −0.0010 |
|
||||
| 5G-NIDD | 0.50 | 1.00 | −0.0086 | −0.0058 | −0.0038 | −0.0007 | −0.0005 |
|
||||
| 5G-NIDD | 0.75 | 0.50 | +0.0168 | −0.0015 | +0.0117 | +0.0013 | +0.0014 |
|
||||
|
||||
Measured facts:
|
||||
|
||||
- The largest positive descriptive rare-F1 difference (+0.0230, X-IIoTID alpha=0.50) is still
|
||||
smaller than that condition's between-seed sample std (0.0267) and than the original_protean
|
||||
cross-seed std (0.0287).
|
||||
- At both alpha=0.25 conditions the selected beta is 0, so the "selected" run **is** the beta=0
|
||||
run; the tiny differences from `original_protean` are floating-point divergence (class B), not
|
||||
a weighting effect.
|
||||
- Selected rare-recall is ≤ original rare-recall in 5 of 6 conditions: emphasis on high-support
|
||||
classes trades a little rare-class recall for rare-class precision.
|
||||
- 5G-NIDD test weighted-F1 sits below accuracy (hard, high-support Benign/UDPFlood); X-IIoTID
|
||||
weighted-F1 exceeds accuracy. This is a dataset characteristic, not a beta effect.
|
||||
|
||||
---
|
||||
|
||||
## 7. Direct prototype displacement
|
||||
|
||||
Authoritative tables: `final__prototype_displacement_by_beta.csv`,
|
||||
`final__prototype_displacement_by_class.csv`.
|
||||
Figures: `figD_prototype_displacement_beta0_vs_beta1.png`,
|
||||
`figE_support_cv_vs_displacement.png`.
|
||||
|
||||
Mean `||Cbar_j(beta) − Cbar_j(0)||` across classes, fixed round-0 snapshots (no retraining):
|
||||
|
||||
| dataset | alpha | β=0.25 | β=0.50 | β=0.75 | β=1.00 |
|
||||
|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 1.476 | 2.403 | 3.003 | **3.422** |
|
||||
| X-IIoTID | 0.50 | 0.826 | 1.374 | 1.780 | **2.099** |
|
||||
| X-IIoTID | 0.75 | 0.477 | 0.829 | 1.097 | **1.305** |
|
||||
| 5G-NIDD | 0.25 | 1.295 | 2.055 | 2.559 | **2.919** |
|
||||
| 5G-NIDD | 0.50 | 0.953 | 1.567 | 2.003 | **2.340** |
|
||||
| 5G-NIDD | 0.75 | 0.497 | 0.875 | 1.163 | **1.385** |
|
||||
|
||||
Measured facts:
|
||||
|
||||
- For a fixed snapshot, displacement is **monotone increasing in beta** in all six conditions
|
||||
(mechanically expected: larger beta shifts weight toward high-`N_ij` clients).
|
||||
- Displacement at beta=1 **decreases as alpha rises** in both datasets, tracking realized
|
||||
support imbalance.
|
||||
- Displacement is a **representation-aggregation** metric only; it is not detection quality.
|
||||
|
||||
---
|
||||
|
||||
## 8. Client-level effects
|
||||
|
||||
Authoritative table: `artifacts/analysis/final__client_effect_summary.csv` (90 rows:
|
||||
6 conditions × 3 seeds × 5 betas, `delta` = validation per-client rare-class macro-F1 relative
|
||||
to beta=0). Figure: `figG_client_effect_at_selected_beta.png`.
|
||||
|
||||
Aggregate at **beta=1.00** (mean of per-client mean delta across 3 seeds):
|
||||
|
||||
| dataset | alpha | mean Δ | improve / degrade / unchanged (clients, summed over seeds) |
|
||||
|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | **−0.0596** | 12 / 15 / 3 |
|
||||
| X-IIoTID | 0.50 | +0.0291 | 19 / 9 / 2 |
|
||||
| X-IIoTID | 0.75 | −0.0031 | 14 / 15 / 1 |
|
||||
| 5G-NIDD | 0.25 | **−0.0238** | 10 / 11 / 9 |
|
||||
| 5G-NIDD | 0.50 | +0.0051 | 16 / 10 / 4 |
|
||||
| 5G-NIDD | 0.75 | +0.0262 | 14 / 10 / 6 |
|
||||
|
||||
Measured facts:
|
||||
|
||||
- Support emphasis at beta=1 is most damaging at alpha=0.25 in both datasets (most negative
|
||||
mean delta, most client degradation), consistent with those conditions selecting beta=0.
|
||||
- Within a single condition, some clients improve while others degrade at every beta
|
||||
(both datasets, all alphas). `N_ij` vs ΔF1 is non-monotonic: the worst-degrading client is
|
||||
not always the lowest-support client.
|
||||
- Client identities are per-seed realizations and are **not pooled** across seeds.
|
||||
|
||||
---
|
||||
|
||||
## 9. RQ1 answer
|
||||
|
||||
> **RQ1.** How are class-support imbalance, differences between local prototypes, and rare-class
|
||||
> detection performance related in PROTEAN?
|
||||
|
||||
Evidence chain (all three alphas, both datasets):
|
||||
|
||||
*Mechanically established.*
|
||||
- Changing beta changes the normalized contribution weights `N_ij^beta / Σ_k N_kj^beta` and
|
||||
therefore the position of the global prototype `Cbar_j`. For a fixed snapshot, mean
|
||||
displacement grows monotonically with beta in all six conditions.
|
||||
|
||||
*Empirically observed (descriptive).*
|
||||
- Larger realized support imbalance (higher support CV, more missing cells, higher client-size
|
||||
ratio) tends to accompany larger beta=1 displacement, within both datasets:
|
||||
X-IIoTID (CV, disp) = (1.442, 3.422) → (1.183, 2.099) → (0.881, 1.305);
|
||||
5G-NIDD = (1.302, 2.919) → (1.225, 2.340) → (0.916, 1.385).
|
||||
- Imbalance also co-varies with selection behavior: both alpha=0.25 conditions are the most
|
||||
imbalanced and both selected beta=0.
|
||||
|
||||
*Not established.*
|
||||
- Larger displacement does **not** automatically imply better rare-class detection: at beta=1
|
||||
(maximal displacement) rare-class performance is worse than at the selected beta in most
|
||||
conditions, and validation response is non-monotone in beta.
|
||||
- The imbalance → displacement link is mechanical under the beta formula; the link from
|
||||
displacement to detection is confounded by end-to-end retraining and cannot be isolated with
|
||||
this design.
|
||||
|
||||
RQ1 can therefore be answered **descriptively**: support imbalance and prototype displacement
|
||||
co-vary, but displacement is not a proxy for rare-class detection quality.
|
||||
|
||||
---
|
||||
|
||||
## 10. RQ2 answer
|
||||
|
||||
> **RQ2.** How does changing beta affect the global prototype and rare-class detection
|
||||
> performance under different non-IID conditions?
|
||||
|
||||
*Direct aggregation effect (fixed snapshot, no retraining).*
|
||||
- Displacement from beta=0 is monotone increasing in beta in every condition; magnitude differs
|
||||
by condition (largest at alpha=0.25).
|
||||
- This is a pure weighting/geometry effect and is measured independently of training.
|
||||
|
||||
*End-to-end training effect (iterative).*
|
||||
- Pooled validation rare-class macro-F1 is **not monotone** in beta. Preference varies by
|
||||
condition: high beta is preferred at 5G-NIDD alpha=0.50 (beta=1.0), moderate beta at both
|
||||
alpha=0.75 conditions (beta=0.5), and the smallest beta at both alpha=0.25 conditions
|
||||
(beta=0.0).
|
||||
- Weighting helps some clients and hurts others within the same condition (section 8), so the
|
||||
condition-level preference is an average over heterogeneous client responses.
|
||||
|
||||
The two effects must not be merged: the geometry changes monotonically, the task-level response
|
||||
does not.
|
||||
|
||||
---
|
||||
|
||||
## 11. RQ3 answer
|
||||
|
||||
> **RQ3.** Does the validation-selected beta change across non-IID levels, and what is the
|
||||
> effect on overall and per-client performance?
|
||||
|
||||
Authoritative selected betas (validation only):
|
||||
|
||||
| dataset | alpha=0.25 | alpha=0.50 | alpha=0.75 |
|
||||
|---|---|---|---|
|
||||
| X-IIoTID | 0.00 | 0.75 | 0.50 |
|
||||
| 5G-NIDD | 0.00 | 1.00 | 0.50 |
|
||||
|
||||
Established conclusions:
|
||||
|
||||
- The selected beta **changes by condition**: it is not a single universal value and not a
|
||||
monotone function of alpha.
|
||||
- Both datasets select beta=0 at alpha=0.25 and beta=0.50 at alpha=0.75; they differ at
|
||||
alpha=0.50 (X-IIoTID 0.75 vs 5G-NIDD 1.00).
|
||||
- Only five discrete candidates were tested; no global optimum is claimed, and each value is
|
||||
"best among the tested candidates under pooled validation for that condition".
|
||||
|
||||
Overall effect (section 6): selected-beta test rare-F1 differences are small and positive in
|
||||
three conditions, small and negative in three; the largest positive difference is smaller than
|
||||
between-seed variability. Per-client effect (section 8): the selected beta improves the
|
||||
validation rare-class metric for a majority of clients in four conditions, but at both alpha=0.25
|
||||
conditions (where the selected beta is 0) there is trivially no weighting change, and at higher
|
||||
beta the effect is client-dependent.
|
||||
|
||||
---
|
||||
|
||||
## 12. Cross-dataset synthesis
|
||||
|
||||
- **Agreement**: matched-alpha selections coincide at alpha=0.25 (both 0.00) and alpha=0.75
|
||||
(both 0.50); they differ at alpha=0.50 (0.75 vs 1.00).
|
||||
- **Displacement ordering is consistent**: beta=1 displacement falls with alpha in both datasets,
|
||||
and ranking of conditions by displacement mirrors ranking by realized support CV.
|
||||
- **Rare-class objects are not comparable across datasets**: 5G-NIDD rare classes are low-volume
|
||||
flood/scan types; X-IIoTID rare classes are low-volume behavioural categories. Absolute metric
|
||||
levels must not be compared across datasets; only within-condition differences are meaningful.
|
||||
- **Metric asymmetry**: 5G-NIDD weighted-F1 < accuracy; X-IIoTID weighted-F1 > accuracy.
|
||||
- No cross-dataset causality, no dataset superiority, no dataset-independent beta rule.
|
||||
|
||||
---
|
||||
|
||||
## 13. Selection uncertainty
|
||||
|
||||
Table: `artifacts/analysis/final__selection_strength.csv`.
|
||||
|
||||
| dataset | alpha | beta* | best | 2nd best | abs gap | rel gap | top-2 gap / selected std | per-seed betas |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| X-IIoTID | 0.25 | 0.00 | 0.5917 | 0.5887 (0.25) | 0.0030 | 0.51% | 0.034 | 0.75 / 0.5 / 0.0 |
|
||||
| X-IIoTID | 0.50 | 0.75 | 0.5794 | 0.5735 (1.0) | 0.0059 | 1.02% | 0.078 | 0.5 / 1.0 / 0.75 |
|
||||
| X-IIoTID | 0.75 | 0.50 | 0.4767 | 0.4744 (0.25) | 0.0024 | 0.50% | 0.051 | 0.5 / 0.5 / 0.0 |
|
||||
| 5G-NIDD | 0.25 | 0.00 | 0.7615 | 0.7493 (0.25) | 0.0122 | 1.60% | 0.556 | 0.25 / 0.0 / 0.0 |
|
||||
| 5G-NIDD | 0.50 | 1.00 | 0.6924 | 0.6884 (0.75) | 0.0040 | 0.58% | 0.029 | 1.0 / 0.0 / 1.0 |
|
||||
| 5G-NIDD | 0.75 | 0.50 | 0.8091 | 0.8043 (1.0) | 0.0048 | 0.59% | 0.081 | 0.5 / 1.0 / 1.0 |
|
||||
|
||||
Quantitative basis for the (descriptive) classification: the top-two gap is compared with the
|
||||
selected beta's across-seed sample std. **In all six conditions the top-two gap is smaller than
|
||||
the selected-beta sample std** (ratios 0.03 – 0.56), so every selection is classified as
|
||||
*small numerical separation relative to seed variability*. No tie-break was ever exercised. The
|
||||
per-seed diagnostic beta agrees with the pooled selection in only 1–2 of 3 seeds per condition.
|
||||
|
||||
No inferential test (t-test, ANOVA, Wilcoxon, Friedman, regression) is applied. With three
|
||||
project-defined seeds per condition, such tests would be underpowered and would overstate the
|
||||
evidence; mean, sample std, per-seed values and the top-two gap are reported instead.
|
||||
|
||||
---
|
||||
|
||||
## 14. Threats to validity
|
||||
|
||||
### Internal validity
|
||||
- **`S_j` reproduction assumption**: the participant set / denominator rule is not verified
|
||||
against official code and is materially exercised in up to 19/27 cells. All results are
|
||||
conditional on it.
|
||||
- **beta=0 floating-point divergence**: beta=0 is semantically the equal mean but not bitwise
|
||||
identical to `original_protean`; selected-vs-original differences at alpha=0.25 are this
|
||||
divergence, not an effect. Equivalence class B everywhere.
|
||||
- **Frozen implementation**: aggregation and training code unchanged across all 108 runs;
|
||||
the same manifest/split/init is reused within each `(dataset, alpha, seed)`.
|
||||
- **Paired partitions/seeds**: verified by content, partition and split hashes.
|
||||
- **Validation/test separation**: beta selected from validation only; `test_used_for_selection`
|
||||
is false in all runs; test read only after selection.
|
||||
|
||||
### Construct validity
|
||||
- `N_ij` is **sample support, not prototype quality**; a larger `N_ij` does not mean a better
|
||||
prototype.
|
||||
- Rare-class definition is operational (two smallest positive-support attack classes per client
|
||||
from the training split); rare and unseen are never merged.
|
||||
- Prototype displacement is a **representation** metric, not detection quality.
|
||||
|
||||
### External validity
|
||||
- Only two IDS datasets (X-IIoTID, 5G-NIDD), 10 clients, three alpha levels, three
|
||||
project-defined seeds, one PROTEAN architecture and training configuration.
|
||||
- Rare-class compositions are dataset-specific and not comparable across datasets.
|
||||
|
||||
### Conclusion validity
|
||||
- Small seed count (3); between-beta spreads are comparable to across-seed std.
|
||||
- Discrete beta grid `{0,0.25,0.5,0.75,1.0}`; the true response between candidates is unobserved.
|
||||
- Weak numerical separation at all six conditions (section 13).
|
||||
- No inferential significance testing was performed or is claimed.
|
||||
|
||||
---
|
||||
|
||||
## 15. Contribution supported by evidence
|
||||
|
||||
1. A **controlled, beta-parameterized support-weighting** operator applied to the server-side
|
||||
prototype aggregation of PROTEAN, with an explicit participant rule and missing-class
|
||||
semantics.
|
||||
2. **Condition-wise, validation-based beta selection** (no test leakage) across six paired
|
||||
conditions.
|
||||
3. A **paired multi-seed analysis** across two IDS datasets and three non-IID levels, with
|
||||
identical partitions within each comparison cell.
|
||||
4. A **direct linkage** between realized support imbalance, weight redistribution, prototype
|
||||
displacement, rare-class detection and per-client consequences — separated into direct
|
||||
aggregation effects (fixed snapshots) and end-to-end training effects.
|
||||
5. Evidence of **when stronger support weighting helps, has little effect, or degrades**
|
||||
rare-class behavior, including the identification that both evaluated alpha=0.25 conditions
|
||||
select beta=0 (no weighting benefit) and that response is not monotone in beta.
|
||||
|
||||
**Novelty caveat.** Sample-count prototype weighting is already discussed in related work, e.g.
|
||||
**FedProto** (Tan et al., 2022). The contribution cannot rest on using `N_ij` or adding beta
|
||||
alone; it rests on the controlled parameterization, the condition-wise validation selection, the
|
||||
paired multi-seed setup, and the resulting behavioral findings.
|
||||
|
||||
---
|
||||
|
||||
## 16. Claims NOT supported by evidence
|
||||
|
||||
The following must not appear as established results:
|
||||
|
||||
- "Support weighting improves rare-class detection." (Mixed and condition-dependent; largest
|
||||
positive difference is within seed noise.)
|
||||
- "Selected-beta beats original PROTEAN." (Only small descriptive differences; no significance
|
||||
test; at alpha=0.25 the difference is floating-point divergence.)
|
||||
- "alpha causally determines the optimal beta." (Only two datasets, three seeds; selection is
|
||||
not monotone in alpha.)
|
||||
- "A larger `N_ij` means a better prototype / greater prototype quality."
|
||||
- "Larger prototype displacement means better (or worse) rare-class detection."
|
||||
- "beta=0 reproduces original PROTEAN" (only semantically equivalent under the shared `S_j`
|
||||
assumption; class B divergence).
|
||||
- "There is a universal / globally optimal beta."
|
||||
- "The method generalizes" or "dataset X is superior".
|
||||
- Any metric interpreted as improvement without the qualifier "numerical/descriptive".
|
||||
|
||||
---
|
||||
|
||||
## 17. Recommended thesis conclusions
|
||||
|
||||
1. Under the validated design, support weighting is **condition-dependent**: the best tested beta
|
||||
differs across non-IID conditions, is not monotone in alpha, and no single beta dominates.
|
||||
2. Changing beta **monotonically displaces** global prototypes at a fixed snapshot, while the
|
||||
rare-class detection response is **non-monotone** and weak relative to seed variability.
|
||||
3. Realized support imbalance and prototype displacement **co-vary**; displacement alone is not
|
||||
evidence of detection quality.
|
||||
4. Weighting **redistributes benefit across clients**; within a condition some clients gain and
|
||||
others lose.
|
||||
5. In both evaluated datasets the most imbalanced condition (alpha=0.25) selected beta=0, i.e.
|
||||
equal weighting over the shared participant set, and realized the largest prototype
|
||||
displacement — a descriptive pattern that motivates broader validation, not a rule.
|
||||
|
||||
---
|
||||
|
||||
## 18. Scientific-gap assessment
|
||||
|
||||
**Classification: A — NO MATERIAL GAP** for the defined thesis scope.
|
||||
|
||||
- RQ1, RQ2 and RQ3 are all answerable descriptively from the 108-run grid and the existing
|
||||
audited artifacts.
|
||||
- The remaining limitations (S_j assumption, three seeds, discrete grid, two datasets, no
|
||||
inferential power) are **scope/validity** boundaries, not missing experiments that prevent the
|
||||
stated research questions from being answered.
|
||||
- A **causal** isolation of the displacement→detection link (e.g. evaluating recomputed global
|
||||
prototypes on trained embeddings at a fixed snapshot, or interpolating beta in prototype space)
|
||||
would strengthen RQ1/RQ2, but it is not required by the defined RQs and would change the design;
|
||||
it is therefore out of scope rather than a blocking gap.
|
||||
- No new training is required or recommended.
|
||||
|
||||
---
|
||||
|
||||
## Artifact index (Session 12 deliverables)
|
||||
|
||||
| artifact | path |
|
||||
|---|---|
|
||||
| master table | `artifacts/analysis/final__six_condition_master.{csv,json}` |
|
||||
| pooled validation curves | `artifacts/analysis/final__pooled_validation_curves.csv` |
|
||||
| realized heterogeneity | `artifacts/analysis/final__realized_heterogeneity.csv` |
|
||||
| displacement by beta | `artifacts/analysis/final__prototype_displacement_by_beta.csv` |
|
||||
| displacement by class (beta=1) | `artifacts/analysis/final__prototype_displacement_by_class.csv` |
|
||||
| client-level effect | `artifacts/analysis/final__client_effect_summary.csv` |
|
||||
| selected vs original | `artifacts/analysis/final__selected_vs_original.csv` |
|
||||
| selection strength | `artifacts/analysis/final__selection_strength.csv` |
|
||||
| figures | `artifacts/figures/final/figA–G*.png` |
|
||||
| Chapter 4 outline | `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md` |
|
||||
| claims register | `docs/thesis/CLAIMS_REGISTER.md` |
|
||||
|
||||
**Environment note.** Figures require `matplotlib`, which is not in the frozen `research`
|
||||
dependency set. It was installed into the git-ignored `.venv` only
|
||||
(`uv pip install --python .venv/bin/python matplotlib`, matplotlib 3.11.2); no tracked
|
||||
dependency file was modified. The synthesis generator is `scripts`-style analysis code kept
|
||||
outside the repository at `/tmp/opencode/s12/synthesize_final_analysis.py`, consistent with the
|
||||
Session-11 convention for ad-hoc analysis helpers.
|
||||
@@ -0,0 +1,148 @@
|
||||
# Chapter 4 — Results and Discussion (Blueprint)
|
||||
|
||||
This is a **blueprint**, not a finished chapter. Each subsection states its purpose, the exact
|
||||
table/figure to use, the key numbers to quote (loaded from audited artifacts), what may be
|
||||
interpreted, and what must not be interpreted. All numbers trace to
|
||||
`docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md` and the `artifacts/analysis/final__*` files.
|
||||
|
||||
Global wording rules: prefer "numerical/descriptive difference", "best among the tested
|
||||
candidates under pooled validation", "co-varies"; never "improvement", "optimal", "causal",
|
||||
"superior", or "generalizes" unless explicitly qualified.
|
||||
|
||||
---
|
||||
|
||||
## 4.1 Experimental setup recap
|
||||
|
||||
- **Purpose**: fix the protocol so later numbers are reproducible and comparable.
|
||||
- **Use**: text; table of grid (Section 2 of synthesis); `docs/research/EXPERIMENT_PROTOCOL.md`.
|
||||
- **Key numbers**: 2 datasets × 3 alpha × 3 seeds × (5 betas + `original_protean`) = **108 runs**;
|
||||
betas `{0, 0.25, 0.5, 0.75, 1}`; 10 clients; 10 rounds; 3 local epochs; batch 256; Adam lr=1e-3,
|
||||
mu=0.1, lambda=1.0; train/val/test 70/10/20; validation-only selection.
|
||||
- **Allowed**: state that `S_j` is a reproduction assumption; state that beta=0 is not bitwise
|
||||
identical to the baseline.
|
||||
- **Avoid**: describing the design as new training beyond the frozen grid; implying the paper's
|
||||
split was 70/10/20.
|
||||
|
||||
## 4.2 Realized non-IID characteristics
|
||||
|
||||
- **Purpose**: show that alpha is a design knob and realized heterogeneity is the measured
|
||||
variable.
|
||||
- **Use**: Table `final__realized_heterogeneity.csv`; Figure `figC_realized_support_cv.png`.
|
||||
- **Key numbers**: support CV 1.442/1.183/0.881 (X-IIoTID) and 1.302/1.225/0.916 (5G-NIDD) for
|
||||
alpha 0.25/0.50/0.75; `|S_j|<M` 19/7/3 and 16/4/4 of 27; missing cells 38/9/4 and 30/4/5;
|
||||
client-size ratio 14.75/7.41/4.67 and 17.60/9.59/6.31.
|
||||
- **Allowed**: realized imbalance generally decreases as alpha rises; per-seed imbalance is not
|
||||
monotone at fixed alpha; rare-class union differs across seeds.
|
||||
- **Avoid**: treating Dirichlet alpha itself as the observed heterogeneity; comparing rare-class
|
||||
metrics across datasets.
|
||||
|
||||
## 4.3 Beta tuning results
|
||||
|
||||
- **Purpose**: present the pooled validation curves and the six selections.
|
||||
- **Use**: Table `final__pooled_validation_curves.csv`; Figures
|
||||
`figA_xiiotid_pooled_validation_beta_curve.png`,
|
||||
`figA_5gnidd_pooled_validation_beta_curve.png`, `figB_selected_beta_by_condition.png`;
|
||||
selection-strength table `final__selection_strength.csv`.
|
||||
- **Key numbers**: selections X-IIoTID 0.00 / 0.75 / 0.50 and 5G-NIDD 0.00 / 1.00 / 0.50 for
|
||||
alpha 0.25 / 0.50 / 0.75; response non-monotone in beta; all top-two gaps < selected-beta
|
||||
across-seed std (relative gaps 0.50% – 1.60%).
|
||||
- **Allowed**: selected beta changes by condition; selection is not monotone in alpha; no global
|
||||
optimum; separation is small relative to seed variability.
|
||||
- **Avoid**: calling the selected beta "optimal"; drawing a continuous optimum; reporting
|
||||
per-seed diagnostic betas as the primary result.
|
||||
|
||||
## 4.4 Effect on global prototypes
|
||||
|
||||
- **Purpose**: separate the direct (fixed-snapshot) aggregation effect from training.
|
||||
- **Use**: Table `final__prototype_displacement_by_beta.csv`; Figure
|
||||
`figD_prototype_displacement_beta0_vs_beta1.png`; per-class file
|
||||
`final__prototype_displacement_by_class.csv`.
|
||||
- **Key numbers**: mean `||Cbar(1)−Cbar(0)||` = 3.422 / 2.099 / 1.305 (X-IIoTID) and
|
||||
2.919 / 2.340 / 1.385 (5G-NIDD); monotone increasing in beta within each condition.
|
||||
- **Allowed**: displacement grows monotonically with beta for a fixed snapshot; magnitude differs
|
||||
by realized condition; larger beta moves the global prototype toward high-support clients.
|
||||
- **Avoid**: interpreting displacement as improved/degraded detection; implying a causal link
|
||||
from displacement to rare-class performance.
|
||||
|
||||
## 4.5 Rare-class detection results
|
||||
|
||||
- **Purpose**: report selected-beta test metrics against `original_protean` descriptively.
|
||||
- **Use**: master table `final__six_condition_master.csv`; table
|
||||
`final__selected_vs_original.csv`; Figure
|
||||
`figF_selected_vs_original_test_rare_macro_f1.png`.
|
||||
- **Key numbers**: Δ test rare-F1 = −0.0007 / +0.0230 / +0.0014 (X-IIoTID) and
|
||||
−0.0010 / −0.0086 / +0.0168 (5G-NIDD); largest positive difference (+0.0230) below that
|
||||
condition's between-seed std (0.0267); alpha=0.25 differences are floating-point divergence.
|
||||
- **Allowed**: differences are numerical/descriptive only; mixed sign; rare-recall is usually
|
||||
slightly lower under weighting.
|
||||
- **Avoid**: significance markers; "beats baseline"; merging rare and unseen; comparing absolute
|
||||
levels across conditions or datasets.
|
||||
|
||||
## 4.6 Per-client effects
|
||||
|
||||
- **Purpose**: show redistribution of benefit across clients.
|
||||
- **Use**: Table `final__client_effect_summary.csv`; Figure
|
||||
`figG_client_effect_at_selected_beta.png`.
|
||||
- **Key numbers**: beta=1 mean Δ = −0.0596 / +0.0291 / −0.0031 (X-IIoTID) and
|
||||
−0.0238 / +0.0051 / +0.0262 (5G-NIDD); improve/degrade/unchanged counts per condition.
|
||||
- **Allowed**: within a condition some clients gain and others lose; most damaging at alpha=0.25;
|
||||
`N_ij` vs ΔF1 is non-monotonic.
|
||||
- **Avoid**: pooling client identities across seeds; claiming weighting helps "all" or
|
||||
"low-support" clients.
|
||||
|
||||
## 4.7 Cross-dataset analysis
|
||||
|
||||
- **Purpose**: compare the two datasets at matched alpha without over-claiming.
|
||||
- **Use**: synthesis Section 12; cross-dataset artifacts `cross_dataset__matched_alpha_summary.csv`.
|
||||
- **Key numbers**: matched selections agree at alpha=0.25 (0.00) and alpha=0.75 (0.50), differ at
|
||||
alpha=0.50 (0.75 vs 1.00).
|
||||
- **Allowed**: descriptive agreement/disagreement; both datasets' displacement falls with alpha.
|
||||
- **Avoid**: cross-dataset metric comparison; dataset causality; universal beta rule; claiming a
|
||||
dataset-independent law.
|
||||
|
||||
## 4.8 RQ1 discussion
|
||||
|
||||
- **Purpose**: answer the support-imbalance ↔ prototype-difference ↔ rare-class relation.
|
||||
- **Use**: synthesis Section 9; sections 4.2, 4.4, 4.5.
|
||||
- **Key numbers**: imbalance ranking and displacement ranking coincide within each dataset.
|
||||
- **Allowed**: mechanical weight/prototype effect is established; imbalance and displacement
|
||||
co-vary descriptively; displacement is not detection quality.
|
||||
- **Avoid**: causal mediation claims; "displacement improves detection".
|
||||
|
||||
## 4.9 RQ2 discussion
|
||||
|
||||
- **Purpose**: answer how beta affects global prototypes and rare-class performance.
|
||||
- **Use**: synthesis Section 10; sections 4.3, 4.4, 4.6.
|
||||
- **Key numbers**: monotone displacement vs non-monotone validation response.
|
||||
- **Allowed**: distinguish direct aggregation effect from end-to-end training effect; weighting
|
||||
helps some clients and harms others.
|
||||
- **Avoid**: treating the monotonically growing displacement as a performance trend.
|
||||
|
||||
## 4.10 RQ3 discussion
|
||||
|
||||
- **Purpose**: answer whether the selected beta changes across non-IID levels and its effect.
|
||||
- **Use**: synthesis Section 11; sections 4.3, 4.5, 4.6.
|
||||
- **Key numbers**: selected betas `{0.00, 0.75, 0.50}` (X-IIoTID) and `{0.00, 1.00, 0.50}`
|
||||
(5G-NIDD); only five candidate betas tested.
|
||||
- **Allowed**: selection changes by condition; not monotone in alpha; no universal beta; both
|
||||
datasets select 0 at alpha=0.25 and 0.5 at alpha=0.75.
|
||||
- **Avoid**: "alpha causally determines beta"; "global optimum"; extrapolating to untested betas.
|
||||
|
||||
## 4.11 Threats / limitations
|
||||
|
||||
- **Purpose**: bound the conclusions honestly.
|
||||
- **Use**: synthesis Section 14.
|
||||
- **Key numbers**: S_j assumption material up to 19/27 cells; three seeds; discrete grid; two
|
||||
datasets; no inferential test.
|
||||
- **Allowed**: internal/construct/external/conclusion validity bullets as written.
|
||||
- **Avoid**: omitting the reproduction assumption; implying statistical significance.
|
||||
|
||||
## 4.12 Chapter summary
|
||||
|
||||
- **Purpose**: one-paragraph takeaway.
|
||||
- **Use**: synthesis Section 17.
|
||||
- **Key numbers**: condition-dependent beta, non-monotone response, displacement co-varies with
|
||||
imbalance, client-level heterogeneity, both alpha=0.25 conditions select beta=0.
|
||||
- **Allowed**: descriptive summary; explicit statement that no global optimum or universal beta
|
||||
was found and that only tested candidates were compared.
|
||||
- **Avoid**: any claim of improvement, causality, superiority or generalization.
|
||||
@@ -0,0 +1,149 @@
|
||||
# Thesis Claims Register
|
||||
|
||||
Every claim considered for Chapter 4/5, with its evidence, status, limitation and a
|
||||
thesis-safe wording. Statuses:
|
||||
|
||||
- **supported** — directly established by audited machine-readable artifacts or by the frozen
|
||||
operator definition.
|
||||
- **descriptive only** — observable in the artifacts but not causally or inferentially established.
|
||||
- **unsupported** — the design cannot defend the claim; must not appear as a result.
|
||||
|
||||
Artifacts referenced live under `artifacts/analysis/final__*`, `artifacts/analysis/*__multiseed__*`,
|
||||
and `docs/research/*`. Numbers are cross-checked in
|
||||
`docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md`.
|
||||
|
||||
---
|
||||
|
||||
## 1. Beta changes prototype positions
|
||||
|
||||
- **Status**: supported (mechanically established).
|
||||
- **Evidence**: `final__prototype_displacement_by_beta.csv`; frozen aggregator definition
|
||||
`Cbar_j(beta) = Σ N_ij^beta C_ij / Σ N_ij^beta`.
|
||||
- **Limitation**: conditional on the shared `S_j` reproduction assumption; fixed round-0 snapshot.
|
||||
- **Safe wording**: "Changing beta changes the normalized contribution weights and therefore the
|
||||
position of each global prototype."
|
||||
|
||||
## 2. Prototype displacement increases with beta
|
||||
|
||||
- **Status**: supported (monotone at fixed snapshot).
|
||||
- **Evidence**: `final__prototype_displacement_by_beta.csv` — mean displacement increases from
|
||||
beta=0.25 to beta=1.00 in all six conditions.
|
||||
- **Limitation**: measured on fixed round-0 snapshots; no retraining; not a detection metric.
|
||||
- **Safe wording**: "For a fixed snapshot, mean `||Cbar(beta)−Cbar(0)||` increases monotonically
|
||||
with beta in every evaluated condition."
|
||||
|
||||
## 3. Realized imbalance and prototype displacement co-vary
|
||||
|
||||
- **Status**: descriptive only.
|
||||
- **Evidence**: `final__realized_heterogeneity.csv` and `final__prototype_displacement_by_beta.csv`;
|
||||
both datasets show support CV and beta=1 displacement falling together as alpha rises.
|
||||
- **Limitation**: six conditions, three seeds; correlation not quantified with inferential
|
||||
statistics; displacement is a representation metric.
|
||||
- **Safe wording**: "Within both datasets, conditions with larger realized support imbalance also
|
||||
exhibit larger beta=1 prototype displacement; this is a descriptive co-variation."
|
||||
|
||||
## 4. Beta selection changes by condition
|
||||
|
||||
- **Status**: supported (measured, validation-only).
|
||||
- **Evidence**: `final__six_condition_master.csv`; `final__selection_strength.csv`.
|
||||
- **Limitation**: only five discrete candidates; selection is "best among tested candidates".
|
||||
- **Safe wording**: "The validation-selected beta differs across conditions and is not monotone in
|
||||
alpha; it is best among the tested candidates for each condition."
|
||||
|
||||
## 5. Beta is not universal
|
||||
|
||||
- **Status**: supported.
|
||||
- **Evidence**: selected betas `{0.00,0.75,0.50}` (X-IIoTID) and `{0.00,1.00,0.50}` (5G-NIDD).
|
||||
- **Limitation**: only two datasets and three alpha levels.
|
||||
- **Safe wording**: "The selected beta takes three distinct values across the six conditions and
|
||||
is not a single universal value; no global optimum is claimed."
|
||||
|
||||
## 6. Beta=0 selected at alpha=0.25 on both datasets
|
||||
|
||||
- **Status**: supported (measured).
|
||||
- **Evidence**: `final__six_condition_master.csv` (X-IIoTID and 5G-NIDD alpha=0.25 both select 0.00).
|
||||
- **Limitation**: two datasets, three seeds; descriptive pattern, not a rule. Because the selected
|
||||
beta is 0, these conditions cannot demonstrate any support-weighting benefit.
|
||||
- **Safe wording**: "In both evaluated datasets the alpha=0.25 condition selected beta=0 among the
|
||||
tested candidates; these conditions also showed the strongest realized support imbalance and the
|
||||
largest prototype displacement."
|
||||
|
||||
## 7. Selected beta beats original PROTEAN
|
||||
|
||||
- **Status**: unsupported.
|
||||
- **Evidence**: `final__selected_vs_original.csv` — differences are mixed in sign and small; the
|
||||
largest positive rare-F1 difference (+0.0230, X-IIoTID alpha=0.50) is below that condition's
|
||||
between-seed std (0.0267); alpha=0.25 differences are floating-point divergence (class B).
|
||||
- **Limitation**: no significance test; three seeds; selected beta is determined post hoc per
|
||||
condition.
|
||||
- **Safe wording**: "Selected-beta test metrics show only small, mixed-sign numerical differences
|
||||
from `original_protean`; no statistical superiority is established."
|
||||
|
||||
## 8. Support weighting improves rare-class detection
|
||||
|
||||
- **Status**: unsupported.
|
||||
- **Evidence**: `final__selected_vs_original.csv`, `final__client_effect_summary.csv` — response is
|
||||
condition-dependent, non-monotone in beta, and some clients degrade at every beta; at beta=1 the
|
||||
mean client delta is negative at both alpha=0.25 conditions and at X-IIoTID alpha=0.75.
|
||||
- **Limitation**: end-to-end results confound the weighting with retraining; small seed count.
|
||||
- **Safe wording**: "Support weighting changes rare-class behavior condition-dependently: it helps
|
||||
a majority of clients in some conditions and degrades most clients at the most imbalanced
|
||||
conditions; no uniform improvement is demonstrated."
|
||||
|
||||
## 9. Alpha causally determines the optimal beta
|
||||
|
||||
- **Status**: unsupported.
|
||||
- **Evidence**: `final__selection_strength.csv` — selection is not monotone in alpha; selected beta
|
||||
is `{0.00,0.75,0.50}` and `{0.00,1.00,0.50}`, differing at alpha=0.50.
|
||||
- **Limitation**: only three alpha levels, two datasets, three seeds; no trend test performed.
|
||||
- **Safe wording**: "The selected beta changes with non-IID level but is not a monotone function of
|
||||
alpha; no causal role of alpha is established."
|
||||
|
||||
## 10. Beta=1 represents prototype quality
|
||||
|
||||
- **Status**: unsupported.
|
||||
- **Evidence**: spec and protocol define `N_ij` as class-specific sample support, not prototype
|
||||
quality; beta=1 weights by support only.
|
||||
- **Limitation**: no ground-truth prototype-quality measure exists in the design.
|
||||
- **Safe wording**: "`N_ij` is class-specific sample support; a larger `N_ij` does not imply a
|
||||
higher-quality prototype, and beta weights support, not quality."
|
||||
|
||||
## 11. Prototype displacement implies detection quality
|
||||
|
||||
- **Status**: unsupported.
|
||||
- **Evidence**: `final__prototype_displacement_by_beta.csv` vs `final__pooled_validation_curves.csv`
|
||||
— displacement is monotone in beta while detection response is non-monotone; at beta=1 detection
|
||||
is usually worse than at the selected beta.
|
||||
- **Limitation**: displacement measured on untreated round-0 embeddings; detection measured
|
||||
end-to-end.
|
||||
- **Safe wording**: "Prototype displacement is a representation-aggregation metric; it is not
|
||||
interpreted as improved or degraded detection."
|
||||
|
||||
## 12. Beta=0 reproduces original PROTEAN
|
||||
|
||||
- **Status**: unsupported (conditional equivalence only).
|
||||
- **Evidence**: `*__multiseed__beta0_equivalence.csv` — same manifest/snapshot/support but
|
||||
independent-aggregation differences ~2e-16 and non-zero history/test metric differences;
|
||||
classification **B** in every condition.
|
||||
- **Limitation**: conditional on the shared `S_j` reproduction assumption; no official code.
|
||||
- **Safe wording**: "Under the shared participant-set assumption, beta=0 is semantically the equal
|
||||
mean over `S_j` and is numerically close to `original_protean`; it is not a proof that beta=0
|
||||
reproduces the original method."
|
||||
|
||||
## 13. Rare-class and unseen-class performance can be merged
|
||||
|
||||
- **Status**: unsupported.
|
||||
- **Evidence**: protocol defines rare (`N_ij>0`, two smallest positive-support attack classes) and
|
||||
unseen (`N_ij=0`) as distinct categories from the training split.
|
||||
- **Limitation**: none; definitional.
|
||||
- **Safe wording**: "Rare-class and unseen-class results are reported separately and are never
|
||||
merged into one metric."
|
||||
|
||||
## 14. Results generalize beyond the tested setting
|
||||
|
||||
- **Status**: unsupported.
|
||||
- **Evidence**: two datasets, 10 clients, three alpha levels, three project-defined seeds, one
|
||||
architecture/training configuration.
|
||||
- **Limitation**: external validity.
|
||||
- **Safe wording**: "Findings are specific to the two evaluated IDS datasets and the frozen
|
||||
PROTEAN configuration; no generalization is claimed."
|
||||