diff --git a/artifacts/analysis/final__client_effect_summary.csv b/artifacts/analysis/final__client_effect_summary.csv new file mode 100644 index 0000000..f805e1b --- /dev/null +++ b/artifacts/analysis/final__client_effect_summary.csv @@ -0,0 +1,91 @@ +dataset,alpha,seed,beta,n_eligible,n_improve_vs_beta0,n_degrade_vs_beta0,n_unchanged,mean_delta_rare_macro_f1,median_delta_rare_macro_f1,min_delta_rare_macro_f1,max_delta_rare_macro_f1 +xiiotid,0.25,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.25,0,0.25,10,2,5,3,-0.0064673967169698865,-0.00222622943520559,-0.04293419597212661,0.01876993547229222 +xiiotid,0.25,0,0.5,10,2,7,1,-0.021132734174687993,-0.029027854121098817,-0.07692307692307698,0.05741836974395109 +xiiotid,0.25,0,0.75,10,5,4,1,0.037652802750874605,0.0007104290991758952,-0.11166156370194247,0.3758987633016968 +xiiotid,0.25,0,1.0,10,4,5,1,-0.037995631629473454,-0.022968705139247758,-0.25698055252395446,0.14104421200350115 +xiiotid,0.25,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.25,1,0.25,10,7,1,2,0.02181236632914685,0.022496227330692564,-0.006350012700025401,0.07175976877899254 +xiiotid,0.25,1,0.5,10,7,2,1,0.04461512377469677,0.02108028499658518,-0.015820270172650433,0.23562664832014674 +xiiotid,0.25,1,0.75,10,5,4,1,0.04324763034707273,0.0052932124515639445,-0.018266942973367145,0.316706904876871 +xiiotid,0.25,1,1.0,10,5,4,1,0.0385170238078155,0.0052932124515639445,-0.038742745973951775,0.37740563530037213 +xiiotid,0.25,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.25,2,0.25,10,5,3,2,-0.024384536677342937,0.013768617160351215,-0.3028515240904621,0.08185020136870569 +xiiotid,0.25,2,0.5,10,4,4,2,-0.04103939310459288,0.0,-0.3397932816537467,0.11851572403806943 +xiiotid,0.25,2,0.75,10,3,5,2,-0.1176216808834886,-0.03792837801605936,-0.6011131725417439,0.12413263414375775 +xiiotid,0.25,2,1.0,10,3,6,1,-0.1793367342416782,-0.1787074870427493,-0.6775352307267201,0.07541312385625187 +xiiotid,0.5,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.5,0,0.25,10,9,1,0,0.014447779488805679,0.008862381334935221,-0.0048289738430582485,0.059259259259259345 +xiiotid,0.5,0,0.5,10,8,2,0,0.02842211408129357,0.01786607568659304,-0.02864627632429495,0.09259259259259256 +xiiotid,0.5,0,0.75,10,7,3,0,0.020663517144586436,0.011529026224022465,-0.03579795256891627,0.09259259259259256 +xiiotid,0.5,0,1.0,10,4,6,0,-0.005896451782293366,-0.0015897910791723424,-0.07604807732022623,0.04139430366632779 +xiiotid,0.5,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.5,1,0.25,10,6,2,2,0.007580540867216318,0.00691255565746568,-0.004456876015858091,0.023990860624524046 +xiiotid,0.5,1,0.5,10,7,2,1,0.016564399764498333,0.016189304599793786,-0.005423903365265936,0.04237609975314893 +xiiotid,0.5,1,0.75,10,8,1,1,0.026535489229920235,0.02453891247906187,-0.002998819235144723,0.0871872501828449 +xiiotid,0.5,1,1.0,10,9,0,1,0.054492862707554934,0.03303785159404127,0.0,0.19273339273339285 +xiiotid,0.5,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.5,2,0.25,10,7,1,2,0.011191488545697032,0.009832777111244473,-0.0007113366760957973,0.029454054607266078 +xiiotid,0.5,2,0.5,10,8,0,2,0.02389697185121845,0.019768783103869897,0.0,0.06966444292595181 +xiiotid,0.5,2,0.75,10,8,1,1,0.05781929956312084,0.0358882506416448,-0.0018572153733706909,0.1770966368667518 +xiiotid,0.5,2,1.0,10,6,3,1,0.038758776558190745,0.0271534489476824,-0.4028071047074897,0.2675649974447385 +xiiotid,0.75,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.75,0,0.25,10,5,5,0,8.859496571733683e-05,-0.001404275185996784,-0.019237420306723707,0.026871905512682204 +xiiotid,0.75,0,0.5,10,6,4,0,0.007584665753678871,0.013188342258148383,-0.0361435910590483,0.048012173012172976 +xiiotid,0.75,0,0.75,10,3,7,0,-0.009935209564988324,-0.010284597102047655,-0.04332234195639206,0.02528490028490027 +xiiotid,0.75,0,1.0,10,1,9,0,-0.017657004095942678,-0.013236301949758722,-0.0650297324550243,0.02220072031392789 +xiiotid,0.75,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.75,1,0.25,10,9,0,1,0.01311050414693552,0.010627333455872107,0.0,0.03177895616667353 +xiiotid,0.75,1,0.5,10,9,0,1,0.014643364516424288,0.010097783138646538,0.0,0.04017223028543784 +xiiotid,0.75,1,0.75,10,9,0,1,0.0127774748399905,0.007494938523124495,0.0,0.03752345215759845 +xiiotid,0.75,1,1.0,10,7,2,1,0.011258579628480089,0.005364141224669883,-0.011620820732722359,0.05776636713735561 +xiiotid,0.75,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +xiiotid,0.75,2,0.25,10,7,3,0,-0.005211888340901932,0.0029998983543945423,-0.12121212121212133,0.03817567567567559 +xiiotid,0.75,2,0.5,10,6,4,0,-0.007096659882482407,0.0024387861756007545,-0.1523809523809524,0.047814685314685246 +xiiotid,0.75,2,0.75,10,5,5,0,-0.00033649270926734557,-0.0004809578413735027,-0.08602150537634423,0.05314748201438846 +xiiotid,0.75,2,1.0,10,6,4,0,-0.002902966134960444,0.0017994539960968226,-0.12121212121212133,0.05314748201438846 +5gnidd,0.25,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.25,0,0.25,10,3,3,4,0.010938546012782963,0.0,-0.07653222666610299,0.18940707383214095 +5gnidd,0.25,0,0.5,10,3,4,3,-0.01890975297353068,0.0,-0.10737418541550403,0.023843748084083516 +5gnidd,0.25,0,0.75,10,3,4,3,-0.023097869142088355,0.0,-0.14782686599098172,0.08120187997137751 +5gnidd,0.25,0,1.0,10,3,4,3,-0.0019048472317972909,0.0,-0.1484580166381656,0.2777777777777778 +5gnidd,0.25,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.25,1,0.25,10,3,3,4,-0.003968051685100719,0.0,-0.035080645161290336,0.006988021376510556 +5gnidd,0.25,1,0.5,10,2,4,4,-0.03304496144720663,0.0,-0.31650978982621863,0.0007581492441447457 +5gnidd,0.25,1,0.75,10,2,5,3,-0.04279009081982739,-1.5429040849390674e-06,-0.4057158717743099,0.05971472095277662 +5gnidd,0.25,1,1.0,10,3,4,3,-0.0319707599550853,0.0,-0.3007384916695693,0.05547720837266114 +5gnidd,0.25,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.25,2,0.25,10,0,4,6,-0.04347321695249963,0.0,-0.20521276595744675,0.0 +5gnidd,0.25,2,0.5,10,1,3,6,-0.05526190135183806,0.0,-0.2831161695447409,0.002564102564102555 +5gnidd,0.25,2,0.75,10,1,4,5,-0.05176638308929169,0.0,-0.26831675979891445,0.09614699026463747 +5gnidd,0.25,2,1.0,10,4,3,3,-0.03757538403361462,0.0,-0.2684826405938592,0.1510489510489511 +5gnidd,0.5,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.5,0,0.25,10,2,7,1,-0.07713608929072921,-0.06868699969879549,-0.20363636363636362,0.030406580954845275 +5gnidd,0.5,0,0.5,10,7,2,1,0.025889785020554917,0.037598391946217946,-0.03433523137896821,0.06985074626865678 +5gnidd,0.5,0,0.75,10,5,4,1,0.012063078830364903,0.005480145076796034,-0.06785615288783342,0.07125000000000004 +5gnidd,0.5,0,1.0,10,6,3,1,0.030279508202206325,0.038493190742204336,-0.0884443881819511,0.10341347704408088 +5gnidd,0.5,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.5,1,0.25,10,3,3,4,-0.05952678669505902,0.0,-0.32307692307692304,0.01253088893341342 +5gnidd,0.5,1,0.5,10,3,5,2,-0.06985503383490366,-0.0002538055290239316,-0.423076923076923,0.018147843026199484 +5gnidd,0.5,1,0.75,10,3,5,2,-0.08992415594170661,-0.010869565217391242,-0.548076923076923,0.023966835082264493 +5gnidd,0.5,1,1.0,10,3,5,2,-0.10147959420243684,-0.010869565217391242,-0.6564102564102563,0.06308143239454578 +5gnidd,0.5,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.5,2,0.25,10,5,4,1,0.01020569435994877,0.00047281323877068626,-0.026210680207824577,0.07260640841702526 +5gnidd,0.5,2,0.5,10,5,4,1,0.006281201274024195,0.0021822149481723896,-0.04423375396450502,0.07378740970072228 +5gnidd,0.5,2,0.75,10,7,2,1,0.08127477908163376,0.08683194022289764,-0.030503434250412576,0.19744802710549003 +5gnidd,0.5,2,1.0,10,7,2,1,0.08661837030164057,0.10629718599862731,-0.033724722765818727,0.2015644281090505 +5gnidd,0.75,0,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.75,0,0.25,10,5,4,1,-0.011293149808252767,0.0008702225660137675,-0.11428571428571432,0.02246687492367061 +5gnidd,0.75,0,0.5,10,4,6,0,0.002110296281339108,-0.002470904928741302,-0.04342273307790534,0.07435002353424269 +5gnidd,0.75,0,0.75,10,4,5,1,-0.010014602019574038,-0.0009496838684330267,-0.1261904761904763,0.08104016437931305 +5gnidd,0.75,0,1.0,10,4,5,1,-0.018907127202940695,-0.008343874739802304,-0.1470238095238096,0.08792669889742655 +5gnidd,0.75,1,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.75,1,0.25,10,8,0,2,0.057402269299170985,0.034854877143638385,0.0,0.15555555555555545 +5gnidd,0.75,1,0.5,10,6,3,1,0.08504839334967931,0.036259163913778814,-0.01717171717171706,0.2666666666666666 +5gnidd,0.75,1,0.75,10,6,3,1,0.06946744905696867,0.036259163913778814,-0.02220504036522919,0.23333333333333328 +5gnidd,0.75,1,1.0,10,6,3,1,0.08868113938603633,0.034854877143638385,-0.03122327240008249,0.32110579479000534 +5gnidd,0.75,2,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +5gnidd,0.75,2,0.25,10,0,6,4,-0.008941010132134792,-0.007204243679679367,-0.022597337621727998,0.0 +5gnidd,0.75,2,0.5,10,4,2,4,0.005804717069634502,0.0,-0.005379760304999981,0.02370370370370367 +5gnidd,0.75,2,0.75,10,1,6,3,-0.019143806586356792,-0.0066193960137049535,-0.07992013632257533,0.0032679738562091387 +5gnidd,0.75,2,1.0,10,4,2,4,0.008864477513365766,0.0,-0.007116151507512436,0.03763472261159395 diff --git a/artifacts/analysis/final__pooled_validation_curves.csv b/artifacts/analysis/final__pooled_validation_curves.csv new file mode 100644 index 0000000..290d671 --- /dev/null +++ b/artifacts/analysis/final__pooled_validation_curves.csv @@ -0,0 +1,31 @@ +dataset,alpha,beta,pooled_validation_mean,sample_std,pooled_rare_recall,selected +xiiotid,0.25,0.0,0.5917038146634539,0.08952883653922933,0.7594821244948339,True +xiiotid,0.25,0.25,0.5886906256417319,0.06643525635398827,0.7807713039840465,False +xiiotid,0.25,0.5,0.5858514801619258,0.044857849968223555,0.7725355473131942,False +xiiotid,0.25,0.75,0.5794633987349401,0.0649060573642385,0.7607912573123681,False +xiiotid,0.25,1.0,0.5320987006423418,0.045718338534042856,0.704574711153059,False +xiiotid,0.5,0.0,0.5443606875936857,0.09144660841267144,0.8390875794416744,False +xiiotid,0.5,0.25,0.5554339572275921,0.09006816161098347,0.8412777237951431,False +xiiotid,0.5,0.5,0.5673218494926892,0.08853898624879539,0.8388718664026588,False +xiiotid,0.5,0.75,0.5793667895728949,0.07512057331236716,0.8290881018460031,True +xiiotid,0.5,1.0,0.5734790834215032,0.1004496749103663,0.8207630276313397,False +xiiotid,0.75,0.0,0.471705692507431,0.055329314644425844,0.7872893964865675,False +xiiotid,0.75,0.25,0.47436809609801456,0.04605501919031881,0.7842747460964024,False +xiiotid,0.75,0.5,0.47674948263663786,0.04662425884469754,0.7865311249627326,True +xiiotid,0.75,0.75,0.47254095002934254,0.04602831582869046,0.7807290076221336,False +xiiotid,0.75,1.0,0.46860522897329,0.044741855341153436,0.7813087218498457,False +5gnidd,0.25,0.0,0.7614534210908004,0.021902537932118035,0.9085770977445251,True +5gnidd,0.25,0.25,0.7492858468825278,0.017106975874172658,0.9071367963077813,False +5gnidd,0.25,0.5,0.7257145491666086,0.01101731431676257,0.8949473995819205,False +5gnidd,0.25,0.75,0.7222353067403979,0.007966771094060871,0.8797107780157537,False +5gnidd,0.25,1.0,0.7376364240173013,0.002755842556750987,0.8889910199323897,False +5gnidd,0.5,0.0,0.6872301309573783,0.052725844344546816,0.9477370387940001,False +5gnidd,0.5,0.25,0.6450777370820986,0.06474313739822393,0.9468616498164765,False +5gnidd,0.5,0.5,0.6746687817772702,0.10225286523806497,0.9436587867425653,False +5gnidd,0.5,0.75,0.6883680316141424,0.12603192659396695,0.9419154392409256,False +5gnidd,0.5,1.0,0.6923695590578484,0.13914040832229815,0.9423005037244669,True +5gnidd,0.75,0.0,0.7780936320958092,0.05096339060419691,0.9598578766446005,False +5gnidd,0.75,0.25,0.7904830018820702,0.05510032325554253,0.9596139786642895,False +5gnidd,0.75,0.5,0.8090814343293601,0.05873543815277214,0.9592678831536903,True +5gnidd,0.75,0.75,0.7915299789128217,0.06497400095064636,0.9596244940396749,False +5gnidd,0.75,1.0,0.8043064619946296,0.05480323650098284,0.9600406871348003,False diff --git a/artifacts/analysis/final__prototype_displacement_by_beta.csv b/artifacts/analysis/final__prototype_displacement_by_beta.csv new file mode 100644 index 0000000..0f23977 --- /dev/null +++ b/artifacts/analysis/final__prototype_displacement_by_beta.csv @@ -0,0 +1,31 @@ +dataset,alpha,beta,mean_displacement,std_sample_across_classes,n_classes +xiiotid,0.25,0.0,0.0,0.0,9 +xiiotid,0.25,0.25,1.4759443927974063,0.3915876677126786,9 +xiiotid,0.25,0.5,2.403216360439701,0.5578785827790653,9 +xiiotid,0.25,0.75,3.0026175335496204,0.6643181322495177,9 +xiiotid,0.25,1.0,3.42212533168931,0.7740752123481485,9 +xiiotid,0.5,0.0,0.0,0.0,9 +xiiotid,0.5,0.25,0.8255823617460618,0.31831330857243423,9 +xiiotid,0.5,0.5,1.3742233752967101,0.43187373764463177,9 +xiiotid,0.5,0.75,1.7801302991334842,0.5043074746507549,9 +xiiotid,0.5,1.0,2.0985368999756338,0.578855443340215,9 +xiiotid,0.75,0.0,0.0,0.0,9 +xiiotid,0.75,0.25,0.47722483146317857,0.12025761466601799,9 +xiiotid,0.75,0.5,0.8294921426085301,0.21286956479369043,9 +xiiotid,0.75,0.75,1.0965820930128614,0.2902562856464354,9 +xiiotid,0.75,1.0,1.305110487896406,0.3553065348087712,9 +5gnidd,0.25,0.0,0.0,0.0,9 +5gnidd,0.25,0.25,1.2954097330224899,0.4912234848812893,9 +5gnidd,0.25,0.5,2.055413931792982,0.6597523296778056,9 +5gnidd,0.25,0.75,2.558521399711773,0.7355506793948603,9 +5gnidd,0.25,1.0,2.91943348262683,0.7911632168313667,9 +5gnidd,0.5,0.0,0.0,0.0,9 +5gnidd,0.5,0.25,0.9534086318837013,0.3722328495814873,9 +5gnidd,0.5,0.5,1.5672291368417097,0.5712537132762766,9 +5gnidd,0.5,0.75,2.0032214267954767,0.6904879863519336,9 +5gnidd,0.5,1.0,2.3399083587182736,0.770453813804466,9 +5gnidd,0.75,0.0,0.0,0.0,9 +5gnidd,0.75,0.25,0.496912846928573,0.2311875190119548,9 +5gnidd,0.75,0.5,0.8749997159830957,0.3878012596482038,9 +5gnidd,0.75,0.75,1.1629465210044903,0.4928020443540843,9 +5gnidd,0.75,1.0,1.3846165631331335,0.5620843843793245,9 diff --git a/artifacts/analysis/final__prototype_displacement_by_class.csv b/artifacts/analysis/final__prototype_displacement_by_class.csv new file mode 100644 index 0000000..6f081cf --- /dev/null +++ b/artifacts/analysis/final__prototype_displacement_by_class.csv @@ -0,0 +1,55 @@ +dataset,alpha,class_id,class_name,mean_displacement,std_sample_displacement,mean_relative_displacement +xiiotid,0.25,0,C&C,4.01952572799183,0.29030488023104745,0.8472326753379348 +xiiotid,0.25,1,Exfiltration,4.316178984369498,0.8975835034935007,0.6638254826227857 +xiiotid,0.25,2,Exploitation,3.464004570499769,1.112779630373774,0.7020410983796918 +xiiotid,0.25,3,Lateral_movement,3.7263813611395125,1.3833916615643342,0.7895377723300525 +xiiotid,0.25,4,Normal,1.9966341269597232,0.5327206047251788,0.3701992118201333 +xiiotid,0.25,5,RDOS,2.5658707936642617,1.2193483512552363,0.3281078404629119 +xiiotid,0.25,6,Reconnaissance,3.3391889586431325,1.1073644700689869,0.6402396428113026 +xiiotid,0.25,7,Tampering,4.247813103636777,0.24113229030491104,0.9653304277937593 +xiiotid,0.25,8,Weaponization,3.1235303582992926,0.585141395295389,0.5634271266802224 +xiiotid,0.5,0,C&C,1.6935576092145832,0.5048307458168824,0.3187155176777392 +xiiotid,0.5,1,Exfiltration,2.0204769683934525,0.5020218016853628,0.2973123924755737 +xiiotid,0.5,2,Exploitation,2.343885289699028,0.4121833629248462,0.4231523311819784 +xiiotid,0.5,3,Lateral_movement,2.7240136758504594,1.0350790616091956,0.40257578575923714 +xiiotid,0.5,4,Normal,1.557331074720638,0.5480258187545208,0.2920883603702228 +xiiotid,0.5,5,RDOS,2.6312389613743776,1.1651317404804817,0.3523347102777825 +xiiotid,0.5,6,Reconnaissance,1.00151995809175,0.26099624924542353,0.16884924727680906 +xiiotid,0.5,7,Tampering,2.354018138564271,1.165407725687578,0.48267240310142356 +xiiotid,0.5,8,Weaponization,2.560790423872144,1.2282783431433644,0.4334295567239335 +xiiotid,0.75,0,C&C,1.2839465030740465,0.5431989970423492,0.2617816727650299 +xiiotid,0.75,1,Exfiltration,1.4475761921343262,0.08598433748869257,0.20679504552799255 +xiiotid,0.75,2,Exploitation,1.5392602070293495,0.25171201225922846,0.2839417733897609 +xiiotid,0.75,3,Lateral_movement,1.3693216274264168,0.5527158531960167,0.2618053240393338 +xiiotid,0.75,4,Normal,0.9031246281714095,0.298425213658255,0.1619501084487235 +xiiotid,0.75,5,RDOS,0.8141011244781585,0.20007357231332454,0.0957646213985643 +xiiotid,0.75,6,Reconnaissance,0.9825580428619212,0.4118545897797079,0.17068982082838355 +xiiotid,0.75,7,Tampering,1.934147031540858,0.4266139988379522,0.397088566305455 +xiiotid,0.75,8,Weaponization,1.4719590343511655,0.17030548300304388,0.23542365279592728 +5gnidd,0.25,0,Benign,2.648344737773736,0.5188602564457683,0.4821737572132332 +5gnidd,0.25,1,HTTPFlood,3.7602841668463944,1.5390234836296783,0.5584139964274445 +5gnidd,0.25,2,ICMPFlood,1.2149615886933787,0.1507085895098706,0.34685106134597526 +5gnidd,0.25,3,SYNFlood,3.1763695442454467,1.0069739670231537,0.9167291408106705 +5gnidd,0.25,4,SYNScan,3.814579780422797,1.4018369744978139,0.5442267347415422 +5gnidd,0.25,5,SlowrateDoS,3.178870743072719,1.1570108600313702,0.5619090743749164 +5gnidd,0.25,6,TCPConnectScan,3.257943080113345,2.5741100653263422,0.8873673593640223 +5gnidd,0.25,7,UDPFlood,2.4077369723424735,0.5783305176966848,0.47667894621080914 +5gnidd,0.25,8,UDPScan,2.815810730131176,1.6041886744624576,0.37142192996652196 +5gnidd,0.5,0,Benign,1.6898663850795066,0.6987889577561244,0.29591075596118094 +5gnidd,0.5,1,HTTPFlood,3.120458869697078,1.2894156117591726,0.43348480339644263 +5gnidd,0.5,2,ICMPFlood,1.1588454233322945,0.09271748323377921,0.32092174099674725 +5gnidd,0.5,3,SYNFlood,2.2617693416611573,0.7483497200280906,0.6584039968772434 +5gnidd,0.5,4,SYNScan,2.8752781124355127,0.9588449812962653,0.3958823033995225 +5gnidd,0.5,5,SlowrateDoS,2.763430109051773,1.4542368289687693,0.4536612328161466 +5gnidd,0.5,6,TCPConnectScan,2.130611279721881,0.21282292441201225,0.5753306009264239 +5gnidd,0.5,7,UDPFlood,1.5982465815194804,0.42810184263788215,0.30005165770255965 +5gnidd,0.5,8,UDPScan,3.460669125965779,2.0113615605321216,0.4243985120323746 +5gnidd,0.75,0,Benign,0.8242192109522325,0.2718678603634198,0.13820856413319157 +5gnidd,0.75,1,HTTPFlood,1.3664020721980037,0.5435449017657998,0.17507931126042764 +5gnidd,0.75,2,ICMPFlood,0.6733630855518974,0.33119344952410124,0.1786782977125397 +5gnidd,0.75,3,SYNFlood,1.8966313690365364,0.1707004572307703,0.5423454423708408 +5gnidd,0.75,4,SYNScan,1.5627721380862394,0.6395395815733285,0.19898769631915747 +5gnidd,0.75,5,SlowrateDoS,1.5649187261569397,0.654138365253272,0.2522141838283814 +5gnidd,0.75,6,TCPConnectScan,1.2260809195578348,0.4757573735353394,0.32576611133643296 +5gnidd,0.75,7,UDPFlood,0.9026363032967967,0.10316225038256528,0.16267861998433658 +5gnidd,0.75,8,UDPScan,2.444525243361723,0.48441013395931937,0.2850436623074378 diff --git a/artifacts/analysis/final__realized_heterogeneity.csv b/artifacts/analysis/final__realized_heterogeneity.csv new file mode 100644 index 0000000..abeae9c --- /dev/null +++ b/artifacts/analysis/final__realized_heterogeneity.csv @@ -0,0 +1,7 @@ +dataset,alpha,cells_sj_lt_M_total,cells_total,missing_client_class_cells_total,client_size_ratio_mean,client_size_ratio_max,support_cv_mean,support_cv_max_where_available,support_max_min_ratio_max,union_rare_mean,union_unseen_mean,max_unseen_on_client,eligible_clients_per_seed,union_rare_per_seed,union_unseen_per_seed +xiiotid,0.25,19,27,38,14.745270966890343,20.837033843500755,1.4423832412721973,2.1341217158156836,122800.0,7.666666666666667,5.666666666666667,4,10|10|10,8|7|8,6|6|5 +xiiotid,0.5,7,27,9,7.405360903230798,9.890694678426321,1.1833344484214374,2.0951833233286665,12215.0,6.666666666666667,2.3333333333333335,2,10|10|10,6|7|7,3|3|1 +xiiotid,0.75,3,27,4,4.668973713188908,5.07283910650696,0.8806570470947873,1.3221152695060867,1903.0,5.0,1.0,1,10|10|10,4|5|6,2|1|0 +5gnidd,0.25,16,27,30,17.600940070872678,27.82808487633464,1.3021194271854393,2.138942553879626,185504.0,6.666666666666667,5.333333333333333,3,10|10|10,6|6|8,6|5|5 +5gnidd,0.5,4,27,4,9.594308791178781,12.717827450589933,1.2251708481787265,1.9640881124634801,88943.0,5.666666666666667,1.3333333333333333,1,10|10|10,6|5|6,2|0|2 +5gnidd,0.75,4,27,5,6.308651536045827,11.355971110240189,0.9162099053779219,1.2651419073123291,7642.5,6.0,1.3333333333333333,1,10|10|10,6|6|6,2|2|0 diff --git a/artifacts/analysis/final__selected_vs_original.csv b/artifacts/analysis/final__selected_vs_original.csv new file mode 100644 index 0000000..abf432b --- /dev/null +++ b/artifacts/analysis/final__selected_vs_original.csv @@ -0,0 +1,7 @@ +dataset,alpha,selected_beta,selected_minus_original_rare_macro_f1,selected_minus_original_rare_recall,selected_minus_original_macro_f1,selected_minus_original_accuracy,selected_minus_original_weighted_f1,descriptive_interpretation +xiiotid,0.25,0.0,-0.000654707802382859,-0.0007083831515385386,-0.0006741622353423349,-0.0014166016708530016,-0.0007698862919814919,numerical/descriptive difference only; not a test of statistical superiority +xiiotid,0.5,0.75,0.022982586416122386,-0.014374000954948563,0.02220443400015426,0.006937663385145676,0.00889883310041717,numerical/descriptive difference only; not a test of statistical superiority +xiiotid,0.75,0.5,0.0013560185452363704,-0.007739443483596897,0.005786987337823679,0.007533373047310832,0.00608024689536979,numerical/descriptive difference only; not a test of statistical superiority +5gnidd,0.25,0.0,-0.0010388235847773597,0.0006605396210892511,-0.0027650156824441385,-0.0007429430689033234,-0.0009720113916064088,numerical/descriptive difference only; not a test of statistical superiority +5gnidd,0.5,1.0,-0.008557880645695959,-0.005809066141944985,-0.003799042805718411,-0.0007182696828510471,-0.0005346795091106804,numerical/descriptive difference only; not a test of statistical superiority +5gnidd,0.75,0.5,0.01683683306978989,-0.0015126750275882284,0.011744866973314672,0.0013186554101197734,0.0014199972720251308,numerical/descriptive difference only; not a test of statistical superiority diff --git a/artifacts/analysis/final__selection_strength.csv b/artifacts/analysis/final__selection_strength.csv new file mode 100644 index 0000000..a985358 --- /dev/null +++ b/artifacts/analysis/final__selection_strength.csv @@ -0,0 +1,7 @@ +dataset,alpha,selected_beta,best_pooled_score,second_best_pooled_score,second_best_beta,absolute_gap,relative_gap,selected_beta_sample_std,min_candidate_sample_std,max_candidate_sample_std,top2_gap_over_selected_std,per_seed_diagnostic_betas,n_seeds_agree_with_pooled,separation_classification +xiiotid,0.25,0.0,0.5917038146634539,0.5886906256417319,0.25,0.0030131890217219848,0.005092394111800361,0.08952883653922933,0.044857849968223555,0.08952883653922933,0.033656072592897816,0.75|0.5|0,1,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std) +xiiotid,0.5,0.75,0.5793667895728949,0.5734790834215032,1.0,0.0058877061513916695,0.010162312126540849,0.07512057331236716,0.07512057331236716,0.1004496749103663,0.07837674676562102,0.5|1|0.75,1,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std) +xiiotid,0.75,0.5,0.47674948263663786,0.47436809609801456,0.25,0.002381386538623298,0.004995047976671449,0.04662425884469754,0.044741855341153436,0.055329314644425844,0.05107612641212262,0.5|0.5|0,2,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std) +5gnidd,0.25,0.0,0.7614534210908004,0.7492858468825278,0.25,0.012167574208272636,0.01597940710653357,0.021902537932118035,0.002755842556750987,0.021902537932118035,0.5555326166302408,0.25|0|0,2,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std) +5gnidd,0.5,1.0,0.6923695590578484,0.6883680316141424,0.75,0.004001527443705966,0.005779467614305719,0.13914040832229815,0.052725844344546816,0.13914040832229815,0.02875891692395368,1|0|1,2,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std) +5gnidd,0.75,0.5,0.8090814343293601,0.8043064619946296,1.0,0.004774972334730454,0.005901720311612864,0.05873543815277214,0.05096339060419691,0.06497400095064636,0.081296275041154,0.5|1|1,1,small numerical separation relative to seed variability (top-2 gap <= selected-beta across-seed sample std) diff --git a/artifacts/analysis/final__six_condition_master.csv b/artifacts/analysis/final__six_condition_master.csv new file mode 100644 index 0000000..084f142 --- /dev/null +++ b/artifacts/analysis/final__six_condition_master.csv @@ -0,0 +1,7 @@ +dataset,alpha,selected_beta,pooled_validation_mean,pooled_validation_std_sample,selected_rare_macro_f1_mean,selected_rare_macro_f1_std,selected_rare_recall_mean,selected_rare_recall_std,selected_macro_f1_mean,selected_macro_f1_std,selected_accuracy_mean,selected_accuracy_std,selected_weighted_f1_mean,selected_weighted_f1_std,original_rare_macro_f1_mean,original_rare_macro_f1_std,original_rare_recall_mean,original_rare_recall_std,original_macro_f1_mean,original_macro_f1_std,original_accuracy_mean,original_accuracy_std,original_weighted_f1_mean,original_weighted_f1_std,cells_sj_lt_M,missing_client_class_cells,client_size_ratio_mean,support_cv_mean,mean_displacement_beta1,beta0_equivalence_class,test_used_for_selection,audit_verdict +xiiotid,0.25,0.0,0.5917038146634539,0.08952883653922933,0.6321355974773142,0.047641778563026725,0.8212568016859243,0.012283281629867224,0.6813433722144538,0.020555132742947534,0.8740145713449322,0.010266545051804108,0.9068823111564912,0.007408425095578143,0.632790305279697,0.04715228886220545,0.8219651848374628,0.010810973971484781,0.6820175344497962,0.02096686007526524,0.8754311730157852,0.009066523301774463,0.9076521974484727,0.0069343939753111275,19,38,14.745270966890343,1.4423832412721973,3.42212533168931,B,False,PASS WITH MINOR CORRECTIONS (Session 7) +xiiotid,0.5,0.75,0.5793667895728949,0.07512057331236716,0.6655025083742895,0.026684712591240056,0.86258272574027,0.013920609948849614,0.7311691156241865,0.006233755140268,0.9063916985504398,0.018450460685614026,0.9336316455838293,0.011230108565482132,0.6425199219581671,0.028683282930463034,0.8769567266952185,0.010770970951613777,0.7089646816240323,0.0011722251041387445,0.8994540351652941,0.008398091032446977,0.9247328124834121,0.006244765764216192,7,9,7.405360903230798,1.1833344484214374,2.0985368999756338,B,False,PASS WITH MINOR CORRECTIONS (Session 6) +xiiotid,0.75,0.5,0.47674948263663786,0.04662425884469754,0.6052456782307091,0.08548886486265879,0.8487646652689419,0.039255535276369505,0.7576359407646712,0.014060416901925105,0.9281095123164508,0.012136684529934048,0.9452163089135119,0.008528481213975155,0.6038896596854727,0.07776445613264914,0.8565041087525388,0.03533794219558259,0.7518489534268475,0.012228464472937675,0.92057613926914,0.012128815595858657,0.9391360620181421,0.008628878095570825,3,4,4.668973713188908,0.8806570470947873,1.305110487896406,B,False,PASS (Session 8) +5gnidd,0.25,0.0,0.7614534210908004,0.021902537932118035,0.8412355814155769,0.024201888695409735,0.9612162422875251,0.03134699156678076,0.8166404182626198,0.01335696520619808,0.730225309136966,0.019524710435296355,0.7044634053674654,0.018175165847202895,0.8422744050003542,0.025370068519978167,0.9605557026664359,0.032423277290702865,0.8194054339450639,0.013918675694370832,0.7309682522058694,0.02143091560670625,0.7054354167590718,0.02036694411653514,16,30,17.600940070872678,1.3021194271854393,2.91943348262683,B,False,PASS (Session 11) +5gnidd,0.5,1.0,0.6923695590578484,0.13914040832229815,0.8719837976555308,0.028500884123071776,0.9671039796439406,0.007942238914573898,0.8385961780131425,0.019627422368850796,0.743256969203502,0.0006668791219600918,0.7181952009373496,0.0016799838366213199,0.8805416783012268,0.0370578200850738,0.9729130457858856,0.0038428107281798614,0.8423952208188609,0.02083962881650294,0.7439752388863531,0.008392660852506539,0.7187298804464602,0.00867754781098361,4,4,9.594308791178781,1.2251708481787265,2.3399083587182736,B,False,PASS (Session 9) +5gnidd,0.75,0.5,0.8090814343293601,0.05873543815277214,0.8967376245747419,0.014470318925337908,0.9597014107775274,0.0010245851497612311,0.8531495688589309,0.009572296127013734,0.7522737210618877,0.0025025090963709863,0.7274674299268966,0.0032168140791904,0.879900791504952,0.023120839413686988,0.9612140858051156,0.002143727670860224,0.8414047018856162,0.014091078328488028,0.750955065651768,0.0035777922288362054,0.7260474326548715,0.004543897028843018,4,5,6.308651536045827,0.9162099053779219,1.3846165631331335,B,False,PASS (Session 10) diff --git a/artifacts/analysis/final__six_condition_master.json b/artifacts/analysis/final__six_condition_master.json new file mode 100644 index 0000000..43adf05 --- /dev/null +++ b/artifacts/analysis/final__six_condition_master.json @@ -0,0 +1,217 @@ +{ + "description": "Authoritative six-condition master table (validation-selected beta; test metrics are post-selection descriptive summaries).", + "source_artifacts": "artifacts/analysis/*__multiseed__*, **partition_heterogeneity**", + "n_conditions": 6, + "rows": [ + { + "dataset": "xiiotid", + "alpha": 0.25, + "selected_beta": 0.0, + "pooled_validation_mean": 0.5917038146634539, + "pooled_validation_std_sample": 0.08952883653922933, + "selected_rare_macro_f1_mean": 0.6321355974773142, + "selected_rare_macro_f1_std": 0.047641778563026725, + "selected_rare_recall_mean": 0.8212568016859243, + "selected_rare_recall_std": 0.012283281629867224, + "selected_macro_f1_mean": 0.6813433722144538, + "selected_macro_f1_std": 0.020555132742947534, + "selected_accuracy_mean": 0.8740145713449322, + "selected_accuracy_std": 0.010266545051804108, + "selected_weighted_f1_mean": 0.9068823111564912, + "selected_weighted_f1_std": 0.007408425095578143, + "original_rare_macro_f1_mean": 0.632790305279697, + "original_rare_macro_f1_std": 0.04715228886220545, + "original_rare_recall_mean": 0.8219651848374628, + "original_rare_recall_std": 0.010810973971484781, + "original_macro_f1_mean": 0.6820175344497962, + "original_macro_f1_std": 0.02096686007526524, + "original_accuracy_mean": 0.8754311730157852, + "original_accuracy_std": 0.009066523301774463, + "original_weighted_f1_mean": 0.9076521974484727, + "original_weighted_f1_std": 0.0069343939753111275, + "cells_sj_lt_M": 19, + "missing_client_class_cells": 38, + "client_size_ratio_mean": 14.745270966890343, + "support_cv_mean": 1.4423832412721973, + "mean_displacement_beta1": 3.42212533168931, + "beta0_equivalence_class": "B", + "test_used_for_selection": false, + "audit_verdict": "PASS WITH MINOR CORRECTIONS (Session 7)" + }, + { + "dataset": "xiiotid", + "alpha": 0.5, + "selected_beta": 0.75, + "pooled_validation_mean": 0.5793667895728949, + "pooled_validation_std_sample": 0.07512057331236716, + "selected_rare_macro_f1_mean": 0.6655025083742895, + "selected_rare_macro_f1_std": 0.026684712591240056, + "selected_rare_recall_mean": 0.86258272574027, + "selected_rare_recall_std": 0.013920609948849614, + "selected_macro_f1_mean": 0.7311691156241865, + "selected_macro_f1_std": 0.006233755140268, + "selected_accuracy_mean": 0.9063916985504398, + "selected_accuracy_std": 0.018450460685614026, + "selected_weighted_f1_mean": 0.9336316455838293, + "selected_weighted_f1_std": 0.011230108565482132, + "original_rare_macro_f1_mean": 0.6425199219581671, + "original_rare_macro_f1_std": 0.028683282930463034, + "original_rare_recall_mean": 0.8769567266952185, + "original_rare_recall_std": 0.010770970951613777, + "original_macro_f1_mean": 0.7089646816240323, + "original_macro_f1_std": 0.0011722251041387445, + "original_accuracy_mean": 0.8994540351652941, + "original_accuracy_std": 0.008398091032446977, + "original_weighted_f1_mean": 0.9247328124834121, + "original_weighted_f1_std": 0.006244765764216192, + "cells_sj_lt_M": 7, + "missing_client_class_cells": 9, + "client_size_ratio_mean": 7.405360903230798, + "support_cv_mean": 1.1833344484214374, + "mean_displacement_beta1": 2.0985368999756338, + "beta0_equivalence_class": "B", + "test_used_for_selection": false, + "audit_verdict": "PASS WITH MINOR CORRECTIONS (Session 6)" + }, + { + "dataset": "xiiotid", + "alpha": 0.75, + "selected_beta": 0.5, + "pooled_validation_mean": 0.47674948263663786, + "pooled_validation_std_sample": 0.04662425884469754, + "selected_rare_macro_f1_mean": 0.6052456782307091, + "selected_rare_macro_f1_std": 0.08548886486265879, + "selected_rare_recall_mean": 0.8487646652689419, + "selected_rare_recall_std": 0.039255535276369505, + "selected_macro_f1_mean": 0.7576359407646712, + "selected_macro_f1_std": 0.014060416901925105, + "selected_accuracy_mean": 0.9281095123164508, + "selected_accuracy_std": 0.012136684529934048, + "selected_weighted_f1_mean": 0.9452163089135119, + "selected_weighted_f1_std": 0.008528481213975155, + "original_rare_macro_f1_mean": 0.6038896596854727, + "original_rare_macro_f1_std": 0.07776445613264914, + "original_rare_recall_mean": 0.8565041087525388, + "original_rare_recall_std": 0.03533794219558259, + "original_macro_f1_mean": 0.7518489534268475, + "original_macro_f1_std": 0.012228464472937675, + "original_accuracy_mean": 0.92057613926914, + "original_accuracy_std": 0.012128815595858657, + "original_weighted_f1_mean": 0.9391360620181421, + "original_weighted_f1_std": 0.008628878095570825, + "cells_sj_lt_M": 3, + "missing_client_class_cells": 4, + "client_size_ratio_mean": 4.668973713188908, + "support_cv_mean": 0.8806570470947873, + "mean_displacement_beta1": 1.305110487896406, + "beta0_equivalence_class": "B", + "test_used_for_selection": false, + "audit_verdict": "PASS (Session 8)" + }, + { + "dataset": "5gnidd", + "alpha": 0.25, + "selected_beta": 0.0, + "pooled_validation_mean": 0.7614534210908004, + "pooled_validation_std_sample": 0.021902537932118035, + "selected_rare_macro_f1_mean": 0.8412355814155769, + "selected_rare_macro_f1_std": 0.024201888695409735, + "selected_rare_recall_mean": 0.9612162422875251, + "selected_rare_recall_std": 0.03134699156678076, + "selected_macro_f1_mean": 0.8166404182626198, + "selected_macro_f1_std": 0.01335696520619808, + "selected_accuracy_mean": 0.730225309136966, + "selected_accuracy_std": 0.019524710435296355, + "selected_weighted_f1_mean": 0.7044634053674654, + "selected_weighted_f1_std": 0.018175165847202895, + "original_rare_macro_f1_mean": 0.8422744050003542, + "original_rare_macro_f1_std": 0.025370068519978167, + "original_rare_recall_mean": 0.9605557026664359, + "original_rare_recall_std": 0.032423277290702865, + "original_macro_f1_mean": 0.8194054339450639, + "original_macro_f1_std": 0.013918675694370832, + "original_accuracy_mean": 0.7309682522058694, + "original_accuracy_std": 0.02143091560670625, + "original_weighted_f1_mean": 0.7054354167590718, + "original_weighted_f1_std": 0.02036694411653514, + "cells_sj_lt_M": 16, + "missing_client_class_cells": 30, + "client_size_ratio_mean": 17.600940070872678, + "support_cv_mean": 1.3021194271854393, + "mean_displacement_beta1": 2.91943348262683, + "beta0_equivalence_class": "B", + "test_used_for_selection": false, + "audit_verdict": "PASS (Session 11)" + }, + { + "dataset": "5gnidd", + "alpha": 0.5, + "selected_beta": 1.0, + "pooled_validation_mean": 0.6923695590578484, + "pooled_validation_std_sample": 0.13914040832229815, + "selected_rare_macro_f1_mean": 0.8719837976555308, + "selected_rare_macro_f1_std": 0.028500884123071776, + "selected_rare_recall_mean": 0.9671039796439406, + "selected_rare_recall_std": 0.007942238914573898, + "selected_macro_f1_mean": 0.8385961780131425, + "selected_macro_f1_std": 0.019627422368850796, + "selected_accuracy_mean": 0.743256969203502, + "selected_accuracy_std": 0.0006668791219600918, + "selected_weighted_f1_mean": 0.7181952009373496, + "selected_weighted_f1_std": 0.0016799838366213199, + "original_rare_macro_f1_mean": 0.8805416783012268, + "original_rare_macro_f1_std": 0.0370578200850738, + "original_rare_recall_mean": 0.9729130457858856, + "original_rare_recall_std": 0.0038428107281798614, + "original_macro_f1_mean": 0.8423952208188609, + "original_macro_f1_std": 0.02083962881650294, + "original_accuracy_mean": 0.7439752388863531, + "original_accuracy_std": 0.008392660852506539, + "original_weighted_f1_mean": 0.7187298804464602, + "original_weighted_f1_std": 0.00867754781098361, + "cells_sj_lt_M": 4, + "missing_client_class_cells": 4, + "client_size_ratio_mean": 9.594308791178781, + "support_cv_mean": 1.2251708481787265, + "mean_displacement_beta1": 2.3399083587182736, + "beta0_equivalence_class": "B", + "test_used_for_selection": false, + "audit_verdict": "PASS (Session 9)" + }, + { + "dataset": "5gnidd", + "alpha": 0.75, + "selected_beta": 0.5, + "pooled_validation_mean": 0.8090814343293601, + "pooled_validation_std_sample": 0.05873543815277214, + "selected_rare_macro_f1_mean": 0.8967376245747419, + "selected_rare_macro_f1_std": 0.014470318925337908, + "selected_rare_recall_mean": 0.9597014107775274, + "selected_rare_recall_std": 0.0010245851497612311, + "selected_macro_f1_mean": 0.8531495688589309, + "selected_macro_f1_std": 0.009572296127013734, + "selected_accuracy_mean": 0.7522737210618877, + "selected_accuracy_std": 0.0025025090963709863, + "selected_weighted_f1_mean": 0.7274674299268966, + "selected_weighted_f1_std": 0.0032168140791904, + "original_rare_macro_f1_mean": 0.879900791504952, + "original_rare_macro_f1_std": 0.023120839413686988, + "original_rare_recall_mean": 0.9612140858051156, + "original_rare_recall_std": 0.002143727670860224, + "original_macro_f1_mean": 0.8414047018856162, + "original_macro_f1_std": 0.014091078328488028, + "original_accuracy_mean": 0.750955065651768, + "original_accuracy_std": 0.0035777922288362054, + "original_weighted_f1_mean": 0.7260474326548715, + "original_weighted_f1_std": 0.004543897028843018, + "cells_sj_lt_M": 4, + "missing_client_class_cells": 5, + "client_size_ratio_mean": 6.308651536045827, + "support_cv_mean": 0.9162099053779219, + "mean_displacement_beta1": 1.3846165631331335, + "beta0_equivalence_class": "B", + "test_used_for_selection": false, + "audit_verdict": "PASS (Session 10)" + } + ] +} \ No newline at end of file diff --git a/artifacts/figures/final/figA_5gnidd_pooled_validation_beta_curve.png b/artifacts/figures/final/figA_5gnidd_pooled_validation_beta_curve.png new file mode 100644 index 0000000..c4e4b9a Binary files /dev/null and b/artifacts/figures/final/figA_5gnidd_pooled_validation_beta_curve.png differ diff --git a/artifacts/figures/final/figA_xiiotid_pooled_validation_beta_curve.png b/artifacts/figures/final/figA_xiiotid_pooled_validation_beta_curve.png new file mode 100644 index 0000000..482f179 Binary files /dev/null and b/artifacts/figures/final/figA_xiiotid_pooled_validation_beta_curve.png differ diff --git a/artifacts/figures/final/figB_selected_beta_by_condition.png b/artifacts/figures/final/figB_selected_beta_by_condition.png new file mode 100644 index 0000000..7da31c5 Binary files /dev/null and b/artifacts/figures/final/figB_selected_beta_by_condition.png differ diff --git a/artifacts/figures/final/figC_realized_support_cv.png b/artifacts/figures/final/figC_realized_support_cv.png new file mode 100644 index 0000000..544bbab Binary files /dev/null and b/artifacts/figures/final/figC_realized_support_cv.png differ diff --git a/artifacts/figures/final/figD_prototype_displacement_beta0_vs_beta1.png b/artifacts/figures/final/figD_prototype_displacement_beta0_vs_beta1.png new file mode 100644 index 0000000..9f8cfc8 Binary files /dev/null and b/artifacts/figures/final/figD_prototype_displacement_beta0_vs_beta1.png differ diff --git a/artifacts/figures/final/figE_support_cv_vs_displacement.png b/artifacts/figures/final/figE_support_cv_vs_displacement.png new file mode 100644 index 0000000..eb14dab Binary files /dev/null and b/artifacts/figures/final/figE_support_cv_vs_displacement.png differ diff --git a/artifacts/figures/final/figF_selected_vs_original_test_rare_macro_f1.png b/artifacts/figures/final/figF_selected_vs_original_test_rare_macro_f1.png new file mode 100644 index 0000000..405a787 Binary files /dev/null and b/artifacts/figures/final/figF_selected_vs_original_test_rare_macro_f1.png differ diff --git a/artifacts/figures/final/figG_client_effect_at_selected_beta.png b/artifacts/figures/final/figG_client_effect_at_selected_beta.png new file mode 100644 index 0000000..324282b Binary files /dev/null and b/artifacts/figures/final/figG_client_effect_at_selected_beta.png differ diff --git a/docs/handoff/SESSION_HANDOFF.md b/docs/handoff/SESSION_HANDOFF.md index 729e922..fc480b3 100644 --- a/docs/handoff/SESSION_HANDOFF.md +++ b/docs/handoff/SESSION_HANDOFF.md @@ -3,30 +3,36 @@ This is the canonical continuation point for the next agent session. It is written to be readable without prior chat history, terminal scrollback, or git history. -Last updated: 2026-09-30 (Session 11). Session 11 type: **final 5G-NIDD beta-validation condition** -(`alpha=0.25`), completing the planned experiment grid. Sessions 6–8 completed the X-IIoTID -three-alpha study (`alpha=0.25/0.50/0.75`, seeds `{0,1,2}`); Session 9 ran 5G-NIDD `alpha=0.50`; -Session 10 ran 5G-NIDD `alpha=0.75`; Session 11 ran **only** 5G-NIDD `alpha=0.25` for seeds -`{0,1,2}` across the five frozen beta candidates plus `original_protean`, using the same -pooled-across-seed validation-only selection rule. **No condition was run after alpha=0.25.** No -commit or push was made. +Last updated: 2026-09-30 (Session 12). Session 12 type: **thesis synthesis of the complete audited +grid** — no training, no partition regeneration, no source/model/aggregation change. Session 11 ran +the final 5G-NIDD condition (`alpha=0.25`), completing the planned grid (108 full runs + 1 smoke). -> **Audit status (Session 11).** Independent audit verdict: **PASS** -> (`docs/research/SESSION11_AUDIT.md`). `beta_star(5gnidd, alpha=0.25) = 0.00` confirmed from -> validation only; run inventory 18/18; pairing hash-identical and reproduced from raw data; -> rare/unseen 0 mismatches over 1,620 cells; leakage audit PASS; `beta=0` class **B**; source/data -> integrity PASS; pytest 75 passed. **Session 11 is safe to checkpoint.** +> **Session 12 scope and status.** Consolidated the 108-run grid into authoritative six-condition +> tables, publication-ready figures, a final RQ1/RQ2/RQ3 synthesis, a Chapter-4 blueprint and a +> claims register, then independently re-verified every number against the audited artifacts. +> **No material scientific gap remains for the defined thesis scope (classification A).** No +> training was executed, no tracked file changed, `planningawal/` and both raw datasets are +> unchanged, `results/` is unchanged (108 non-smoke run dirs), and pytest is **75 passed**. +> Deliverables: `docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md`, +> `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md`, `docs/thesis/CLAIMS_REGISTER.md`, +> `artifacts/analysis/final__*.{csv,json}`, `artifacts/figures/final/figA–G*.png`. -> **Headline (Session 11).** PRIMARY pooled-validation selected beta for 5G-NIDD `alpha=0.25` = -> **0.00** (validation only; per-seed diagnostics 0.25 / 0.00 / 0.00). The pooled validation curve -> declines from 0.761 (beta=0) to 0.722 (beta=0.75) and rises slightly to 0.738 (beta=1.0), with -> the between-beta spread (0.039) exceeding the across-seed std (0.003–0.022). This is the first -> condition where the selected beta is 0, i.e. the equal-`S_j` mean; the selected mode is therefore -> numerically the baseline and shows no support-weighting benefit. `beta=0` is class **B** for all -> three seeds. Realized heterogeneity is the highest of the three 5G-NIDD conditions -> (16/27 `|S_j| rare-class union differs across seeds (6/6/8), so this condition's test rare support is not -> constant. +> **Final RQ findings (Session 12).** (1) Changing beta changes contribution weights and prototype +> positions; for a fixed snapshot displacement is monotone increasing in beta in all six +> conditions, and realized support imbalance and displacement co-vary within each dataset. (2) The +> pooled validation rare-class response is **not monotone** in beta and is weak relative to seed +> variance; the direct aggregation effect and the end-to-end training effect remain distinct. +> (3) Selected betas are condition-dependent and non-monotone in alpha: +> X-IIoTID `{0.00, 0.75, 0.50}` and 5G-NIDD `{0.00, 1.00, 0.50}` for alpha `{0.25, 0.50, 0.75}`; +> both datasets select beta=0 at alpha=0.25 (no weighting benefit) and beta=0.50 at alpha=0.75; +> no universal/global optimum. All six top-two selection gaps are smaller than the selected beta's +> across-seed std. No inferential test was added (3 seeds). + +> **Session 11 audit (preserved).** Verdict **PASS** (`docs/research/SESSION11_AUDIT.md`); +> `beta_star(5gnidd, alpha=0.25) = 0.00` validation-only (per-seed 0.25/0.00/0.00); run inventory +> 18/18; pairing hash-identical; rare/unseen 0 mismatches; leakage PASS; `beta=0` class **B**; +> source/data integrity PASS; pytest 75 passed. Realized heterogeneity highest of the three 5G-NIDD +> conditions (16/27 cells, 30 missing, client ratio 17.60, mean CV 1.302). --- @@ -304,11 +310,13 @@ Six-condition, validation-only pooled selections: ## 15. Git status -No commit or push was made. `HEAD = bbd35085063194db58e4472e70b204d4be4a3607`, branch `main`, in -sync with `origin/main`. Only untracked docs/analysis artifacts exist (no tracked modification); -`results/` is gitignored. `planningawal/` and both raw datasets are unchanged. `equal_available` -absent. No test-based selection. The 18 runs are the only `results/5gnidd__alpha0.25__*` -directories. +No commit or push was made in Session 12. `HEAD = d30e9831e6c17fc6b28383ae4592070ccd83657e`, +branch `main`, in sync with `origin/main`. `git diff` / `git diff --cached` are empty (no tracked +modification); only untracked docs/analysis/figures artifacts exist. `planningawal/` and both raw +datasets are unchanged (SHA-256 match). `src/`, `tests/`, `configs/`, `scripts/`, `pyproject.toml` +are unchanged. `results/` is gitignored and holds exactly 108 non-smoke run directories plus 1 +`smoke`. `equal_available` absent. No test-based selection. `matplotlib 3.11.2` was installed into +the git-ignored `.venv` only; no dependency file was modified. ## 16. Session 11 audit result @@ -321,45 +329,51 @@ directories. - Direct-effect, client-level, cross-alpha, cross-dataset labels: PASS. - Source/data integrity PASS. pytest: **75 passed, 0 failed, 0 skipped**. -## 17. Files created / modified this session +## 17. Files created / modified (Session 12) Created: -- `docs/research/5GNIDD_ALPHA025_BETA_VALIDATION.md` -- `docs/research/5GNIDD_CROSS_ALPHA_ANALYSIS.md` -- `docs/research/CROSS_DATASET_BETA_ANALYSIS.md` -- `docs/research/SESSION11_AUDIT.md` -- `artifacts/analysis/5gnidd__alpha0.25__*` — 18 prototype snapshots + 3 per-seed - effect/weights CSVs + `5gnidd__alpha0.25__multiseed__*` (validation per-seed/pooled, - client detail/summary/deltas, test descriptive, prototype effect pooled, beta0 equivalence, - runtime, selection summary, audit recomputation) + `5gnidd__alpha0.25__partition_characterization.csv` -- `artifacts/analysis/5gnidd__cross_alpha__*` (regenerated to include alpha=0.25) -- `artifacts/analysis/cross_dataset__matched_alpha_summary.{csv,json}`, - `artifacts/analysis/cross_dataset__beta_curve.csv` -- `results/5gnidd__alpha0.25__seed{0,1,2}__*` (18 run dirs, gitignored) +- `docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md` — final six-condition synthesis, RQ1/RQ2/RQ3 + answers, threats to validity, contribution, unsupported claims, gap assessment. +- `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md` — precise Chapter-4 blueprint (purpose / artifact / + key numbers / allowed / avoided per subsection). +- `docs/thesis/CLAIMS_REGISTER.md` — claim / evidence / status / limitation / thesis-safe wording. +- `artifacts/analysis/final__six_condition_master.{csv,json}` +- `artifacts/analysis/final__pooled_validation_curves.csv` (30 rows) +- `artifacts/analysis/final__realized_heterogeneity.csv` +- `artifacts/analysis/final__prototype_displacement_by_beta.csv` +- `artifacts/analysis/final__prototype_displacement_by_class.csv` +- `artifacts/analysis/final__client_effect_summary.csv` (90 rows) +- `artifacts/analysis/final__selected_vs_original.csv` +- `artifacts/analysis/final__selection_strength.csv` +- `artifacts/figures/final/figA–G*.png` — 8 thesis-friendly figures (matplotlib only). Modified: -- `docs/handoff/SESSION_HANDOFF.md` (this file) -- `artifacts/analysis/5gnidd__cross_alpha__*` (extended from 2 to 3 alphas; previous 2-alpha - content was superseded, not silently lost — it is reproducible from the frozen manifests) +- `docs/handoff/SESSION_HANDOFF.md` (this file). -No source or test code was changed. +No source, test, config, model or aggregation file was changed. The synthesis generator is kept +outside the repository at `/tmp/opencode/s12/synthesize_final_analysis.py` (Session-11 convention +for ad-hoc analysis helpers). `matplotlib 3.11.2` was installed into the git-ignored `.venv`. ## 18. Exact recommended next phase -1. **Session 11 is audited and safe to checkpoint** (verdict PASS). Before checkpointing, stage - only the intended docs/analysis artifacts; do not commit `results/` or the raw datasets. -2. **The planned experiment grid is COMPLETE** (two datasets × three alphas × three seeds × five - betas + baseline = 108 runs). Do **not** run any new condition. -3. Remaining work is analysis/writing, not new training: - - optionally add an inferential comparison (e.g. paired sign/permutation test over seeds) to - formalise the weak selection effects — clearly labelled as post-hoc if added; - - freeze the final thesis tables/report from the six-condition artifacts; - - decide and document the `beta=0` vs `original_protean` reporting convention. -4. Do not infer a monotone alpha→beta law; do not claim dataset causality or superiority. +1. **The experiment grid is COMPLETE** (2 datasets × 3 alphas × 3 seeds × (5 betas + baseline) = + 108 runs). Do **not** run any new condition; no new training is required (gap classification A). +2. **Synthesis is complete and verified.** All six selections, selected/original test summaries, + heterogeneity values, displacement summaries and client counts were independently recomputed + from raw artifacts with zero mismatches. +3. Remaining work is **writing and formatting**, not experiments: + - draft Chapter 4 from `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md` using the `final__*` tables + and `figA–G` figures, respecting `docs/thesis/CLAIMS_REGISTER.md`; + - optionally add appendices with the full per-seed and per-class tables; + - if the examiners require inferential support, add a clearly post-hoc, explicitly + underpowered analysis and label it as such (not done by default). +4. Do not infer a monotone alpha→beta law; do not claim dataset causality, superiority, + generalization, or a global optimum. 5. Update `EXPERIMENT_PROTOCOL.md` only if a real ambiguity appears; update this handoff. -6. Do not commit or push unless asked. +6. Do not commit or push unless asked. When checkpointing, stage only intended docs/analysis/ + figure artifacts; never commit `results/` or raw datasets. ## 19. Important commands @@ -394,16 +408,15 @@ No source or test code was changed. ## 20. Recommended next-session prompt ``` -You are continuing the perdalam-protean research repository (Session 12). +You are continuing the perdalam-protean research repository (Session 13). FIRST, read completely: 1. docs/handoff/SESSION_HANDOFF.md - 2. docs/research/SESSION11_AUDIT.md - 3. docs/research/5GNIDD_ALPHA025_BETA_VALIDATION.md - 4. docs/research/5GNIDD_CROSS_ALPHA_ANALYSIS.md + 2. docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md + 3. docs/thesis/CHAPTER4_RESULTS_OUTLINE.md + 4. docs/thesis/CLAIMS_REGISTER.md 5. docs/research/CROSS_DATASET_BETA_ANALYSIS.md - 6. docs/research/XIIOTID_CROSS_ALPHA_ANALYSIS.md - 7. docs/research/EXPERIMENT_PROTOCOL.md + 6. docs/research/EXPERIMENT_PROTOCOL.md Critical rules: - planningawal/ is immutable. @@ -412,14 +425,16 @@ Critical rules: - Do NOT add an executable equal_available mode. - Beta selection uses validation only; PRIMARY is pooled across clients and seeds. - The planned 2-dataset x 3-alpha experiment grid is COMPLETE. Do NOT run new conditions. + - Scientific gap classification is A (NO MATERIAL GAP); no new experiment is required. -Context: Session 11 ran the final condition 5G-NIDD alpha=0.25 (18 runs, ~19.11 h), audited PASS, -and selected beta_star=0.00 (validation only; per-seed 0.25/0.00/0.00). The six-condition pooled -selections are 5G-NIDD 0.25/0.50/0.75 = 0.00/1.00/0.50 and X-IIoTID = 0.00/0.75/0.50; no universal -or monotone beta. beta=0 equivalence class B. pytest 75 passed. +Context: Session 12 completed the thesis synthesis with no training. Six pooled selections: +X-IIoTID 0.25/0.50/0.75 = 0.00/0.75/0.50 and 5G-NIDD = 0.00/1.00/0.50; no universal/monotone beta; +all top-two gaps below selected-beta across-seed std; beta=0 equivalence class B everywhere. +Authoritative artifacts: artifacts/analysis/final__*.{csv,json}; figures artifacts/figures/final/. +All synthesis numbers independently verified; pytest 75 passed; no tracked file changed. -Task: no new training. Consolidate the final thesis-ready six-condition tables/report from the -existing audited artifacts, optionally add a clearly post-hoc inferential comparison of the weak -pooled selections, and keep all limitations (S_j assumption, three seeds, no global optimum) -explicit. Use .venv/bin/python. Do not commit or push unless asked. +Task: write/format thesis chapters from the existing synthesis. Use the final__* tables and +figA-G figures, obey CLAIMS_REGISTER.md wording, keep all limitations explicit, and do not run +training or add inferential tests by default. Use .venv/bin/python. Do not commit or push unless +asked. ``` diff --git a/docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md b/docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md new file mode 100644 index 0000000..8c261bc --- /dev/null +++ b/docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md @@ -0,0 +1,491 @@ +# Final Experimental Synthesis + +Session 12 synthesis of the complete PROTEAN support-weighted prototype aggregation study. +This document is **read-only with respect to experiments**: every number is loaded from the +audited machine-readable artifacts under `artifacts/analysis/`. No training was run, no +partition was regenerated, no source/model/aggregation code was modified, and no beta was +re-selected from test metrics. + +- Repository HEAD at synthesis: `d30e9831e6c17fc6b28383ae4592070ccd83657e` (branch `main`, in sync with `origin/main`). +- Experimental grid: **2 datasets × 3 alpha levels × 3 seeds × (5 betas + original baseline) = 108 full runs** (plus one unrelated `smoke` directory, excluded). +- Tests: `75 passed, 0 failed, 0 skipped`. +- Beta candidates (frozen): `{0.00, 0.25, 0.50, 0.75, 1.00}`. +- Selection rule (frozen): maximize **pooled validation per-client-mean rare-class macro-F1**, validation only; per-seed selection is diagnostic only; test is never used for selection. +- `planningawal/` and both raw datasets unchanged (re-verified). + +Terminology used throughout: + +- **Measured**: directly read or recomputed from an audited artifact. +- **Descriptive interpretation**: a neutral reading of measured values. +- **Unsupported / not established**: a claim that the design cannot defend (causal, universal, superiority). + +--- + +## 1. Scope and protocol + +The thesis modifies **only** the server-side prototype aggregation of PROTEAN (Chennoufi et al., +2025). Each contributing client `i` provides a class-`j` local prototype `C_ij` (local class +embedding mean) and a class-specific training support `N_ij`. The candidate aggregation is + +``` +Cbar_j(beta) = ( sum_{i in S_j} N_ij^beta * C_ij ) / ( sum_{i in S_j} N_ij^beta ) +S_j = { i | N_ij > 0 and a valid local prototype C_ij exists } +``` + +`S_j` is an explicit **reproduction assumption** (no verified official PROTEAN code is +available); `N_ij` is class-specific **sample support, not prototype quality**. The frozen +`original_protean` mode uses the equal-mean reference. `beta=0` is arithmetically the equal mean +over the shared `S_j` but is **not bitwise identical** to the baseline (floating-point +divergence; equivalence class **B** in every condition). + +Protocol (frozen, `docs/research/EXPERIMENT_PROTOCOL.md`): datasets X-IIoTID (`class2`, +benign `Normal`, K=9) and 5G-NIDD (`Attack Type`, benign `Benign`, K=9); 10 clients; symmetric +Dirichlet non-IID with `alpha ∈ {0.25, 0.50, 0.75}`; seeds `{0,1,2}`; stratified train/validation/test += 70/10/20 (validation is a project decision; the paper uses 80/20); 10 rounds; 3 local epochs; +batch 256; Adam; `lr=1e-3`, `mu=0.1`, `lambda=1.0` (project decisions). All compared configurations +within a `(dataset, alpha, seed)` reuse exactly the same manifest, split and initialization +(verified by content and partition hashes). + +--- + +## 2. Completed experimental grid + +| dataset | alpha | seeds | betas + baseline | runs | audit | +|---|---|---|---|---|---| +| X-IIoTID | 0.25 | 0,1,2 | 5 + 1 | 18 | PASS WITH MINOR CORRECTIONS (Session 7) | +| X-IIoTID | 0.50 | 0,1,2 | 5 + 1 | 18 | PASS WITH MINOR CORRECTIONS (Session 6) | +| X-IIoTID | 0.75 | 0,1,2 | 5 + 1 | 18 | PASS (Session 8) | +| 5G-NIDD | 0.25 | 0,1,2 | 5 + 1 | 18 | PASS (Session 11) | +| 5G-NIDD | 0.50 | 0,1,2 | 5 + 1 | 18 | PASS (Session 9) | +| 5G-NIDD | 0.75 | 0,1,2 | 5 + 1 | 18 | PASS (Session 10) | +| **total** | | | | **108** | | + +--- + +## 3. Six-condition master result + +Authoritative table: `artifacts/analysis/final__six_condition_master.csv` / `.json`. + +| dataset | alpha | beta* | pooled val (std) | sel test rare-F1 | orig test rare-F1 | sel rare-recall | orig rare-recall | sel macro-F1 | sel acc | sel wF1 | +|---|---|---|---|---|---|---|---|---|---|---| +| X-IIoTID | 0.25 | **0.00** | 0.5917 (0.0895) | 0.6321 | 0.6328 | 0.8213 | 0.8220 | 0.6813 | 0.8740 | 0.9069 | +| X-IIoTID | 0.50 | **0.75** | 0.5794 (0.0751) | 0.6655 | 0.6425 | 0.8626 | 0.8770 | 0.7312 | 0.9064 | 0.9336 | +| X-IIoTID | 0.75 | **0.50** | 0.4767 (0.0466) | 0.6052 | 0.6039 | 0.8488 | 0.8565 | 0.7576 | 0.9281 | 0.9452 | +| 5G-NIDD | 0.25 | **0.00** | 0.7615 (0.0219) | 0.8412 | 0.8423 | 0.9612 | 0.9606 | 0.8166 | 0.7302 | 0.7045 | +| 5G-NIDD | 0.50 | **1.00** | 0.6924 (0.1391) | 0.8720 | 0.8805 | 0.9671 | 0.9729 | 0.8386 | 0.7433 | 0.7182 | +| 5G-NIDD | 0.75 | **0.50** | 0.8091 (0.0587) | 0.8967 | 0.8799 | 0.9597 | 0.9612 | 0.8531 | 0.7523 | 0.7275 | + +All selected betas are validation-only; `test_used_for_selection = false` in every run; +`beta0_equivalence_class = B` in every condition. The selected beta takes values +`{0.00, 0.50, 1.00}` across the six conditions: **not universal, not monotone, not a function of +alpha alone**. + +Selected-beta test metrics are **post-selection descriptive summaries**, not selection inputs. + +--- + +## 4. Realized non-IID characteristics + +Dirichlet `alpha` is *not* a realized-heterogeneity measure. Authoritative realized table: +`artifacts/analysis/final__realized_heterogeneity.csv`. + +| dataset | alpha | `|S_j| **RQ1.** How are class-support imbalance, differences between local prototypes, and rare-class +> detection performance related in PROTEAN? + +Evidence chain (all three alphas, both datasets): + +*Mechanically established.* +- Changing beta changes the normalized contribution weights `N_ij^beta / Σ_k N_kj^beta` and + therefore the position of the global prototype `Cbar_j`. For a fixed snapshot, mean + displacement grows monotonically with beta in all six conditions. + +*Empirically observed (descriptive).* +- Larger realized support imbalance (higher support CV, more missing cells, higher client-size + ratio) tends to accompany larger beta=1 displacement, within both datasets: + X-IIoTID (CV, disp) = (1.442, 3.422) → (1.183, 2.099) → (0.881, 1.305); + 5G-NIDD = (1.302, 2.919) → (1.225, 2.340) → (0.916, 1.385). +- Imbalance also co-varies with selection behavior: both alpha=0.25 conditions are the most + imbalanced and both selected beta=0. + +*Not established.* +- Larger displacement does **not** automatically imply better rare-class detection: at beta=1 + (maximal displacement) rare-class performance is worse than at the selected beta in most + conditions, and validation response is non-monotone in beta. +- The imbalance → displacement link is mechanical under the beta formula; the link from + displacement to detection is confounded by end-to-end retraining and cannot be isolated with + this design. + +RQ1 can therefore be answered **descriptively**: support imbalance and prototype displacement +co-vary, but displacement is not a proxy for rare-class detection quality. + +--- + +## 10. RQ2 answer + +> **RQ2.** How does changing beta affect the global prototype and rare-class detection +> performance under different non-IID conditions? + +*Direct aggregation effect (fixed snapshot, no retraining).* +- Displacement from beta=0 is monotone increasing in beta in every condition; magnitude differs + by condition (largest at alpha=0.25). +- This is a pure weighting/geometry effect and is measured independently of training. + +*End-to-end training effect (iterative).* +- Pooled validation rare-class macro-F1 is **not monotone** in beta. Preference varies by + condition: high beta is preferred at 5G-NIDD alpha=0.50 (beta=1.0), moderate beta at both + alpha=0.75 conditions (beta=0.5), and the smallest beta at both alpha=0.25 conditions + (beta=0.0). +- Weighting helps some clients and hurts others within the same condition (section 8), so the + condition-level preference is an average over heterogeneous client responses. + +The two effects must not be merged: the geometry changes monotonically, the task-level response +does not. + +--- + +## 11. RQ3 answer + +> **RQ3.** Does the validation-selected beta change across non-IID levels, and what is the +> effect on overall and per-client performance? + +Authoritative selected betas (validation only): + +| dataset | alpha=0.25 | alpha=0.50 | alpha=0.75 | +|---|---|---|---| +| X-IIoTID | 0.00 | 0.75 | 0.50 | +| 5G-NIDD | 0.00 | 1.00 | 0.50 | + +Established conclusions: + +- The selected beta **changes by condition**: it is not a single universal value and not a + monotone function of alpha. +- Both datasets select beta=0 at alpha=0.25 and beta=0.50 at alpha=0.75; they differ at + alpha=0.50 (X-IIoTID 0.75 vs 5G-NIDD 1.00). +- Only five discrete candidates were tested; no global optimum is claimed, and each value is + "best among the tested candidates under pooled validation for that condition". + +Overall effect (section 6): selected-beta test rare-F1 differences are small and positive in +three conditions, small and negative in three; the largest positive difference is smaller than +between-seed variability. Per-client effect (section 8): the selected beta improves the +validation rare-class metric for a majority of clients in four conditions, but at both alpha=0.25 +conditions (where the selected beta is 0) there is trivially no weighting change, and at higher +beta the effect is client-dependent. + +--- + +## 12. Cross-dataset synthesis + +- **Agreement**: matched-alpha selections coincide at alpha=0.25 (both 0.00) and alpha=0.75 + (both 0.50); they differ at alpha=0.50 (0.75 vs 1.00). +- **Displacement ordering is consistent**: beta=1 displacement falls with alpha in both datasets, + and ranking of conditions by displacement mirrors ranking by realized support CV. +- **Rare-class objects are not comparable across datasets**: 5G-NIDD rare classes are low-volume + flood/scan types; X-IIoTID rare classes are low-volume behavioural categories. Absolute metric + levels must not be compared across datasets; only within-condition differences are meaningful. +- **Metric asymmetry**: 5G-NIDD weighted-F1 < accuracy; X-IIoTID weighted-F1 > accuracy. +- No cross-dataset causality, no dataset superiority, no dataset-independent beta rule. + +--- + +## 13. Selection uncertainty + +Table: `artifacts/analysis/final__selection_strength.csv`. + +| dataset | alpha | beta* | best | 2nd best | abs gap | rel gap | top-2 gap / selected std | per-seed betas | +|---|---|---|---|---|---|---|---|---| +| X-IIoTID | 0.25 | 0.00 | 0.5917 | 0.5887 (0.25) | 0.0030 | 0.51% | 0.034 | 0.75 / 0.5 / 0.0 | +| X-IIoTID | 0.50 | 0.75 | 0.5794 | 0.5735 (1.0) | 0.0059 | 1.02% | 0.078 | 0.5 / 1.0 / 0.75 | +| X-IIoTID | 0.75 | 0.50 | 0.4767 | 0.4744 (0.25) | 0.0024 | 0.50% | 0.051 | 0.5 / 0.5 / 0.0 | +| 5G-NIDD | 0.25 | 0.00 | 0.7615 | 0.7493 (0.25) | 0.0122 | 1.60% | 0.556 | 0.25 / 0.0 / 0.0 | +| 5G-NIDD | 0.50 | 1.00 | 0.6924 | 0.6884 (0.75) | 0.0040 | 0.58% | 0.029 | 1.0 / 0.0 / 1.0 | +| 5G-NIDD | 0.75 | 0.50 | 0.8091 | 0.8043 (1.0) | 0.0048 | 0.59% | 0.081 | 0.5 / 1.0 / 1.0 | + +Quantitative basis for the (descriptive) classification: the top-two gap is compared with the +selected beta's across-seed sample std. **In all six conditions the top-two gap is smaller than +the selected-beta sample std** (ratios 0.03 – 0.56), so every selection is classified as +*small numerical separation relative to seed variability*. No tie-break was ever exercised. The +per-seed diagnostic beta agrees with the pooled selection in only 1–2 of 3 seeds per condition. + +No inferential test (t-test, ANOVA, Wilcoxon, Friedman, regression) is applied. With three +project-defined seeds per condition, such tests would be underpowered and would overstate the +evidence; mean, sample std, per-seed values and the top-two gap are reported instead. + +--- + +## 14. Threats to validity + +### Internal validity +- **`S_j` reproduction assumption**: the participant set / denominator rule is not verified + against official code and is materially exercised in up to 19/27 cells. All results are + conditional on it. +- **beta=0 floating-point divergence**: beta=0 is semantically the equal mean but not bitwise + identical to `original_protean`; selected-vs-original differences at alpha=0.25 are this + divergence, not an effect. Equivalence class B everywhere. +- **Frozen implementation**: aggregation and training code unchanged across all 108 runs; + the same manifest/split/init is reused within each `(dataset, alpha, seed)`. +- **Paired partitions/seeds**: verified by content, partition and split hashes. +- **Validation/test separation**: beta selected from validation only; `test_used_for_selection` + is false in all runs; test read only after selection. + +### Construct validity +- `N_ij` is **sample support, not prototype quality**; a larger `N_ij` does not mean a better + prototype. +- Rare-class definition is operational (two smallest positive-support attack classes per client + from the training split); rare and unseen are never merged. +- Prototype displacement is a **representation** metric, not detection quality. + +### External validity +- Only two IDS datasets (X-IIoTID, 5G-NIDD), 10 clients, three alpha levels, three + project-defined seeds, one PROTEAN architecture and training configuration. +- Rare-class compositions are dataset-specific and not comparable across datasets. + +### Conclusion validity +- Small seed count (3); between-beta spreads are comparable to across-seed std. +- Discrete beta grid `{0,0.25,0.5,0.75,1.0}`; the true response between candidates is unobserved. +- Weak numerical separation at all six conditions (section 13). +- No inferential significance testing was performed or is claimed. + +--- + +## 15. Contribution supported by evidence + +1. A **controlled, beta-parameterized support-weighting** operator applied to the server-side + prototype aggregation of PROTEAN, with an explicit participant rule and missing-class + semantics. +2. **Condition-wise, validation-based beta selection** (no test leakage) across six paired + conditions. +3. A **paired multi-seed analysis** across two IDS datasets and three non-IID levels, with + identical partitions within each comparison cell. +4. A **direct linkage** between realized support imbalance, weight redistribution, prototype + displacement, rare-class detection and per-client consequences — separated into direct + aggregation effects (fixed snapshots) and end-to-end training effects. +5. Evidence of **when stronger support weighting helps, has little effect, or degrades** + rare-class behavior, including the identification that both evaluated alpha=0.25 conditions + select beta=0 (no weighting benefit) and that response is not monotone in beta. + +**Novelty caveat.** Sample-count prototype weighting is already discussed in related work, e.g. +**FedProto** (Tan et al., 2022). The contribution cannot rest on using `N_ij` or adding beta +alone; it rests on the controlled parameterization, the condition-wise validation selection, the +paired multi-seed setup, and the resulting behavioral findings. + +--- + +## 16. Claims NOT supported by evidence + +The following must not appear as established results: + +- "Support weighting improves rare-class detection." (Mixed and condition-dependent; largest + positive difference is within seed noise.) +- "Selected-beta beats original PROTEAN." (Only small descriptive differences; no significance + test; at alpha=0.25 the difference is floating-point divergence.) +- "alpha causally determines the optimal beta." (Only two datasets, three seeds; selection is + not monotone in alpha.) +- "A larger `N_ij` means a better prototype / greater prototype quality." +- "Larger prototype displacement means better (or worse) rare-class detection." +- "beta=0 reproduces original PROTEAN" (only semantically equivalent under the shared `S_j` + assumption; class B divergence). +- "There is a universal / globally optimal beta." +- "The method generalizes" or "dataset X is superior". +- Any metric interpreted as improvement without the qualifier "numerical/descriptive". + +--- + +## 17. Recommended thesis conclusions + +1. Under the validated design, support weighting is **condition-dependent**: the best tested beta + differs across non-IID conditions, is not monotone in alpha, and no single beta dominates. +2. Changing beta **monotonically displaces** global prototypes at a fixed snapshot, while the + rare-class detection response is **non-monotone** and weak relative to seed variability. +3. Realized support imbalance and prototype displacement **co-vary**; displacement alone is not + evidence of detection quality. +4. Weighting **redistributes benefit across clients**; within a condition some clients gain and + others lose. +5. In both evaluated datasets the most imbalanced condition (alpha=0.25) selected beta=0, i.e. + equal weighting over the shared participant set, and realized the largest prototype + displacement — a descriptive pattern that motivates broader validation, not a rule. + +--- + +## 18. Scientific-gap assessment + +**Classification: A — NO MATERIAL GAP** for the defined thesis scope. + +- RQ1, RQ2 and RQ3 are all answerable descriptively from the 108-run grid and the existing + audited artifacts. +- The remaining limitations (S_j assumption, three seeds, discrete grid, two datasets, no + inferential power) are **scope/validity** boundaries, not missing experiments that prevent the + stated research questions from being answered. +- A **causal** isolation of the displacement→detection link (e.g. evaluating recomputed global + prototypes on trained embeddings at a fixed snapshot, or interpolating beta in prototype space) + would strengthen RQ1/RQ2, but it is not required by the defined RQs and would change the design; + it is therefore out of scope rather than a blocking gap. +- No new training is required or recommended. + +--- + +## Artifact index (Session 12 deliverables) + +| artifact | path | +|---|---| +| master table | `artifacts/analysis/final__six_condition_master.{csv,json}` | +| pooled validation curves | `artifacts/analysis/final__pooled_validation_curves.csv` | +| realized heterogeneity | `artifacts/analysis/final__realized_heterogeneity.csv` | +| displacement by beta | `artifacts/analysis/final__prototype_displacement_by_beta.csv` | +| displacement by class (beta=1) | `artifacts/analysis/final__prototype_displacement_by_class.csv` | +| client-level effect | `artifacts/analysis/final__client_effect_summary.csv` | +| selected vs original | `artifacts/analysis/final__selected_vs_original.csv` | +| selection strength | `artifacts/analysis/final__selection_strength.csv` | +| figures | `artifacts/figures/final/figA–G*.png` | +| Chapter 4 outline | `docs/thesis/CHAPTER4_RESULTS_OUTLINE.md` | +| claims register | `docs/thesis/CLAIMS_REGISTER.md` | + +**Environment note.** Figures require `matplotlib`, which is not in the frozen `research` +dependency set. It was installed into the git-ignored `.venv` only +(`uv pip install --python .venv/bin/python matplotlib`, matplotlib 3.11.2); no tracked +dependency file was modified. The synthesis generator is `scripts`-style analysis code kept +outside the repository at `/tmp/opencode/s12/synthesize_final_analysis.py`, consistent with the +Session-11 convention for ad-hoc analysis helpers. diff --git a/docs/thesis/CHAPTER4_RESULTS_OUTLINE.md b/docs/thesis/CHAPTER4_RESULTS_OUTLINE.md new file mode 100644 index 0000000..aff9b70 --- /dev/null +++ b/docs/thesis/CHAPTER4_RESULTS_OUTLINE.md @@ -0,0 +1,148 @@ +# Chapter 4 — Results and Discussion (Blueprint) + +This is a **blueprint**, not a finished chapter. Each subsection states its purpose, the exact +table/figure to use, the key numbers to quote (loaded from audited artifacts), what may be +interpreted, and what must not be interpreted. All numbers trace to +`docs/research/FINAL_EXPERIMENTAL_SYNTHESIS.md` and the `artifacts/analysis/final__*` files. + +Global wording rules: prefer "numerical/descriptive difference", "best among the tested +candidates under pooled validation", "co-varies"; never "improvement", "optimal", "causal", +"superior", or "generalizes" unless explicitly qualified. + +--- + +## 4.1 Experimental setup recap + +- **Purpose**: fix the protocol so later numbers are reproducible and comparable. +- **Use**: text; table of grid (Section 2 of synthesis); `docs/research/EXPERIMENT_PROTOCOL.md`. +- **Key numbers**: 2 datasets × 3 alpha × 3 seeds × (5 betas + `original_protean`) = **108 runs**; + betas `{0, 0.25, 0.5, 0.75, 1}`; 10 clients; 10 rounds; 3 local epochs; batch 256; Adam lr=1e-3, + mu=0.1, lambda=1.0; train/val/test 70/10/20; validation-only selection. +- **Allowed**: state that `S_j` is a reproduction assumption; state that beta=0 is not bitwise + identical to the baseline. +- **Avoid**: describing the design as new training beyond the frozen grid; implying the paper's + split was 70/10/20. + +## 4.2 Realized non-IID characteristics + +- **Purpose**: show that alpha is a design knob and realized heterogeneity is the measured + variable. +- **Use**: Table `final__realized_heterogeneity.csv`; Figure `figC_realized_support_cv.png`. +- **Key numbers**: support CV 1.442/1.183/0.881 (X-IIoTID) and 1.302/1.225/0.916 (5G-NIDD) for + alpha 0.25/0.50/0.75; `|S_j|0`, two smallest positive-support attack classes) and + unseen (`N_ij=0`) as distinct categories from the training split. +- **Limitation**: none; definitional. +- **Safe wording**: "Rare-class and unseen-class results are reported separately and are never + merged into one metric." + +## 14. Results generalize beyond the tested setting + +- **Status**: unsupported. +- **Evidence**: two datasets, 10 clients, three alpha levels, three project-defined seeds, one + architecture/training configuration. +- **Limitation**: external validity. +- **Safe wording**: "Findings are specific to the two evaluated IDS datasets and the frozen + PROTEAN configuration; no generalization is claimed."