diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_deltas.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_deltas.csv new file mode 100644 index 0000000..f55d694 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_deltas.csv @@ -0,0 +1,151 @@ +seed,beta,client_id,n_samples,val_rare_macro_f1,val_rare_recall +0,0.0,0,3354,0.8875379939209728,0.9864864864864865 +0,0.0,1,5840,0.43045783132530124,0.6944444444444444 +0,0.0,2,6116,0.7407407407407407,1.0 +0,0.0,3,9307,0.7126436781609196,0.6888888888888889 +0,0.0,4,7750,0.8619718309859155,0.9807692307692307 +0,0.0,5,8396,0.28180563997182917,0.9848484848484849 +0,0.0,6,8194,0.5320018964086761,1.0 +0,0.0,7,2749,0.29411764705882354,0.9444444444444444 +0,0.0,8,17172,0.5012611838161273,0.9813953488372094 +0,0.0,9,12536,0.3308578745198464,0.641025641025641 +0,0.25,0,3354,0.8971962616822431,0.972972972972973 +0,0.25,1,5840,0.4363790334821964,0.7013888888888888 +0,0.25,2,6116,0.8,1.0 +0,0.25,3,9307,0.7272727272727273,0.7111111111111111 +0,0.25,4,7750,0.8571428571428572,0.9807692307692307 +0,0.25,5,8396,0.3131615278416935,0.9898989898989898 +0,0.25,6,8194,0.5400683913172762,0.9642857142857143 +0,0.25,7,2749,0.29858525688756515,0.9444444444444444 +0,0.25,8,17172,0.5063361579319026,0.9837209302325581 +0,0.25,9,12536,0.3417318982387475,0.6688034188034189 +0,0.5,0,3354,0.9306930693069307,0.9527027027027027 +0,0.5,1,5840,0.4413828325877919,0.7124999999999999 +0,0.5,2,6116,0.8333333333333333,1.0 +0,0.5,3,9307,0.7555555555555555,0.7555555555555555 +0,0.5,4,7750,0.8660968660968661,0.9743589743589743 +0,0.5,5,8396,0.3727944926058133,0.9898989898989898 +0,0.5,6,8194,0.536609144974164,0.9642857142857143 +0,0.5,7,2749,0.2654713707345286,0.9444444444444444 +0,0.5,8,17172,0.5000157678965627,0.9767441860465116 +0,0.5,9,12536,0.35566502463054184,0.6965811965811965 +0,0.75,0,3354,0.9166666666666667,0.8918918918918919 +0,0.75,1,5840,0.4390983440522611,0.711111111111111 +0,0.75,2,6116,0.8333333333333333,1.0 +0,0.75,3,9307,0.7415730337078651,0.7333333333333333 +0,0.75,4,7750,0.8411764705882353,0.9166666666666666 +0,0.75,5,8396,0.37074068149096145,0.9898989898989898 +0,0.75,6,8194,0.5318530266983875,0.9642857142857143 +0,0.75,7,2749,0.25831969448990727,0.9444444444444444 +0,0.75,8,17172,0.5019948230864674,0.9304909560723513 +0,0.75,9,12536,0.34527541424093144,0.6965811965811965 +0,1.0,0,3354,0.9077490774907748,0.831081081081081 +0,1.0,1,5840,0.4280463143311497,0.6916666666666667 +0,1.0,2,6116,0.7692307692307693,1.0 +0,1.0,3,9307,0.7272727272727273,0.7111111111111111 +0,1.0,4,7750,0.7859237536656892,0.8589743589743589 +0,1.0,5,8396,0.32319994363815696,0.9848484848484849 +0,1.0,6,8194,0.5186591560629684,0.9285714285714286 +0,1.0,7,2749,0.2510204081632653,0.9444444444444444 +0,1.0,8,17172,0.5004931186519341,0.9304909560723513 +0,1.0,9,12536,0.3028365305787834,0.6688034188034189 +1,0.0,0,9771,0.555984555984556,0.972972972972973 +1,0.0,1,11934,0.5795225653221874,0.9976043805612593 +1,0.0,2,3021,0.42962962962962964,0.8382352941176471 +1,0.0,3,4707,0.27739361702127663,0.8181818181818181 +1,0.0,4,3335,0.6538461538461537,0.5483870967741935 +1,0.0,5,15714,0.5279805352798054,0.7439024390243902 +1,0.0,6,8732,0.47346514047866806,0.49242424242424243 +1,0.0,7,1467,1.0,1.0 +1,0.0,8,17580,0.8076923076923077,1.0 +1,0.0,9,5153,0.9807296026788589,0.9850571725571726 +1,0.25,0,9771,0.5658914728682171,0.9864864864864865 +1,0.25,1,11934,0.6023883135587408,0.9973990417522245 +1,0.25,2,3021,0.44103313840155944,0.8382352941176471 +1,0.25,3,4707,0.28566573008247986,0.8181818181818181 +1,0.25,4,3335,0.6538461538461537,0.5483870967741935 +1,0.25,5,15714,0.5335335335335335,0.7439024390243902 +1,0.25,6,8732,0.46900826446280997,0.49134199134199136 +1,0.25,7,1467,1.0,1.0 +1,0.25,8,17580,0.8316831683168318,1.0 +1,0.25,9,5153,0.9789997415352805,0.9816787941787942 +1,0.5,0,9771,0.5983606557377049,0.9864864864864865 +1,0.5,1,11934,0.5950263055479481,0.9395013952508819 +1,0.5,2,3021,0.4659111350962404,0.8382352941176471 +1,0.5,3,4707,0.3038850776042427,0.8181818181818181 +1,0.5,4,3335,0.679245283018868,0.5806451612903226 +1,0.5,5,15714,0.5448554042536322,0.7439024390243902 +1,0.5,6,8732,0.4680412371134021,0.49134199134199136 +1,0.5,7,1467,1.0,1.0 +1,0.5,8,17580,0.8199999999999998,0.9761904761904762 +1,0.5,9,5153,0.9765630072063884,0.9768711018711018 +1,0.75,0,9771,0.6431718061674009,0.9864864864864865 +1,0.75,1,11934,0.5892614182467265,0.900697625440952 +1,0.75,2,3021,0.47251332965618675,0.8382352941176471 +1,0.75,3,4707,0.31549441865059763,0.8181818181818181 +1,0.75,4,3335,0.679245283018868,0.5806451612903226 +1,0.75,5,15714,0.5516592310652149,0.7439024390243902 +1,0.75,6,8732,0.47046632124352333,0.49134199134199136 +1,0.75,7,1467,1.0,1.0 +1,0.75,8,17580,0.8484848484848484,1.0 +1,0.75,9,5153,0.9813023436992794,0.9816787941787942 +1,1.0,0,9771,0.7487179487179488,0.9864864864864865 +1,1.0,1,11934,0.5900895998512916,0.900697625440952 +1,1.0,2,3021,0.5188275566966196,0.8382352941176471 +1,1.0,3,4707,0.3760424007517031,0.8181818181818181 +1,1.0,4,3335,0.7037037037037037,0.6129032258064516 +1,1.0,5,15714,0.5441986886103379,0.7439024390243902 +1,1.0,6,8732,0.4794086589229145,0.49134199134199136 +1,1.0,7,1467,1.0,1.0 +1,1.0,8,17580,0.8842105263157896,1.0 +1,1.0,9,5153,0.9859736514386839,0.9816787941787942 +2,0.0,0,8416,0.09876543209876543,0.3076923076923077 +2,0.0,1,4533,0.5263157894736842,0.625 +2,0.0,2,7382,0.2881288368014032,0.8690476190476191 +2,0.0,3,12132,0.25392765801972955,0.75 +2,0.0,4,3509,0.9690721649484536,0.94 +2,0.0,5,9143,0.4811991415541446,0.8555825242718447 +2,0.0,6,4387,0.9733738458483634,0.9900486434639627 +2,0.0,7,7862,0.4408369408369409,0.9318181818181819 +2,0.0,8,20564,0.046121316241575255,0.8157894736842105 +2,0.0,9,3486,0.3934390771449171,0.7885802469135803 +2,0.25,0,8416,0.1176470588235294,0.3076923076923077 +2,0.25,1,4533,0.5263157894736842,0.625 +2,0.25,2,7382,0.3118580765639589,0.8690476190476191 +2,0.25,3,12132,0.28338171262699563,0.8125 +2,0.25,4,3509,0.9690721649484536,0.94 +2,0.25,5,9143,0.4804878048780488,0.8410194174757282 +2,0.25,6,4387,0.9859407908145934,0.9900486434639627 +2,0.25,7,7862,0.44609149455918273,0.9318181818181819 +2,0.25,8,20564,0.06176250933532487,0.8157894736842105 +2,0.25,9,3486,0.40053768640117604,0.7824074074074074 +2,0.5,0,8416,0.14545454545454545,0.3076923076923077 +2,0.5,1,4533,0.5263157894736842,0.625 +2,0.5,2,7382,0.35779327972735503,0.8690476190476191 +2,0.5,3,12132,0.27168831168831165,0.8125 +2,0.5,4,3509,0.9690721649484536,0.94 +2,0.5,5,9143,0.48409645995339823,0.8264563106796117 +2,0.5,6,4387,0.9886441490819402,0.9886441490819402 +2,0.5,7,7862,0.468078381795196,0.8863636363636364 +2,0.5,8,20564,0.0837908496732026,0.8157894736842105 +2,0.5,9,3486,0.4152159896840748,0.7762345679012346 +2,0.75,0,8416,0.27586206896551724,0.3076923076923077 +2,0.75,1,4533,0.5555555555555556,0.625 +2,0.75,2,7382,0.38409090909090915,0.8571428571428572 +2,0.75,3,12132,0.25207044264635886,0.78125 +2,0.75,4,3509,0.9690721649484536,0.94 +2,0.75,5,9143,0.4842105263157895,0.816747572815534 +2,0.75,6,4387,0.9878070464559553,0.9816216771718279 +2,0.75,7,7862,0.4953242835595777,0.8636363636363636 +2,0.75,8,20564,0.20940438871473352,0.8157894736842105 +2,0.75,9,3486,0.4359758123463353,0.7638888888888888 +2,1.0,0,8416,0.34782608695652173,0.3076923076923077 +2,1.0,1,4533,0.5263157894736842,0.625 +2,1.0,2,7382,0.399138268703486,0.8452380952380952 +2,1.0,3,12132,0.25128900949796473,0.78125 +2,1.0,4,3509,0.5662650602409639,0.94 +2,1.0,5,9143,0.4798898963730569,0.8070388349514563 +2,1.0,6,4387,0.974498660479089,0.9530236594500776 +2,1.0,7,7862,0.5532377227292482,0.8636363636363636 +2,1.0,8,20564,0.3136863136863137,0.8157894736842105 +2,1.0,9,3486,0.44662116040955635,0.7608024691358024 diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_detail.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_detail.csv new file mode 100644 index 0000000..b021b2b --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_detail.csv @@ -0,0 +1,151 @@ +seed,beta,client_id,min_positive_attack_Nij,val_rare_macro_f1,val_rare_recall,delta_vs_beta0 +0,0.0,0,18,0.8875379939209728,0.9864864864864865,0.0 +0,0.0,1,69,0.43045783132530124,0.6944444444444444,0.0 +0,0.0,2,5,0.7407407407407407,1.0,0.0 +0,0.0,3,140,0.7126436781609196,0.6888888888888889,0.0 +0,0.0,4,1,0.8619718309859155,0.9807692307692307,0.0 +0,0.0,5,2,0.28180563997182917,0.9848484848484849,0.0 +0,0.0,6,44,0.5320018964086761,1.0,0.0 +0,0.0,7,10,0.29411764705882354,0.9444444444444444,0.0 +0,0.0,8,4,0.5012611838161273,0.9813953488372094,0.0 +0,0.0,9,12,0.3308578745198464,0.641025641025641,0.0 +0,0.25,0,18,0.8971962616822431,0.972972972972973,0.009658267761270323 +0,0.25,1,69,0.4363790334821964,0.7013888888888888,0.005921202156895178 +0,0.25,2,5,0.8,1.0,0.059259259259259345 +0,0.25,3,140,0.7272727272727273,0.7111111111111111,0.014629049111807735 +0,0.25,4,1,0.8571428571428572,0.9807692307692307,-0.0048289738430582485 +0,0.25,5,2,0.3131615278416935,0.9898989898989898,0.03135588786986432 +0,0.25,6,44,0.5400683913172762,0.9642857142857143,0.00806649490860012 +0,0.25,7,10,0.29858525688756515,0.9444444444444444,0.004467609828741614 +0,0.25,8,4,0.5063361579319026,0.9837209302325581,0.005074974115775288 +0,0.25,9,12,0.3417318982387475,0.6688034188034189,0.010874023718901116 +0,0.5,0,18,0.9306930693069307,0.9527027027027027,0.043155075385957975 +0,0.5,1,69,0.4413828325877919,0.7124999999999999,0.01092500126249063 +0,0.5,2,5,0.8333333333333333,1.0,0.09259259259259256 +0,0.5,3,140,0.7555555555555555,0.7555555555555555,0.04291187739463598 +0,0.5,4,1,0.8660968660968661,0.9743589743589743,0.004125035110950637 +0,0.5,5,2,0.3727944926058133,0.9898989898989898,0.09098885263398415 +0,0.5,6,44,0.536609144974164,0.9642857142857143,0.004607248565487887 +0,0.5,7,10,0.2654713707345286,0.9444444444444444,-0.02864627632429495 +0,0.5,8,4,0.5000157678965627,0.9767441860465116,-0.0012454159195646186 +0,0.5,9,12,0.35566502463054184,0.6965811965811965,0.024807150110695453 +0,0.75,0,18,0.9166666666666667,0.8918918918918919,0.029128672745693973 +0,0.75,1,69,0.4390983440522611,0.711111111111111,0.00864051272695987 +0,0.75,2,5,0.8333333333333333,1.0,0.09259259259259256 +0,0.75,3,140,0.7415730337078651,0.7333333333333333,0.028929355546945534 +0,0.75,4,1,0.8411764705882353,0.9166666666666666,-0.02079536039768015 +0,0.75,5,2,0.37074068149096145,0.9898989898989898,0.08893504151913229 +0,0.75,6,44,0.5318530266983875,0.9642857142857143,-0.00014886971028860163 +0,0.75,7,10,0.25831969448990727,0.9444444444444444,-0.03579795256891627 +0,0.75,8,4,0.5019948230864674,0.9304909560723513,0.0007336392703400918 +0,0.75,9,12,0.34527541424093144,0.6965811965811965,0.01441753972108506 +0,1.0,0,18,0.9077490774907748,0.831081081081081,0.020211083569802057 +0,1.0,1,69,0.4280463143311497,0.6916666666666667,-0.0024115169941515413 +0,1.0,2,5,0.7692307692307693,1.0,0.028490028490028574 +0,1.0,3,140,0.7272727272727273,0.7111111111111111,0.014629049111807735 +0,1.0,4,1,0.7859237536656892,0.8589743589743589,-0.07604807732022623 +0,1.0,5,2,0.32319994363815696,0.9848484848484849,0.04139430366632779 +0,1.0,6,44,0.5186591560629684,0.9285714285714286,-0.013342740345707704 +0,1.0,7,10,0.2510204081632653,0.9444444444444444,-0.04309723889555822 +0,1.0,8,4,0.5004931186519341,0.9304909560723513,-0.0007680651641931435 +0,1.0,9,12,0.3028365305787834,0.6688034188034189,-0.02802134394106298 +1,0.0,0,2,0.555984555984556,0.972972972972973,0.0 +1,0.0,1,3,0.5795225653221874,0.9976043805612593,0.0 +1,0.0,2,50,0.42962962962962964,0.8382352941176471,0.0 +1,0.0,3,86,0.27739361702127663,0.8181818181818181,0.0 +1,0.0,4,46,0.6538461538461537,0.5483870967741935,0.0 +1,0.0,5,6,0.5279805352798054,0.7439024390243902,0.0 +1,0.0,6,9,0.47346514047866806,0.49242424242424243,0.0 +1,0.0,7,8,1.0,1.0,0.0 +1,0.0,8,1,0.8076923076923077,1.0,0.0 +1,0.0,9,75,0.9807296026788589,0.9850571725571726,0.0 +1,0.25,0,2,0.5658914728682171,0.9864864864864865,0.009906916883661077 +1,0.25,1,3,0.6023883135587408,0.9973990417522245,0.022865748236553407 +1,0.25,2,50,0.44103313840155944,0.8382352941176471,0.011403508771929804 +1,0.25,3,86,0.28566573008247986,0.8181818181818181,0.008272113061203235 +1,0.25,4,46,0.6538461538461537,0.5483870967741935,0.0 +1,0.25,5,6,0.5335335335335335,0.7439024390243902,0.005552998253728125 +1,0.25,6,9,0.46900826446280997,0.49134199134199136,-0.004456876015858091 +1,0.25,7,8,1.0,1.0,0.0 +1,0.25,8,1,0.8316831683168318,1.0,0.023990860624524046 +1,0.25,9,75,0.9789997415352805,0.9816787941787942,-0.0017298611435784261 +1,0.5,0,2,0.5983606557377049,0.9864864864864865,0.04237609975314893 +1,0.5,1,3,0.5950263055479481,0.9395013952508819,0.015503740225760732 +1,0.5,2,50,0.4659111350962404,0.8382352941176471,0.03628150546661074 +1,0.5,3,86,0.3038850776042427,0.8181818181818181,0.026491460582966087 +1,0.5,4,46,0.679245283018868,0.5806451612903226,0.025399129172714252 +1,0.5,5,6,0.5448554042536322,0.7439024390243902,0.01687486897382684 +1,0.5,6,9,0.4680412371134021,0.49134199134199136,-0.005423903365265936 +1,0.5,7,8,1.0,1.0,0.0 +1,0.5,8,1,0.8199999999999998,0.9761904761904762,0.01230769230769213 +1,0.5,9,75,0.9765630072063884,0.9768711018711018,-0.004166595472470447 +1,0.75,0,2,0.6431718061674009,0.9864864864864865,0.0871872501828449 +1,0.75,1,3,0.5892614182467265,0.900697625440952,0.009738852924539132 +1,0.75,2,50,0.47251332965618675,0.8382352941176471,0.042883700026557114 +1,0.75,3,86,0.31549441865059763,0.8181818181818181,0.038100801629321 +1,0.75,4,46,0.679245283018868,0.5806451612903226,0.025399129172714252 +1,0.75,5,6,0.5516592310652149,0.7439024390243902,0.023678695785409487 +1,0.75,6,9,0.47046632124352333,0.49134199134199136,-0.002998819235144723 +1,0.75,7,8,1.0,1.0,0.0 +1,0.75,8,1,0.8484848484848484,1.0,0.04079254079254069 +1,0.75,9,75,0.9813023436992794,0.9816787941787942,0.0005727410204204864 +1,1.0,0,2,0.7487179487179488,0.9864864864864865,0.19273339273339285 +1,1.0,1,3,0.5900895998512916,0.900697625440952,0.010567034529104236 +1,1.0,2,50,0.5188275566966196,0.8382352941176471,0.08919792706698998 +1,1.0,3,86,0.3760424007517031,0.8181818181818181,0.09864878373042646 +1,1.0,4,46,0.7037037037037037,0.6129032258064516,0.049857549857549976 +1,1.0,5,6,0.5441986886103379,0.7439024390243902,0.01621815333053256 +1,1.0,6,9,0.4794086589229145,0.49134199134199136,0.005943518444246454 +1,1.0,7,8,1.0,1.0,0.0 +1,1.0,8,1,0.8842105263157896,1.0,0.07651821862348185 +1,1.0,9,75,0.9859736514386839,0.9816787941787942,0.005244048759824982 +2,0.0,0,10,0.09876543209876543,0.3076923076923077,0.0 +2,0.0,1,1,0.5263157894736842,0.625,0.0 +2,0.0,2,135,0.2881288368014032,0.8690476190476191,0.0 +2,0.0,3,2,0.25392765801972955,0.75,0.0 +2,0.0,4,30,0.9690721649484536,0.94,0.0 +2,0.0,5,132,0.4811991415541446,0.8555825242718447,0.0 +2,0.0,6,4,0.9733738458483634,0.9900486434639627,0.0 +2,0.0,7,13,0.4408369408369409,0.9318181818181819,0.0 +2,0.0,8,7,0.046121316241575255,0.8157894736842105,0.0 +2,0.0,9,6,0.3934390771449171,0.7885802469135803,0.0 +2,0.25,0,10,0.1176470588235294,0.3076923076923077,0.01888162672476397 +2,0.25,1,1,0.5263157894736842,0.625,0.0 +2,0.25,2,135,0.3118580765639589,0.8690476190476191,0.0237292397625557 +2,0.25,3,2,0.28338171262699563,0.8125,0.029454054607266078 +2,0.25,4,30,0.9690721649484536,0.94,0.0 +2,0.25,5,132,0.4804878048780488,0.8410194174757282,-0.0007113366760957973 +2,0.25,6,4,0.9859407908145934,0.9900486434639627,0.01256694496623001 +2,0.25,7,13,0.44609149455918273,0.9318181818181819,0.005254553722241817 +2,0.25,8,7,0.06176250933532487,0.8157894736842105,0.015641193093749617 +2,0.25,9,6,0.40053768640117604,0.7824074074074074,0.0070986092562589365 +2,0.5,0,10,0.14545454545454545,0.3076923076923077,0.04668911335578002 +2,0.5,1,1,0.5263157894736842,0.625,0.0 +2,0.5,2,135,0.35779327972735503,0.8690476190476191,0.06966444292595181 +2,0.5,3,2,0.27168831168831165,0.8125,0.017760653668582094 +2,0.5,4,30,0.9690721649484536,0.94,0.0 +2,0.5,5,132,0.48409645995339823,0.8264563106796117,0.0028973183992536122 +2,0.5,6,4,0.9886441490819402,0.9886441490819402,0.01527030323357681 +2,0.5,7,13,0.468078381795196,0.8863636363636364,0.027241440958255092 +2,0.5,8,7,0.0837908496732026,0.8157894736842105,0.03766953343162735 +2,0.5,9,6,0.4152159896840748,0.7762345679012346,0.0217769125391577 +2,0.75,0,10,0.27586206896551724,0.3076923076923077,0.1770966368667518 +2,0.75,1,1,0.5555555555555556,0.625,0.0292397660818714 +2,0.75,2,135,0.38409090909090915,0.8571428571428572,0.09596207228950593 +2,0.75,3,2,0.25207044264635886,0.78125,-0.0018572153733706909 +2,0.75,4,30,0.9690721649484536,0.94,0.0 +2,0.75,5,132,0.4842105263157895,0.816747572815534,0.003011384761644864 +2,0.75,6,4,0.9878070464559553,0.9816216771718279,0.0144332006075919 +2,0.75,7,13,0.4953242835595777,0.8636363636363636,0.05448734272263678 +2,0.75,8,7,0.20940438871473352,0.8157894736842105,0.16328307247315826 +2,0.75,9,6,0.4359758123463353,0.7638888888888888,0.04253673520141821 +2,1.0,0,10,0.34782608695652173,0.3076923076923077,0.2490606548577563 +2,1.0,1,1,0.5263157894736842,0.625,0.0 +2,1.0,2,135,0.399138268703486,0.8452380952380952,0.11100943190208279 +2,1.0,3,2,0.25128900949796473,0.78125,-0.0026386485217648192 +2,1.0,4,30,0.5662650602409639,0.94,-0.4028071047074897 +2,1.0,5,132,0.4798898963730569,0.8070388349514563,-0.0013092451810877148 +2,1.0,6,4,0.974498660479089,0.9530236594500776,0.0011248146307255569 +2,1.0,7,13,0.5532377227292482,0.8636363636363636,0.11240078189230729 +2,1.0,8,7,0.3136863136863137,0.8157894736842105,0.2675649974447385 +2,1.0,9,6,0.44662116040955635,0.7608024691358024,0.053182083264639246 diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_summary.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_summary.csv new file mode 100644 index 0000000..b3382ef --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_summary.csv @@ -0,0 +1,16 @@ +seed,beta,n_eligible,n_improve_vs_beta0,n_degrade_vs_beta0,n_unchanged,mean_delta_vs_beta0,median_delta_vs_beta0,min_delta_vs_beta0,max_delta_vs_beta0 +0,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +0,0.25,10,9,1,0,0.014447779488805679,0.008862381334935221,-0.0048289738430582485,0.059259259259259345 +0,0.5,10,8,2,0,0.02842211408129357,0.01786607568659304,-0.02864627632429495,0.09259259259259256 +0,0.75,10,7,3,0,0.020663517144586436,0.011529026224022465,-0.03579795256891627,0.09259259259259256 +0,1.0,10,4,6,0,-0.005896451782293366,-0.0015897910791723424,-0.07604807732022623,0.04139430366632779 +1,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +1,0.25,10,6,2,2,0.007580540867216318,0.00691255565746568,-0.004456876015858091,0.023990860624524046 +1,0.5,10,7,2,1,0.016564399764498333,0.016189304599793786,-0.005423903365265936,0.04237609975314893 +1,0.75,10,8,1,1,0.026535489229920235,0.02453891247906187,-0.002998819235144723,0.0871872501828449 +1,1.0,10,9,0,1,0.054492862707554934,0.03303785159404127,0.0,0.19273339273339285 +2,0.0,10,0,0,10,0.0,0.0,0.0,0.0 +2,0.25,10,7,1,2,0.011191488545697032,0.009832777111244473,-0.0007113366760957973,0.029454054607266078 +2,0.5,10,8,0,2,0.02389697185121845,0.019768783103869897,0.0,0.06966444292595181 +2,0.75,10,8,1,1,0.05781929956312084,0.0358882506416448,-0.0018572153733706909,0.1770966368667518 +2,1.0,10,6,3,1,0.038758776558190745,0.0271534489476824,-0.4028071047074897,0.2675649974447385 diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__prototype_effect_pooled.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__prototype_effect_pooled.csv new file mode 100644 index 0000000..f723e93 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__prototype_effect_pooled.csv @@ -0,0 +1,46 @@ +class_id,class_name,beta,n_seeds,n_contributors_seed0,mean_displacement,std_sample_displacement,min_displacement,max_displacement,mean_relative_displacement +0,C&C,0.0,3,10,0.0,0.0,0.0,0.0,0.0 +0,C&C,0.25,3,10,0.6538969764641525,0.2476139923256249,0.36797724022308037,0.7971996511065712,0.12339295466205748 +0,C&C,0.5,3,10,1.118626143061394,0.3963727050225245,0.6695589690144707,1.4197451476394127,0.2105520814834464 +0,C&C,0.75,3,10,1.4486883045785106,0.4772384272097319,0.9254131398589358,1.8599716115561264,0.2724794282105077 +0,C&C,1.0,3,10,1.6935576092145832,0.5048307458168824,1.1548168541805874,2.1557306865858097,0.3187155176777392 +1,Exfiltration,0.0,3,9,0.0,0.0,0.0,0.0,0.0 +1,Exfiltration,0.25,3,9,0.8338879088602954,0.2741219923879293,0.564975966349673,1.112938631376158,0.12284229408877616 +1,Exfiltration,0.5,3,9,1.3517411074356709,0.42853570203269503,0.9364837301255268,1.792423075103891,0.19904477988621203 +1,Exfiltration,0.75,3,9,1.7154927988410469,0.49900998925508244,1.2417976657519203,2.2364541806817226,0.2524571065984375 +1,Exfiltration,1.0,3,9,2.0204769683934525,0.5020218016853628,1.5471979064159855,2.5469976586314798,0.2973123924755737 +2,Exploitation,0.0,3,8,0.0,0.0,0.0,0.0,0.0 +2,Exploitation,0.25,3,8,0.6666685285017905,0.18200087099143822,0.4992592412018676,0.8603955105014754,0.12232141909162007 +2,Exploitation,0.5,3,8,1.290546444813813,0.27066503671707776,1.0125335941087894,1.5532104750000217,0.2357165771334672 +2,Exploitation,0.75,3,8,1.85529485339079,0.309252194657293,1.4982123491722337,2.0362822833565866,0.33680609491935254 +2,Exploitation,1.0,3,8,2.343885289699028,0.4121833629248462,1.934383194968876,2.7586985908860333,0.4231523311819784 +3,Lateral_movement,0.0,3,10,0.0,0.0,0.0,0.0,0.0 +3,Lateral_movement,0.25,3,10,1.4060488264140794,1.112581110058643,0.7543102387618712,2.6907008570910045,0.1950351203781113 +3,Lateral_movement,0.5,3,10,2.1062819395775434,1.2786883236883044,1.3479271988117953,3.5825998636295417,0.30073851924782274 +3,Lateral_movement,0.75,3,10,2.4882102973029516,1.1721414711156182,1.776607909870438,3.841073254381612,0.36273794263869424 +3,Lateral_movement,1.0,3,10,2.7240136758504594,1.0350790616091956,2.0688588443755878,3.9173091599439407,0.40257578575923714 +4,Normal,0.0,3,10,0.0,0.0,0.0,0.0,0.0 +4,Normal,0.25,3,10,0.7591659420154394,0.32709942089138133,0.4522766775093571,1.1032890094562697,0.1434738312600832 +4,Normal,0.5,3,10,1.1795350343849011,0.4753882408285596,0.7146990338285016,1.664815841256164,0.2223203173446142 +4,Normal,0.75,3,10,1.4114463532549024,0.5293831411155098,0.8756391811740736,1.934162646704862,0.2653104861812456 +4,Normal,1.0,3,10,1.557331074720638,0.5480258187545208,0.9899061725928462,2.083642028161429,0.2920883603702228 +5,RDOS,0.0,3,10,0.0,0.0,0.0,0.0,0.0 +5,RDOS,0.25,3,10,1.2054779842601517,0.7293986290124911,0.3633215970184691,1.6366469943162223,0.16305203219584322 +5,RDOS,0.5,3,10,1.8233255406714963,0.9982539168312503,0.6718616497957892,2.444982027217194,0.24557116018390387 +5,RDOS,0.75,3,10,2.258386664022743,1.1111681429896514,0.9758862376670907,2.932638911884029,0.3031764701965988 +5,RDOS,1.0,3,10,2.6312389613743776,1.1651317404804817,1.2860320043774807,3.322434234042245,0.3523347102777825 +6,Reconnaissance,0.0,3,10,0.0,0.0,0.0,0.0,0.0 +6,Reconnaissance,0.25,3,10,0.32567775116244774,0.11787216678901793,0.19750804124277122,0.4294267183380856,0.055493937112848946 +6,Reconnaissance,0.5,3,10,0.6024083113978019,0.19509381934930423,0.37937859322083467,0.7413987406482649,0.10221941764104735 +6,Reconnaissance,0.75,3,10,0.8264969491723915,0.24209312478661804,0.5470003591054624,0.970753367392759,0.13974777812154845 +6,Reconnaissance,1.0,3,10,1.00151995809175,0.26099624924542353,0.7025557661555466,1.183931540468372,0.16884924727680906 +7,Tampering,0.0,3,9,0.0,0.0,0.0,0.0,0.0 +7,Tampering,0.25,3,9,0.6958022407930811,0.08690380480265254,0.5957873490790528,0.7528833961261016,0.14078908166802012 +7,Tampering,0.5,3,9,1.3112838021058792,0.35103438443835877,1.0369597200237384,1.706873516878616,0.26700132395623627 +7,Tampering,0.75,3,9,1.8761038090972573,0.7753681393042942,1.3704230949002763,2.768796503000988,0.3837387923644013 +7,Tampering,1.0,3,9,2.354018138564271,1.165407725687578,1.6272957425211183,3.6982366587327884,0.48267240310142356 +8,Weaponization,0.0,3,10,0.0,0.0,0.0,0.0,0.0 +8,Weaponization,0.25,3,10,0.8836150972431183,0.469653346490459,0.36048411330770536,1.2689737477647698,0.15212044699184105 +8,Weaponization,0.5,3,10,1.5842620542218915,0.7696695063647051,0.695561899244867,2.0356955585328023,0.2703303067352452 +8,Weaponization,0.75,3,10,2.141052662540764,1.025083084023739,0.9758406496021348,2.9039530195295855,0.36329011806065653 +8,Weaponization,1.0,3,10,2.560790423872144,1.2282783431433644,1.189062205569606,3.5588066747258478,0.4334295567239335 diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__runtime.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__runtime.csv new file mode 100644 index 0000000..d3d0da0 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__runtime.csv @@ -0,0 +1,19 @@ +seed,tag,wall_seconds,cpu_percent,peak_rss_kb,source +0,original,2526.78,740,6509448,Session5 docs +0,beta0.0,2477,740,6509448,Session5 docs +0,beta0.25,2531,738,6509448,Session5 docs +0,beta0.5,2577,737,6509448,Session5 docs +0,beta0.75,2625,733,6509448,Session5 docs +0,beta1.0,2528,737,6509448,Session5 docs +1,original,2481.16,738,7840956,/tmp/opencode/s6_seed1_original.log +1,beta0.0,2438.89,744,7840864,/tmp/opencode/s6_seed1_beta0.0.log +1,beta0.25,2432.74,744,7840716,/tmp/opencode/s6_seed1_beta0.25.log +1,beta0.5,2443.99,743,7840960,/tmp/opencode/s6_seed1_beta0.5.log +1,beta0.75,2441.36,745,7840720,/tmp/opencode/s6_seed1_beta0.75.log +1,beta1.0,2468.86,744,7841004,/tmp/opencode/s6_seed1_beta1.0.log +2,original,2469.7,743,6764120,/tmp/opencode/s6_seed2_original.log +2,beta0.0,2440.14,742,6764008,/tmp/opencode/s6_seed2_beta0.0.log +2,beta0.25,2485.22,740,6764080,/tmp/opencode/s6_seed2_beta0.25.log +2,beta0.5,2423.42,742,6764128,/tmp/opencode/s6_seed2_beta0.5.log +2,beta0.75,2465.82,740,6764032,/tmp/opencode/s6_seed2_beta0.75.log +2,beta1.0,2540.66,741,6763848,/tmp/opencode/s6_seed2_beta1.0.log diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__selection_summary.json b/artifacts/analysis/xiiotid__alpha0.5__multiseed__selection_summary.json new file mode 100644 index 0000000..14958ed --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__selection_summary.json @@ -0,0 +1,111 @@ +{ + "alpha": 0.5, + "betas": [ + 0.0, + 0.25, + 0.5, + 0.75, + 1.0 + ], + "dataset": "xiiotid", + "missing_artifacts": [], + "n_missing_artifacts": 0, + "outputs": { + "client_deltas": "/home/willtanoe/projects/perdalam-protean/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_deltas.csv", + "client_validation_detail": "/home/willtanoe/projects/perdalam-protean/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_detail.csv", + "client_validation_summary": "/home/willtanoe/projects/perdalam-protean/artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_summary.csv", + "per_seed": "/home/willtanoe/projects/perdalam-protean/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_per_seed.csv", + "pooled": "/home/willtanoe/projects/perdalam-protean/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_pooled.csv", + "prototype_effect_pooled": "/home/willtanoe/projects/perdalam-protean/artifacts/analysis/xiiotid__alpha0.5__multiseed__prototype_effect_pooled.csv", + "test_descriptive": "/home/willtanoe/projects/perdalam-protean/artifacts/analysis/xiiotid__alpha0.5__multiseed__test_descriptive.csv" + }, + "per_seed_diagnostic": [ + { + "seed": 0, + "selected_beta": 0.5, + "test_used_for_selection": false, + "val_rare_macro_f1_per_client_mean": 0.5857617457722089 + }, + { + "seed": 1, + "selected_beta": 1.0, + "test_used_for_selection": false, + "val_rare_macro_f1_per_client_mean": 0.6831172735008992 + }, + { + "seed": 2, + "selected_beta": 0.75, + "test_used_for_selection": false, + "val_rare_macro_f1_per_client_mean": 0.5049373198599186 + } + ], + "pooled_table": [ + { + "beta": 0.0, + "max_val_rare_macro_f1": 0.6286244107933443, + "min_val_rare_macro_f1": 0.4471180202967977, + "n_seeds": 3, + "pooled_val_rare_macro_f1_per_client_mean": 0.5443606875936857, + "pooled_val_rare_recall_global": 0.8851218526247052, + "pooled_val_rare_recall_per_client_mean": 0.8390875794416744, + "std_pop_val_rare_macro_f1": 0.07466584310638286, + "std_sample_val_rare_macro_f1": 0.09144660841267144 + }, + { + "beta": 0.25, + "max_val_rare_macro_f1": 0.6362049516605607, + "min_val_rare_macro_f1": 0.4583095088424948, + "n_seeds": 3, + "pooled_val_rare_macro_f1_per_client_mean": 0.5554339572275921, + "pooled_val_rare_recall_global": 0.8859991678322592, + "pooled_val_rare_recall_per_client_mean": 0.8412777237951431, + "std_pop_val_rare_macro_f1": 0.07354034600581387, + "std_sample_val_rare_macro_f1": 0.09006816161098347 + }, + { + "beta": 0.5, + "max_val_rare_macro_f1": 0.6451888105578426, + "min_val_rare_macro_f1": 0.4710149921480162, + "n_seeds": 3, + "pooled_val_rare_macro_f1_per_client_mean": 0.5673218494926892, + "pooled_val_rare_recall_global": 0.8796543921076175, + "pooled_val_rare_recall_per_client_mean": 0.8388718664026588, + "std_pop_val_rare_macro_f1": 0.07229177955094838, + "std_sample_val_rare_macro_f1": 0.08853898624879539 + }, + { + "beta": 0.75, + "max_val_rare_macro_f1": 0.6551599000232645, + "min_val_rare_macro_f1": 0.5049373198599186, + "n_seeds": 3, + "pooled_val_rare_macro_f1_per_client_mean": 0.5793667895728949, + "pooled_val_rare_recall_global": 0.8724082935139957, + "pooled_val_rare_recall_per_client_mean": 0.8290881018460031, + "std_pop_val_rare_macro_f1": 0.06133569126687837, + "std_sample_val_rare_macro_f1": 0.07512057331236716 + }, + { + "beta": 1.0, + "max_val_rare_macro_f1": 0.6831172735008992, + "min_val_rare_macro_f1": 0.4858767968549885, + "n_seeds": 3, + "pooled_val_rare_macro_f1_per_client_mean": 0.5734790834215032, + "pooled_val_rare_recall_global": 0.8605920713174823, + "pooled_val_rare_recall_per_client_mean": 0.8207630276313397, + "std_pop_val_rare_macro_f1": 0.08201681611961567, + "std_sample_val_rare_macro_f1": 0.1004496749103663 + } + ], + "primary_selection": { + "rule": "max pooled validation per-client-mean rare-class macro-F1 (tie-break: recall, then lower std, then smaller beta)", + "selected_beta": 0.75, + "selection_split": "validation", + "test_used_for_selection": false, + "tie_break_used": false + }, + "seeds": [ + 0, + 1, + 2 + ] +} diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__test_descriptive.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__test_descriptive.csv new file mode 100644 index 0000000..7873571 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__test_descriptive.csv @@ -0,0 +1,19 @@ +seed,aggregation,beta,test_rare_macro_f1,test_rare_recall,test_macro_f1,test_accuracy,test_weighted_f1,test_rare_support +0,support_weighted_beta0,0.0,0.6135753984293303,0.8735084256028207,0.7104808187657585,0.8919738870362524,0.9183694890613716,25454 +0,support_weighted_beta0.25,0.25,0.6321541115681123,0.8773498581417057,0.725299460216051,0.9035196000761526,0.926451884570777,25454 +0,support_weighted_beta0.5,0.5,0.6408294064651883,0.8715828942623447,0.7307425502959082,0.898158213116667,0.926333154717474,25454 +0,support_weighted_beta0.75,0.75,0.6361887569975536,0.8575434903523123,0.7262984173916737,0.8886698478790893,0.9222775427473946,25454 +0,support_weighted_beta1,1.0,0.6155094451472803,0.8324066639868958,0.716383356046256,0.8846349896518476,0.9219098532801282,25454 +0,original_protean,,0.6096198579015474,0.8714872155034664,0.7077011542142579,0.8904324115186911,0.9175853183547894,25454 +1,support_weighted_beta0,0.0,0.6564742559815336,0.8897768142213348,0.7097666280372251,0.9014376869269365,0.9293555331942173,53706 +1,support_weighted_beta0.25,0.25,0.6666094974054222,0.8837877786397739,0.7233667411923812,0.9209118656766832,0.9413989745515946,53706 +1,support_weighted_beta0.5,0.5,0.6686778369724466,0.8622069637058571,0.7259377884762019,0.9243510142417599,0.9436538792949344,53706 +1,support_weighted_beta0.75,0.75,0.6719373483509694,0.8518834947182458,0.7290145924987934,0.9254933028723031,0.9447335103607959,53706 +1,support_weighted_beta1,1.0,0.6875134088934267,0.8534327539272342,0.7411311663307938,0.9261934152587652,0.9462160368470328,53706 +1,original_protean,,0.655665402810942,0.8893650106408222,0.7091760460445067,0.9008849666218349,0.929132450103975,53706 +2,support_weighted_beta0,0.0,0.662764587938588,0.8705512070669401,0.7120319647362015,0.9103364838390724,0.92974243713613,53706 +2,support_weighted_beta0.25,0.25,0.6734586913167508,0.8753721375097582,0.7196254203975677,0.9089976724333818,0.9317651035879723,53706 +2,support_weighted_beta0.5,0.5,0.6798480896619509,0.8754195835969492,0.7328456679652509,0.9228586694179856,0.942730385386783,53706 +2,support_weighted_beta0.75,0.75,0.6883814197743454,0.878321192150252,0.7381943369820925,0.9050119448999269,0.9338838836432976,53706 +2,support_weighted_beta1,1.0,0.6765282329293555,0.866988007382502,0.7274401402636781,0.8852982540179696,0.9219916576006046,53706 +2,original_protean,,0.662274505162012,0.8700179539413673,0.7100168446133323,0.9070447273553561,0.9274806689914715,53706 diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_per_seed.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_per_seed.csv new file mode 100644 index 0000000..1105bd1 --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_per_seed.csv @@ -0,0 +1,16 @@ +seed,beta,eligible_clients,val_rare_macro_f1_per_client_mean,val_rare_macro_f1_global,val_rare_recall_per_client_mean,val_rare_recall_global,val_macro_f1_global,val_accuracy_global,val_weighted_f1_global,manifest_content_hash,manifest_reload_ok +0,0.0,10,0.5573396316909152,0.611641191657799,0.8902302969744831,0.884183934007262,0.7089290421528118,0.891038396344609,0.9177409701361711,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +0,0.25,10,0.5717874111797209,0.6294751042494888,0.8917395701407329,0.8884876105371173,0.7234313878791793,0.9029528090991721,0.9259328206398422,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +0,0.5,10,0.5857617457722089,0.641708906658835,0.896707176387409,0.8871727193240352,0.7311655917391926,0.8978308399046847,0.9260054050251649,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +0,0.75,10,0.5780031488355016,0.6371069192487316,0.87787043042857,0.8739100629025399,0.7266028483648479,0.8878939740093842,0.9216960401530047,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +0,1.0,10,0.5514431799086219,0.6153071361625583,0.8549991950573347,0.8470433259940177,0.7159511763355666,0.8832633208047755,0.9211822149143563,e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c,True +1,0.0,10,0.6286244107933443,0.6569352416154539,0.8396765416613696,0.8905198283723711,0.7093242919319133,0.8996118603679957,0.927382668794825,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +1,0.25,10,0.6362049516605607,0.666240024874223,0.8405612961857546,0.884108824349407,0.7223669677789497,0.9192030854644164,0.9396709981211578,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +1,0.5,10,0.6451888105578426,0.6673830150950097,0.8351356163755115,0.8638326783844901,0.7242124501702869,0.9225931657945808,0.9418322096361332,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +1,0.75,10,0.6551599000232645,0.6704724297885282,0.8341169610062401,0.8555902715554661,0.727133822228491,0.9237600412705431,0.9429953735951887,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +1,1.0,10,0.6831172735008992,0.6857068076012445,0.8373427674578531,0.8568761784410628,0.7390058704092461,0.9244970152553614,0.944519795156427,54ea1df16a2e7998cc21c3cee86318dc02a2fd8396894371ebaab7f87bad7e98,True +2,0.0,10,0.4471180202967977,0.6587202228452144,0.7873558996891707,0.8806617954944825,0.7084642613349315,0.907890534797455,0.9275343400311638,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +2,0.25,10,0.4583095088424948,0.6692421879798821,0.7915323050589418,0.885401068610253,0.7161623907336252,0.9070184489154199,0.930311668857389,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +2,0.5,10,0.4710149921480162,0.6778923943006044,0.7847728064450561,0.8879577786143277,0.7309557561546376,0.920259414842656,0.9410565694226878,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +2,0.75,10,0.5049373198599186,0.6861239514935437,0.775276914103199,0.887724546083981,0.7359120983111951,0.9023509470115705,0.9321055597645047,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True +2,1.0,10,0.4858767968549885,0.6763138704582848,0.7699471203788313,0.8778567095173665,0.7269217682937931,0.883533544599209,0.9208500397966657,be881ac63bc0057eca7c82d762d082344e1f4fbd1dff56a475898c6de977712a,True diff --git a/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_pooled.csv b/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_pooled.csv new file mode 100644 index 0000000..31df2cd --- /dev/null +++ b/artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_pooled.csv @@ -0,0 +1,6 @@ +beta,n_seeds,pooled_val_rare_macro_f1_per_client_mean,std_sample_val_rare_macro_f1,std_pop_val_rare_macro_f1,min_val_rare_macro_f1,max_val_rare_macro_f1,pooled_val_rare_recall_per_client_mean,pooled_val_rare_recall_global +0.0,3,0.5443606875936857,0.09144660841267144,0.07466584310638286,0.4471180202967977,0.6286244107933443,0.8390875794416744,0.8851218526247052 +0.25,3,0.5554339572275921,0.09006816161098347,0.07354034600581387,0.4583095088424948,0.6362049516605607,0.8412777237951431,0.8859991678322592 +0.5,3,0.5673218494926892,0.08853898624879539,0.07229177955094838,0.4710149921480162,0.6451888105578426,0.8388718664026588,0.8796543921076175 +0.75,3,0.5793667895728949,0.07512057331236716,0.06133569126687837,0.5049373198599186,0.6551599000232645,0.8290881018460031,0.8724082935139957 +1.0,3,0.5734790834215032,0.1004496749103663,0.08201681611961567,0.4858767968549885,0.6831172735008992,0.8207630276313397,0.8605920713174823 diff --git a/docs/handoff/SESSION_HANDOFF.md b/docs/handoff/SESSION_HANDOFF.md index 9b0e986..a30dd21 100644 --- a/docs/handoff/SESSION_HANDOFF.md +++ b/docs/handoff/SESSION_HANDOFF.md @@ -3,11 +3,25 @@ This is the canonical continuation point for the next agent session. It is written to be readable without prior chat history, terminal scrollback, or git history. -Last updated: 2026-09-25 (Session 5, one paired full beta condition + validation-only -selection). Session 5 type: **protocol-validation experiment**. Exactly one -`(dataset, alpha, seed)` condition was run across all five beta candidates; beta was -selected on validation only and the selected beta evaluated once on test. **No full grid -was launched.** +Last updated: 2026-09-25 (Session 6 + post-experiment audit). Session 6 type: +**protocol-scaling / stability experiment**. The single Session-5 condition was extended +from seed 0 to seeds `{0,1,2}`; beta was selected by the **pooled-across-seed validation** +rule and the selected beta evaluated on test. **No full grid was launched; no +alpha/dataset expansion was run.** A strict independent audit +(`docs/research/SESSION6_AUDIT.md`) returned **PASS WITH MINOR CORRECTIONS**; `beta_star = +0.75` was independently reproduced from validation artifacts. + +> **Audit summary (Session 6).** Verdict: **PASS WITH MINOR CORRECTIONS**. Independently +> recomputed `beta_star = 0.75` from `val_per_client.csv`/`val_selection.json`; all 15 +> seed×beta seed-level scores and the pooled mean/sample-std match the report exactly. +> Validation-only selection confirmed in code (`analyze_multiseed_beta.py` reads only +> `val_selection.json`; test read in a later block); `test_used_for_selection=false` +> upheld. Rare-class flags match the training-derived definition for all 30 cells; +> manifests reproduce from raw data with disjoint splits. beta=0 vs `original_protean` = +> semantically equivalent + floating-point divergence (class B). Minor corrections: +> persisted `artifacts/analysis/xiiotid__alpha0.5__multiseed__runtime.csv` (runtime +> provenance) and clarified the client improve/degrade counts (unchanged/tie clients). +> **Session 6 is safe to checkpoint.** --- @@ -31,188 +45,253 @@ bitwise identical to the baseline (see §7). ## 2. Authoritative source and immutability - `planningawal/PROTEAN_Revisi_Lurus.md` — authoritative research specification. -- `planningawal/` is immutable. Re-verified unchanged this session: `git status -- - planningawal` and `git diff -- planningawal` are both empty. +- `planningawal/` is immutable. Re-verified unchanged this session: `git status --short + planningawal/` and `git diff -- planningawal/` are both empty. - Raw datasets re-verified unchanged (SHA-256): `~/datasets/X-IIoTID/X-IIoTID-dataset.csv` `7b9290057ee42e784da3c0d84b781815502c9205c74175c96374e71a5ffd98a0`; `~/datasets/5G-NIDD/5G-NIDD.csv` `fa36f80859585f474504ca69eb951a079b35145537976c86b00aae3aab46ee59`. +- Frozen baseline (`original_protean`) source unchanged; no `equal_available` mode added; + beta candidate set unchanged (`{0, .25, .5, .75, 1}`). -## 3. Session 5 objective and outcome +## 3. Session 6 objective and outcome -Objective: run ONE paired full condition (X-IIoTID, alpha=0.5, seed=0) across -`beta ∈ {0, 0.25, 0.5, 0.75, 1}`, reusing the frozen manifest and training settings; -select beta on validation only; evaluate the selected beta on test; add the direct -prototype-effect and client-level analyses; update docs. **All acceptance items were met.** -No commit or push was made. +Objective: extend the validated paired protocol from one seed to seeds `{0,1,2}` for +X-IIoTID alpha=0.50, reuse the seed-0 Session-5 results, run the two new seeds across the +five beta candidates plus `original_protean`, select beta with **pooled validation only**, +evaluate the selected beta on test, and produce cross-seed direct-effect / client-level / +stability analyses + docs. **All acceptance items were met.** No commit or push was made. + +Outcome: **PRIMARY pooled-validation selected beta = 0.75** (validation only). Per-seed +diagnostic selections varied (0.50 / 1.00 / 0.75), and between-seed variance is comparable +to the between-beta spread, so the selection is arithmetically clear but statistically +weak. The protocol ran cleanly end to end. ## 4. Condition and pairing | Field | Value | |---|---| -| dataset / alpha / seed | `xiiotid` / 0.50 / 0 | -| manifest | `artifacts/partitions/xiiotid__alpha0.5__seed0.json` (reused) | -| manifest content hash | `e81a6538ecefcda202aa7602310ff8d8c9529031ab9dcbec4ebbabc0e63b636c` | -| partition hash | `2465485539a3e923548dcc0c73d8acc5076d456536b78682d512897ea586e779` | -| split hash / sizes | `d0acc0c14e3ecbeba02dfa9e6227c983da12dec2d6111df0b714002e6e03c29d` / 569,909 train, 81,414 val, 162,831 test | -| preprocessing hash | `8580cee41c698ce43cf3269e4a2b3a01c92d6df22820f4004c0267478bd2579a` | +| dataset / alpha / seeds | `xiiotid` / 0.50 / `{0,1,2}` | +| runs | 15 `support_weighted` + 3 `original_protean` (seed 0 reused; 12 run this session) | | training | 10 clients, 10 rounds, 3 local epochs, batch 256, lr 1e-3, mu 0.1, lambda 1.0, Adam | -| code revision | HEAD `ad9aa53ba76f60671ace991fcee85f003781f196`; only `scripts/run_baseline.py` modified (hash-object `2b469e35ecbbc6ea3dbd4b0632419993e4d5130b`) | +| code revision | unchanged from HEAD `50f2da72166e4f609077d65b4916f745da5e8e3b`; only read-only analysis code added | -Pairing proof: the round-0 prototype snapshot is **byte-identical** (SHA-256 prefix -`e27f41b3a62eb83c`) across all five beta runs and the frozen `original_protean` run; -`train_support_Nij.csv`, `characterization.csv`, `rare_unseen.csv` are identical across -runs. All runs are in `results/xiiotid__alpha0.5__seed0__support_weighted__beta{0,0.25,0.5,0.75,1}/`. +Manifests (reused, never regenerated; one content hash per seed across all six runs): -## 5. Validation results and beta selection +| seed | manifest | content hash | partition hash | split hash | rng seed | +|---|---|---|---|---|---| +| 0 | `artifacts/partitions/xiiotid__alpha0.5__seed0.json` | `e81a6538…` | `2465485539…` | `d0acc0c1…` | 2009062324 | +| 1 | `artifacts/partitions/xiiotid__alpha0.5__seed1.json` | `54ea1df1…` | `4061c5a1…` | `880a9f82…` | 621679668 | +| 2 | `artifacts/partitions/xiiotid__alpha0.5__seed2.json` | `be881ac6…` | `1118e7fa…` | `8e0db0be…` | 1438483982 | -Primary selection metric: **validation per-client-mean rare-class macro-F1** (per-client -rare F1 on each client's validation eval partition, averaged over clients with rare -classes). Global union-rare validation metrics also shown. +Pairing proof: round-0 snapshot byte-identical across the five betas **and** +`original_protean` within each seed — seed 0 `e27f41b3a62eb83c…`, seed 1 +`b7f61c995e97474c…`, seed 2 `5b749f4185e11fc7…`. `train_support_Nij.csv`, +`characterization.csv`, `rare_unseen.csv` identical across each seed's runs. +`manifest_reload_ok = true` everywhere. -| beta | val rareMacroF1 (pc mean) | val rareMacroF1 (global) | val rareRecall | val macroF1 | val acc | val wF1 | -|---|---|---|---|---|---|---| -| 0.00 | 0.557340 | 0.611641 | 0.884184 | 0.708929 | 0.891038 | 0.917741 | -| 0.25 | 0.571787 | 0.629475 | 0.888488 | 0.723431 | 0.902953 | 0.925933 | -| **0.50** | **0.585762** | **0.641709** | 0.887173 | **0.731166** | 0.897831 | **0.926005** | -| 0.75 | 0.578003 | 0.637107 | 0.873910 | 0.726603 | 0.887894 | 0.921696 | -| 1.00 | 0.551443 | 0.615307 | 0.847043 | 0.715951 | 0.883263 | 0.921182 | +## 5. Validation results (USED FOR SELECTION) and pooled selection -**Selected beta = 0.50** (best among tested candidates under this validation condition; -no tie; both validation readings agree). Test was **not** used for selection. +Primary metric: validation per-client-mean rare-class macro-F1. All 10 clients eligible +in every seed. Full per-seed table: +`artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_per_seed.csv`. -## 6. Final test evaluation of the selected beta +Per-seed validation per-client-mean rare-class macro-F1: -| metric | original_protean | support_weighted beta=0.50 | -|---|---|---| -| rare-class macro-F1 | 0.609620 | 0.640829 | -| rare-class recall | 0.871487 | 0.871583 | -| overall macro-F1 | 0.707701 | 0.730743 | -| accuracy | 0.890432 | 0.898158 | -| weighted-F1 | 0.917585 | 0.926333 | +| beta | seed 0 | seed 1 | seed 2 | +|---|---|---|---| +| 0.00 | 0.557340 | 0.628624 | 0.447118 | +| 0.25 | 0.571787 | 0.636205 | 0.458310 | +| 0.50 | 0.585762 | 0.645189 | 0.471015 | +| 0.75 | 0.578003 | 0.655160 | 0.504937 | +| 1.00 | 0.551443 | 0.683117 | 0.485877 | -All five candidates' test metrics (transparency only) are in -`docs/research/PAIRED_BETA_VALIDATION.md` §B.2. No superiority is claimed: single -dataset / alpha / seed. +Pooled across seeds (`...__validation_pooled.csv`): -## 7. `beta = 0` equivalence result (important) +| beta | pooled mean | std (sample) | min | max | selected | +|---|---|---|---|---|---| +| 0.00 | 0.544361 | 0.091447 | 0.447118 | 0.628624 | | +| 0.25 | 0.555434 | 0.090068 | 0.458310 | 0.636205 | | +| 0.50 | 0.567322 | 0.088539 | 0.471015 | 0.645189 | | +| **0.75** | **0.579367** | **0.075121** | 0.504937 | 0.655160 | **yes** | +| 1.00 | 0.573479 | 0.100450 | 0.485877 | 0.683117 | | -`support_weighted beta=0` vs frozen `original_protean`: +- **PRIMARY selected beta (`beta_star`) = 0.75**, max pooled validation per-client-mean + rare-class macro-F1; **no tie-break used**; `test_used_for_selection = false` for all 15 + runs. The secondary pooled recall is highest at beta=0.25 (recall is only a tie-break). +- **SECONDARY per-seed diagnostics only:** seed 0 → 0.50, seed 1 → 1.00, seed 2 → 0.75. + These are stability diagnostics, not the condition's selection rule. -- Same frozen manifest and split: **yes**. -- Same initialisation/round-0 setup: **yes** (round-0 snapshot byte-identical). -- Training behaviour: **mathematically identical, not bitwise**. `original_protean` - computes `Σ(C_ij/count)`; `support_weighted beta=0` computes `Σ((1/count)·C_ij)`. - Per-element difference ≤ `2.2e-16`–`4.4e-16` (~1 ULP, relative ~1e-16). -- Validation/test metrics: **not identical** (baseline slightly lower; e.g. test - accuracy 0.890432 vs 0.891974). The ~1 ULP aggregation difference feeds back into - round-1 prototype-alignment training and the trajectories diverge slowly from round 1. -- Deterministic artifacts identical: round-0 snapshot, `train_support_Nij.csv`, - `characterization.csv`, `rare_unseen.csv`; `history.json`/per-client/per-class differ. +## 6. Final test evaluation of the pooled-selected beta -This is floating-point arithmetic ordering, not a semantic change; no code was altered to -force bitwise equality. **Conditional on the shared `S_j` reproduction assumption** and -does not prove `beta = 0` reproduces original PROTEAN. +`beta_star = 0.75` was fixed from pooled validation before test was read. -## 8. Direct prototype-effect findings (fixed round-0 snapshot, no retraining) +| metric (test) | seed 0 | seed 1 | seed 2 | mean ± std | +|---|---|---|---|---| +| rare-class macro-F1 | 0.636189 | 0.671937 | 0.688381 | 0.665503 ± 0.026685 | +| rare-class recall | 0.857543 | 0.851883 | 0.878321 | 0.862583 ± 0.013921 | +| macro-F1 | 0.726298 | 0.729015 | 0.738194 | 0.731169 ± 0.006234 | +| accuracy | 0.888670 | 0.925493 | 0.905012 | 0.906392 ± 0.018450 | +| weighted-F1 | 0.922278 | 0.944734 | 0.933884 | 0.933632 ± 0.011230 | -At `beta = 1.0`, largest `||Cbar_j(1)-Cbar_j(0)||`: -`Weaponization` 3.559 (rel 0.577; most support-imbalanced, cv 2.10, max/min 1,800), -`C&C` 2.156, `Lateral_movement` 2.069, `Exploitation` 1.934, `Tampering` 1.737, -`Exfiltration` 1.547; smallest: `Normal` 0.990 (cv 0.76), `RDOS` 1.286, -`Reconnaissance` 1.184. Every class has `|S_j| ≥ 8` and unequal support, so no class has -zero direct effect. Outputs: -`artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0__prototype_snapshot__prototype_effect.csv` -and `...__prototype_weights.csv`. Displacement is a direct-aggregation measure only, not -a detection result. +`original_protean` for the same seeds/condition: -## 9. Client-level findings (test, vs beta=0) +| metric (test) | seed 0 | seed 1 | seed 2 | mean ± std | +|---|---|---|---|---| +| rare-class macro-F1 | 0.609620 | 0.655665 | 0.662275 | 0.642520 ± 0.028683 | +| rare-class recall | 0.871487 | 0.889365 | 0.870018 | 0.876957 ± 0.010771 | +| macro-F1 | 0.707701 | 0.709176 | 0.710017 | 0.708965 ± 0.001172 | +| accuracy | 0.890432 | 0.900885 | 0.907045 | 0.899454 ± 0.008398 | +| weighted-F1 | 0.917585 | 0.929132 | 0.927481 | 0.924733 ± 0.006245 | -Full paired table: `artifacts/analysis/xiiotid__alpha0.5__seed0__paired_client_beta_metrics.csv`. -At `beta=0.50` six of ten clients improve; only client 3 is materially negative (−0.068). -At `beta=0.75/1.00` several clients lose, with client 3 worst (−0.257/−0.342). The -`min positive N_ij` vs ΔF1 relationship is **not monotonic** in this single condition: -client 3 (`min N_ij=140`) is the worst loser while client 5 (`min N_ij=2`) is the biggest -gainer. Descriptive only; no causal claim. +Test columns are **descriptive only** and played no part in selection. No superiority or +generalisation is claimed: single dataset, single alpha, three project-defined seeds, and +between-seed spread ~ between-method difference. + +## 7. `beta = 0` equivalence result (seeds 1 and 2) + +For seeds 1 and 2, `support_weighted beta=0` vs frozen `original_protean`: + +- Same manifest/split/hashes: **yes**. +- Round-0 snapshot byte-identical: **yes** (seed 1 `b7f61c99…`, seed 2 `5b749f41…`). +- `train_support_Nij.csv`, `characterization.csv`, `rare_unseen.csv`: **identical**. +- Aggregation: **mathematically equivalent, not bitwise** (`Σ(C/count)` vs + `Σ((1/count)·C)`, ~1 ULP ordering difference that feeds back through round≥1). +- Test/history/per-client/per-class: **differ slightly**. +- Classification: **mathematically equivalent + floating-point numerical divergence; + no semantic discrepancy** in either seed. Same result as Session 5 seed 0. + +Conditional on the shared `S_j` reproduction assumption; not proof `beta=0` reproduces +original PROTEAN. + +## 8. Direct prototype-effect findings (fixed round-0 snapshots, no retraining) + +`||Cbar_j(1) - Cbar_j(0)||`, cross-seed mean (std): + +- Largest means: `Lateral_movement` 2.724 (1.035), `RDOS` 2.631 (1.165), + `Weaponization` 2.561 (1.228), `Tampering` 2.354 (1.165), `Exploitation` 2.344 (0.412). +- Smallest: `Reconnaissance` 1.002 (0.261), `Normal` 1.557 (0.548), `C&C` 1.694 (0.505). +- Every class moves for beta>0 (all classes have unequal support); the **rank order is + seed-sensitive** (seed 0 top = Weaponization, seed 1 top = RDOS, seed 2 top = + Lateral_movement). Displacement is a direct-aggregation measure only, not a detection + result. Outputs: `...__prototype_effect_pooled.csv` and the per-seed + `...__prototype_snapshot__prototype_effect.csv` / `..._weights.csv`. + +## 9. Client-level findings (validation, vs beta = 0) + +Full detail: `...__multiseed__client_validation_detail.csv`; summary: +`...__client_validation_summary.csv`. + +- At `beta = 0.75`, improve/degrade/**unchanged** counts are 7/3/0 (seed 0), 8/1/1 + (seed 1, tie = client 7), 8/1/1 (seed 2, tie = client 4); mean Δ is positive in all + three (+0.021 / +0.027 / +0.058). +- At `beta = 1.00`, seed 0 flips to 4 improve / 6 degrade / 0 unchanged, while seed 1 + improves 9/10 with 1 unchanged (client 7) — the high-beta response is seed-dependent. +- `min positive attack N_ij` vs ΔF1 is **non-monotonic**: N=1 clients range −0.021…+0.041 + at beta=0.75; the largest gain (+0.177) is an N=10 client; low support neither + systematically benefits nor degrades. Client ids are different realisations across + seeds and are not treated as one population. Descriptive only. ## 10. Runtime / resources -CPU only (no GPU). Per run: 41–44 min wall-clock, ~733–740 % CPU, peak RSS ~6.21 GiB, -no swap, exit 0. Total ~3.5 h for the five paired candidates. Preprocessing/load -negligible. +CPU only, no GPU, no swap, exit 0 on all 18 runs (15 new-session + reused). Per-run +wall-clock 40:23–42:21; CPU 733–745 %; peak RSS 6.21 GiB (seed 0), 7.48 GiB (seed 1), +6.45 GiB (seed 2). New compute this session: 12 runs, ≈ 8 h 11 m wall-clock, run +sequentially (8 cores; concurrent runs would not reduce total CPU work). +Seed-0 numbers were reused and not re-run. Machine-readable per-run values: +`artifacts/analysis/xiiotid__alpha0.5__multiseed__runtime.csv` (added by the audit; +seed 0 sourced from Session-5 docs, seeds 1–2 from `/usr/bin/time -v` logs). ## 11. Methodological decisions / assumptions 1. **`S_j` participant/denominator rule** (`S_j = {i | N_ij > 0 and valid C_ij}`, equal - weights in the baseline) is a **REPRODUCTION ASSUMPTION**, not verified original - behaviour; material in 53/162 class-condition cells (32.7 %), 15/18 conditions - (`docs/baseline/BASELINE_VALIDATION.md` §D). Carry this caveat in all reporting. + weights in the baseline) is a **REPRODUCTION ASSUMPTION**; material in 53/162 + class-condition cells (32.7 %), 15/18 conditions (`BASELINE_VALIDATION.md` §D). + Carry this caveat in all reporting. 2. `N_ij` is class-specific sample support, not prototype quality. 3. Beta is one value for all classes/clients; candidates frozen at `{0,.25,.5,.75,1}`. -4. `lambda=1.0`, Adam, lr 1e-3, batch 256 remain PROJECT DECISIONS. -5. **New this session (protocol clarification, documented in - `EXPERIMENT_PROTOCOL.md` §8):** candidate validation metric = final-model validation - metrics; primary selection = per-client-mean rare-class macro-F1 using a deterministic - Dirichlet validation eval partition (mirrors the existing test per-client procedure); - tie-break = higher val rare macro-F1 → higher val rare recall → smaller beta. -6. No `equal_available` mode was added. +4. **PRIMARY selection = pooled validation across clients and seeds**; per-seed selection + is secondary stability diagnostics; test used only after the pooled beta is fixed. + Documented in `EXPERIMENT_PROTOCOL.md` §8 (updated this session). +5. The union of rare classes (and global rare support) differs by seed because partitions + differ; only the per-client-mean metric is directly comparable across seeds. +6. `lambda=1.0`, Adam, lr 1e-3, batch 256 remain PROJECT DECISIONS. No `equal_available` + mode was added. ## 12. Unresolved questions / blockers 1. `S_j` reproduction ambiguity — unresolved; no official PROTEAN code exists. -2. `beta = 0` is not bitwise identical to `original_protean` (float ordering). Whether the - thesis should report this explicitly, or whether an equivalence check should use a - tolerance, is a documentation decision for later. -3. Single condition only; no multi-seed/multi-alpha evidence yet. +2. **Pooled beta selection is statistically weak here:** pooled means span only + 0.544→0.579 while within-beta across-seed std is 0.075–0.100; three seeds give a + standard error of the same order as the between-beta spread. The beta effect is small + relative to seed variance. +3. `beta = 0` is not bitwise identical to `original_protean`; documentation decision on how + to report this remains open. 4. Session-2 manifests still store `preprocessing_hash = null` / `config_hash = null`. +5. Only alpha=0.50, X-IIoTID, three seeds measured. + +**Audit status (see `docs/research/SESSION6_AUDIT.md`).** No unresolved issue that could +change `beta_star` or the scientific interpretation. One inherent audit limitation: the +trained model is not persisted, so per-client prediction metrics cannot be recomputed +without retraining (forbidden); they were verified via defining source code, exact +agreement of their mean with `val_selection.json`, and exact agreement of the global +union-rare metric recomputed from `val_confusion_matrix.json`. Pooled std uses the +**sample** convention (ddof=1), labelled "(sample)" in the report. ## 13. Git status -No commit or push was made. `git status --short`: +No commit or push was made. `git status --short` (results/ is ignored): ``` - M scripts/run_baseline.py -?? artifacts/analysis/xiiotid__alpha0.5__seed0__paired_client_beta_metrics.csv -?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0__prototype_snapshot.json -?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0__prototype_snapshot__prototype_effect.csv -?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0__prototype_snapshot__prototype_weights.csv -?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0.25__prototype_snapshot.json -?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0.5__prototype_snapshot.json -?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0.75__prototype_snapshot.json -?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta1__prototype_snapshot.json + M docs/handoff/SESSION_HANDOFF.md + M docs/research/EXPERIMENT_PROTOCOL.md +?? docs/research/MULTISEED_BETA_VALIDATION.md +?? docs/research/SESSION6_AUDIT.md +?? scripts/analyze_multiseed_beta.py +?? artifacts/analysis/xiiotid__alpha0.5__multiseed__*.csv / __selection_summary.json +?? artifacts/analysis/xiiotid__alpha0.5__seed{1,2}__*__prototype_snapshot.json +?? artifacts/analysis/xiiotid__alpha0.5__seed{1,2}__support_weighted__beta0__prototype_snapshot__{prototype_effect,prototype_weights}.csv +?? artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta{0.25,0.5,0.75,1}__prototype_snapshot.json (pre-existing from Session 5) ``` -`planningawal/` and both raw datasets are unchanged; `results/` and -`artifacts/processed/` are ignored; `artifacts/analysis/` is tracked. +`planningawal/` and both raw datasets are unchanged; `results/` is gitignored. +`HEAD = 50f2da72166e4f609077d65b4916f745da5e8e3b`. No commit or push was made. ## 14. Files created / modified this session Created: -- `docs/research/PAIRED_BETA_VALIDATION.md` -- `artifacts/analysis/xiiotid__alpha0.5__seed0__paired_client_beta_metrics.csv` -- five `artifacts/analysis/...__support_weighted__beta*__prototype_snapshot.json` -- two `artifacts/analysis/...__prototype_snapshot__prototype_effect.csv` / `__prototype_weights.csv` +- `docs/research/MULTISEED_BETA_VALIDATION.md` +- `docs/research/SESSION6_AUDIT.md` (post-experiment audit) +- `scripts/analyze_multiseed_beta.py` +- `artifacts/analysis/xiiotid__alpha0.5__multiseed__*` (validation per-seed/pooled, + client summary/detail/deltas, prototype-effect pooled, test descriptive, selection + summary, and `...__runtime.csv` added by the audit) +- seed-1/seed-2 `results/...` run dirs (gitignored) and their snapshots / effect CSVs Modified: -- `scripts/run_baseline.py` (added validation-only evaluation artifacts; test/training paths untouched) -- `docs/research/EXPERIMENT_PROTOCOL.md` (§8 Session-5 clarification) +- `docs/research/EXPERIMENT_PROTOCOL.md` (§8 selection hierarchy clarified) - `docs/handoff/SESSION_HANDOFF.md` (this file) +Reused (not regenerated): all seed-0 Session-5 runs and artifacts. + ## 15. Exact recommended next phase -**Recommendation: B — checkpoint Session 5, then proceed to a controlled -multi-seed/multi-alpha experiment.** The protocol ran cleanly (validation/test separation -held, artifacts reproducible, pairing proven by byte-identical round-0 snapshots, no test -leakage into selection), so the next step is to scale to seeds `{0,1,2}` at a small alpha -set, still paired. Do **not** run the entire `dataset × alpha × seed × beta` grid at once; -grow it in controlled, inspectable increments. +**Session 6 is safe to checkpoint** (audit PASS WITH MINOR CORRECTIONS; no issue affects +`beta_star`). The protocol is **mechanically** ready to scale, but Session 6 shows the beta +effect at alpha=0.50 is small relative to seed-to-seed variance. Recommended, in order: -Before scaling: decide how to report the float-level `beta = 0` vs `original_protean` -non-identity (§7), and whether to re-run `original_protean` per condition with the new -validation artifacts (the existing baseline run predates the validation-artifact change). +1. **Do not treat beta=0.75 as a settled optimum.** Report it as "best among tested + candidates under pooled validation for X-IIoTID alpha=0.50". +2. Before expanding alpha, decide the interpretation question: is the goal to characterise + small beta effects against seed noise (requires more seeds), or to run alpha=0.25/0.75 + as a paired characterisation (accepting the same weak-effect caveat)? +3. If expanding, run alpha=0.25 then alpha=0.75 for seeds `{0,1,2}`, the five betas each, + reusing frozen manifests, sequentially (~8 h per alpha), with the same pooled + validation-only selection. Extend `scripts/analyze_multiseed_beta.py` with `--alpha`. +4. Do **not** launch 5G-NIDD or the full grid until the alpha results are inspected. ## 16. Important commands @@ -220,14 +299,18 @@ validation artifacts (the existing baseline run predates the validation-artifact # Tests (75 expected) .venv/bin/python -m pytest -# One full paired beta run (CPU, ~42 min, ~6.2 GiB) -.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 0 \ +# One full paired beta run (CPU, ~41 min, ~6.5-7.5 GiB) +.venv/bin/python scripts/run_baseline.py --dataset xiiotid --alpha 0.5 --seed 1 \ --clients 10 --rounds 10 --local-epochs 3 \ --aggregation-mode support_weighted --beta 0.5 +# Pooled multi-seed validation analysis (read-only, no retraining) +.venv/bin/python scripts/analyze_multiseed_beta.py --dataset xiiotid --alpha 0.5 \ + --seeds 0 1 2 + # Direct prototype-effect analysis (no retraining) .venv/bin/python scripts/analyze_prototype_effect.py \ - --snapshot artifacts/analysis/xiiotid__alpha0.5__seed0__support_weighted__beta0__prototype_snapshot.json \ + --snapshot artifacts/analysis/xiiotid__alpha0.5__seed1__support_weighted__beta0__prototype_snapshot.json \ --class-names "C&C,Exfiltration,Exploitation,Lateral_movement,Normal,RDOS,Reconnaissance,Tampering,Weaponization" \ --output-dir artifacts/analysis ``` @@ -235,13 +318,13 @@ validation artifacts (the existing baseline run predates the validation-artifact ## 17. Recommended next-session prompt ``` -You are continuing the perdalam-protean research repository (Session 6). +You are continuing the perdalam-protean research repository (Session 7). FIRST, read completely: 1. docs/handoff/SESSION_HANDOFF.md - 2. docs/research/PAIRED_BETA_VALIDATION.md - 3. docs/research/EXPERIMENT_PROTOCOL.md - 4. docs/research/BETA_AGGREGATION_IMPLEMENTATION.md + 2. docs/research/MULTISEED_BETA_VALIDATION.md + 3. docs/research/SESSION6_AUDIT.md + 4. docs/research/EXPERIMENT_PROTOCOL.md 5. docs/baseline/BASELINE_VALIDATION.md Critical rules: @@ -253,18 +336,22 @@ Critical rules: - Do NOT add an executable equal_available mode. - Do NOT launch the full alpha x seed x dataset x beta grid. - Beta selection uses validation metrics only; test never influences selection. + - PRIMARY selection is pooled across clients and seeds; per-seed selection is + diagnostic only. -Context: Session 5 ran one paired full condition (xiiotid, alpha=0.5, seed=0) across -beta in {0,0.25,0.5,0.75,1}. Selected beta = 0.50 on validation per-client-mean -rare-class macro-F1; test rare-class macro-F1 0.640829 vs original_protean 0.609620. -beta=0 is mathematically but not bitwise equal to original_protean (~1 ULP aggregation -ordering). scripts/run_baseline.py now writes validation-only artifacts. +Context: Session 6 completed X-IIoTID alpha=0.50 for seeds {0,1,2} across beta in +{0,0.25,0.5,0.75,1}. Seed 0 was reused from Session 5. Pooled validation selected +beta=0.75 (val per-client-mean rare macro-F1 0.579367); per-seed diagnostics were +0.50/1.00/0.75. Pooled-selected test rare macro-F1 0.665503 +/- 0.026685 vs +original_protean 0.642520 +/- 0.028683. beta=0 is mathematically but not bitwise equal +to original_protean (float ordering), no semantic discrepancy. Between-seed variance is +comparable to the between-beta spread, so the selection is weak. A strict independent +audit (docs/research/SESSION6_AUDIT.md) returned PASS WITH MINOR CORRECTIONS and +independently reproduced beta_star=0.75 from validation artifacts alone. -Task: scale the paired protocol to a controlled multi-seed/multi-alpha set (e.g. seeds -{0,1,2} at alpha 0.50, reusing frozen manifests; grow only as compute allows), keeping -validation-only beta selection and reporting all candidates. Decide/document how to -report the beta=0 vs original_protean float non-identity. Update docs and -docs/handoff/SESSION_HANDOFF.md. Measure runtime before scaling further. - -Use .venv/bin/python for all Python commands. Do not commit or push unless asked. +Task: decide and document the alpha-expansion plan (recommended: alpha=0.25 then 0.75, +seeds {0,1,2}, five betas each, sequential, reusing frozen manifests, pooled +validation-only selection) OR argue for more seeds at alpha=0.50 first. Do not launch +5G-NIDD or the full grid. Update docs and docs/handoff/SESSION_HANDOFF.md. Use +.venv/bin/python. Do not commit or push unless asked. ``` diff --git a/docs/research/EXPERIMENT_PROTOCOL.md b/docs/research/EXPERIMENT_PROTOCOL.md index d40b508..2d65e09 100644 --- a/docs/research/EXPERIMENT_PROTOCOL.md +++ b/docs/research/EXPERIMENT_PROTOCOL.md @@ -121,19 +121,40 @@ Persisted metadata (minimum one row per `(dataset, alpha, seed, client, class)`) ## 8. Validation-based beta selection +Selection hierarchy (strictly ordered). Test data is never used in PRIMARY or +SECONDARY. + +**PRIMARY — pooled validation across clients and random seeds for one non-IID +condition.** For each beta, average the per-client rare-class macro-F1 over eligible +clients within each seed, then average those seed-level scores over the condition's +seeds. Select `beta_star = argmax` of this pooled score using validation only. This +is the beta reported for the condition. Tie-break: higher pooled per-client-mean +rare-class macro-F1 -> higher pooled validation rare-class recall -> lower +across-seed std -> smaller beta. + +**SECONDARY — per-seed beta selection, for stability analysis only.** For each seed +independently, choose the validation-best beta with the same metric. Per-seed choices +are diagnostics of seed sensitivity and are **not** the condition's selection rule. + +**TEST — used only after `beta_star` is fixed from pooled validation.** Evaluate the +fixed `beta_star` on test, and report all candidates' test metrics for transparency. +No test metric may influence `beta_star`. + Procedure, strictly ordered: 1. **Candidate evaluation (validation only).** For each beta candidate, train with the paired - manifest and compute validation metrics. Primary criterion: rare-class macro-F1, computed - per client that has rare classes, then averaged over those clients. -2. **Beta selection.** Choose the beta with the best mean validation rare-class macro-F1 (ties - broken by smaller beta). The result is "best among tested candidates under this - procedure", never a global optimum. + manifest and compute validation metrics. Primary metric: per-client rare-class macro-F1, + computed per client that has rare classes, then averaged over those clients. +2. **PRIMARY beta selection.** Choose the beta with the best pooled-across-seed mean + validation per-client rare-class macro-F1 (tie-break per the hierarchy above). The result + is "best among tested candidates under this procedure", never a global optimum. 3. **Final test evaluation.** Evaluate the selected beta on the test split. Report overall macro-F1, rare-class recall, rare-class macro-F1, and per-client results, plus the test results of all candidates for transparency. -Test data is never used in steps 1–2. +Session-5 (single-seed) used the one-seed special case of step 2. Session 6 applies the +full pooled rule across seeds `{0,1,2}` for X-IIoTID alpha=0.50; see +`docs/research/MULTISEED_BETA_VALIDATION.md`. **Session-5 clarification (implementation detail, not a design change).** Running the protocol surfaced two concrete points that the original text left implicit. @@ -151,7 +172,9 @@ They do not alter the frozen design; they pin how step 1 is computed. evaluation, applied to the validation split (`split_name="validation"`); this is a PROJECT DECISION, paired across all beta candidates and seeds. 2. **Tie-break order.** Higher validation rare-class macro-F1, then higher - validation rare-class recall, then smaller beta. + validation rare-class recall, then lower across-seed std (pooled PRIMARY only), + then smaller beta. The pooled PRIMARY tie-break is authoritative; the per-seed + SECONDARY uses the same order without the std step. `scripts/run_baseline.py` writes `val_global_metrics.json`, `val_per_class.csv`, `val_per_client.csv` and `val_selection.json` to make this explicit; the test diff --git a/docs/research/MULTISEED_BETA_VALIDATION.md b/docs/research/MULTISEED_BETA_VALIDATION.md new file mode 100644 index 0000000..8dbb0b1 --- /dev/null +++ b/docs/research/MULTISEED_BETA_VALIDATION.md @@ -0,0 +1,399 @@ +# Multi-Seed Beta Validation (Session 6) + +Status: **three-seed paired protocol completed for one non-IID condition** with +pooled-across-seed validation-only beta selection, followed by a single final test +evaluation of the pooled-selected beta. This is a **stability / protocol-scaling +experiment**, not a sweep and not evidence of a global optimum. + +Companion documents: + +- `docs/research/EXPERIMENT_PROTOCOL.md` — frozen protocol and selection hierarchy. +- `docs/research/PAIRED_BETA_VALIDATION.md` — Session-5 single-seed condition. +- `docs/research/BETA_AGGREGATION_IMPLEMENTATION.md` — beta mode implementation. +- `docs/baseline/BASELINE_VALIDATION.md` — frozen baseline evidence. +- `docs/handoff/SESSION_HANDOFF.md` — continuation point. + +--- + +## A. Scope + +| Field | Value | +|---|---| +| dataset | `xiiotid` (X-IIoTID) | +| alpha | 0.50 | +| seeds | 0, 1, 2 | +| aggregation | `original_protean` (per seed) + `support_weighted` for `beta ∈ {0, 0.25, 0.5, 0.75, 1}` | +| runs | 15 `support_weighted` + 3 `original_protean` | +| reused | seed 0 (all five betas + `original_protean`) from Session 5, not re-run | +| new this session | seeds 1 and 2 (five betas each + `original_protean` each) | + +No other alpha, dataset, seed, mode or beta was run. `equal_available` was not added. + +--- + +## B. Frozen protocol + +Training config is identical to Session 5 and to the frozen protocol +(`EXPERIMENT_PROTOCOL.md` §14): 10 clients, 10 rounds, 3 local epochs, batch 256, +Adam `lr=1e-3`, `mu=0.1`, `lambda=1.0`; model architecture, preprocessing, +train/validation/test split recipe, partition recipe, loss terms and inference +unchanged. No training/aggregation source was changed this session; the only code +changes are the read-only analysis script `scripts/analyze_multiseed_beta.py` (and +the Session-5 `scripts/run_baseline.py` validation-artifact emission already at HEAD). + +### B.1 Manifest identity (reused, never regenerated) + +| seed | manifest path | content hash | partition hash | split hash | rng seed | train/val/test sizes | +|---|---|---|---|---|---|---| +| 0 | `artifacts/partitions/xiiotid__alpha0.5__seed0.json` | `e81a6538…` | `2465485539…` | `d0acc0c1…` | 2009062324 | 569,909 / 81,414 / 162,831 | +| 1 | `artifacts/partitions/xiiotid__alpha0.5__seed1.json` | `54ea1df1…` | `4061c5a1…` | `880a9f82…` | 621679668 | 569,909 / 81,414 / 162,831 | +| 2 | `artifacts/partitions/xiiotid__alpha0.5__seed2.json` | `be881ac6…` | `1118e7fa…` | `8e0db0be…` | 1438483982 | 569,909 / 81,414 / 162,831 | + +`manifest_reload_ok = true` and the per-seed manifest content hash is identical +across all six runs (five betas + `original_protean`) of that seed, so pairing holds. + +### B.2 Build-level pairing proof (round-0 snapshot byte-identity) + +The round-0 prototype snapshot is byte-identical across the five beta runs **and** +`original_protean` within each seed: + +| seed | SHA-256 of round-0 snapshot (all six runs) | +|---|---| +| 0 | `e27f41b3a62eb83c…` | +| 1 | `b7f61c995e97474c…` | +| 2 | `5b749f4185e11fc7…` | + +`train_support_Nij.csv`, `characterization.csv` and `rare_unseen.csv` are also +byte-identical across the runs of each seed. Beta only changes server aggregation. + +### B.3 Rare/unseen derivation is per-seed + +Rare classes are derived from the manifest training split (`EXPERIMENT_PROTOCOL.md` +§7). The Dirichlet partitions differ by seed, so the union of rare classes differs: + +| seed | per-client rare count | union of rare classes | +|---|---|---| +| 0 | 2 per client (10/10 clients) | C&C, Exfiltration, Exploitation, Lateral_movement, Tampering, Weaponization | +| 1 | 2 per client (10/10 clients) | + RDOS | +| 2 | 2 per client (10/10 clients) | + RDOS | + +Integrity checks (all pass): rare classes always have `N_ij > 0`; unseen attack +classes always have `N_ij = 0`; the benign class (`Normal`) is never rare or unseen; +rare and unseen are never merged; every client has exactly two rare classes. The +different union rare set is why global test rare support is 25,454 (seed 0) versus +53,706 (seeds 1, 2); the primary per-client-mean metric uses each client's own rare +classes and so is comparable across seeds. + +### B.4 Selection rule + +- **PRIMARY:** pooled-across-seed validation per-client-mean rare-class macro-F1 + (mean of each seed's per-client-mean score over seeds `{0,1,2}`), selected with + validation only. +- **SECONDARY (diagnostic):** per-seed validation selection, for stability analysis. +- **TEST:** computed for all candidates by the frozen pipeline but used only after + the pooled beta is fixed. No test metric influenced selection. + +--- + +## C. Per-seed candidate validation results + +`val rareMacroF1 (pc mean)` is the primary metric: each client scored on its +deterministic Dirichlet **validation** eval partition with that client's rare class +ids, then averaged over the 10 eligible clients. All numbers are validation only. + +### Seed 0 (reused from Session 5) + +| beta | eligible | val rareMacroF1 (pc mean) | val rareMacroF1 (global) | val rareRecall (global) | val macroF1 | val acc | +|---|---|---|---|---|---|---| +| 0.00 | 10 | 0.557340 | 0.611641 | 0.884184 | 0.708929 | 0.891038 | +| 0.25 | 10 | 0.571787 | 0.629475 | 0.888488 | 0.723431 | 0.902953 | +| **0.50** | 10 | **0.585762** | **0.641709** | 0.887173 | **0.731166** | 0.897831 | +| 0.75 | 10 | 0.578003 | 0.637107 | 0.873910 | 0.726603 | 0.887894 | +| 1.00 | 10 | 0.551443 | 0.615307 | 0.847043 | 0.715951 | 0.883263 | + +### Seed 1 (new) + +| beta | eligible | val rareMacroF1 (pc mean) | val rareMacroF1 (global) | val rareRecall (global) | val macroF1 | val acc | +|---|---|---|---|---|---|---| +| 0.00 | 10 | 0.628624 | 0.656935 | 0.890520 | 0.709324 | 0.899612 | +| 0.25 | 10 | 0.636205 | 0.666240 | 0.884109 | 0.722367 | 0.919203 | +| 0.50 | 10 | 0.645189 | 0.667383 | 0.863833 | 0.724212 | 0.922593 | +| 0.75 | 10 | 0.655160 | 0.670472 | 0.855590 | 0.727134 | 0.923760 | +| **1.00** | 10 | **0.683117** | **0.685707** | 0.856876 | **0.739006** | **0.924497** | + +### Seed 2 (new) + +| beta | eligible | val rareMacroF1 (pc mean) | val rareMacroF1 (global) | val rareRecall (global) | val macroF1 | val acc | +|---|---|---|---|---|---|---| +| 0.00 | 10 | 0.447118 | 0.658720 | 0.880662 | 0.708464 | 0.907891 | +| 0.25 | 10 | 0.458310 | 0.669242 | 0.885401 | 0.716162 | 0.907018 | +| 0.50 | 10 | 0.471015 | 0.677892 | 0.887958 | 0.730956 | 0.920259 | +| **0.75** | 10 | **0.504937** | **0.686124** | 0.887725 | **0.735912** | 0.902351 | +| 1.00 | 10 | 0.485877 | 0.676314 | 0.877857 | 0.726922 | 0.883534 | + +Machine-readable: `artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_per_seed.csv`. + +--- + +## D. Per-seed diagnostic beta selections (SECONDARY, stability only) + +| seed | selected beta | val rareMacroF1 (pc mean) | tie-break used | test_used_for_selection | +|---|---|---|---|---| +| 0 | 0.50 | 0.585762 | no | false | +| 1 | 1.00 | 0.683117 | no | false | +| 2 | 0.75 | 0.504937 | no | false | + +**These per-seed choices are diagnostic only.** They are not the primary selection +rule for this condition; they measure seed sensitivity. The preferred beta is +seed-dependent: 0.50, 1.00, 0.75. `val_selection.json` records +`"test_used_for_selection": false` and `"selection_split": "validation"` for all 15 +`support_weighted` runs. + +--- + +## E. PRIMARY pooled-across-seed validation selection + +For each beta, `score_seed(beta)` is the seed-level validation per-client-mean +rare-class macro-F1; `score_pooled(beta)` is its mean over seeds `{0,1,2}`. + +| beta | mean (pooled) | std (sample, ddof=1) | std (pop) | min | max | pooled val rareRecall (pc mean) | selected | +|---|---|---|---|---|---|---|---| +| 0.00 | 0.544361 | 0.091447 | 0.074666 | 0.447118 | 0.628624 | 0.839088 | | +| 0.25 | 0.555434 | 0.090068 | 0.073540 | 0.458310 | 0.636205 | **0.841278** | | +| 0.50 | 0.567322 | 0.088539 | 0.072292 | 0.471015 | 0.645189 | 0.838872 | | +| **0.75** | **0.579367** | **0.075121** | 0.061336 | 0.504937 | 0.655160 | 0.829088 | **yes** | +| 1.00 | 0.573479 | 0.100450 | 0.082017 | 0.485877 | 0.683117 | 0.820763 | | + +- **PRIMARY selected beta (`beta_star`) = 0.75.** +- **Criterion:** maximum pooled validation per-client-mean rare-class macro-F1. +- **Tie-break:** not used (no exact tie). The secondary pooled recall is *highest* at + `beta = 0.25`, but recall is only a tie-break and the primary metric governs. +- **`test_used_for_selection = false`.** Test metrics were not inspected until + `beta = 0.75` was fixed. +- **Interpretation:** `beta = 0.75` is the *best among the tested candidates under + pooled validation across the three project-defined seeds for X-IIoTID alpha=0.50.* + It is **not** a global optimum. + +Machine-readable: `artifacts/analysis/xiiotid__alpha0.5__multiseed__validation_pooled.csv` +and `...__selection_summary.json`. + +--- + +## F. Final test evaluation after the pooled selection + +`beta_star = 0.75` was fixed from pooled validation; only then were test metrics +read. Per-seed and mean ± std (sample) across the three seeds: + +| metric | seed 0 | seed 1 | seed 2 | mean ± std | +|---|---|---|---|---| +| test rare-class macro-F1 | 0.636189 | 0.671937 | 0.688381 | 0.665503 ± 0.026685 | +| test rare-class recall | 0.857543 | 0.851883 | 0.878321 | 0.862583 ± 0.013921 | +| test macro-F1 | 0.726298 | 0.729015 | 0.738194 | 0.731169 ± 0.006234 | +| test accuracy | 0.888670 | 0.925493 | 0.905012 | 0.906392 ± 0.018450 | +| test weighted-F1 | 0.922278 | 0.944734 | 0.933884 | 0.933632 ± 0.011230 | + +`original_protean` reference for the same three seeds and condition: + +| metric | seed 0 | seed 1 | seed 2 | mean ± std | +|---|---|---|---|---| +| test rare-class macro-F1 | 0.609620 | 0.655665 | 0.662275 | 0.642520 ± 0.028683 | +| test rare-class recall | 0.871487 | 0.889365 | 0.870018 | 0.876957 ± 0.010771 | +| test macro-F1 | 0.707701 | 0.709176 | 0.710017 | 0.708965 ± 0.001172 | +| test accuracy | 0.890432 | 0.900885 | 0.907045 | 0.899454 ± 0.008398 | +| test weighted-F1 | 0.917585 | 0.929132 | 0.927481 | 0.924733 ± 0.006245 | + +Descriptive difference (`beta_star` − `original_protean`) for test rare-class +macro-F1 is +0.026569 (seed 0), +0.016272 (seed 1), +0.026106 (seed 2); mean ++0.022982. **No superiority or generalisation is claimed:** this is a single +dataset, a single alpha, three project-defined seeds, and the between-seed spread +is comparable to the between-method difference. + +Test columns are descriptive only. They are also recorded in +`artifacts/analysis/xiiotid__alpha0.5__multiseed__test_descriptive.csv`, clearly +separated from the validation tables. + +--- + +## G. Direct prototype-effect analysis (fixed round-0 snapshots, no retraining) + +Computed with the production aggregation from each seed's fixed `{C_ij, N_ij}` +round-0 snapshot. `||Cbar_j(beta) - Cbar_j(0)||` at `beta = 1.0`: + +| class | mean disp | std across seeds | min | max | mean rel disp | typical `|S_j|` | +|---|---|---|---|---|---|---| +| Lateral_movement | 2.724 | 1.035 | 2.069 | 3.917 | 0.403 | 10 | +| RDOS | 2.631 | 1.165 | 1.286 | 3.322 | 0.352 | 10 | +| Weaponization | 2.561 | 1.228 | 1.189 | 3.559 | 0.433 | 10 | +| Tampering | 2.354 | 1.165 | 1.627 | 3.698 | 0.483 | 9–10 | +| Exploitation | 2.344 | 0.412 | 1.934 | 2.759 | 0.423 | 8–10 | +| Exfiltration | 2.020 | 0.502 | 1.547 | 2.547 | 0.297 | 9–10 | +| C&C | 1.694 | 0.505 | 1.155 | 2.156 | 0.319 | 8–10 | +| Normal | 1.557 | 0.548 | 0.990 | 2.084 | 0.292 | 10 | +| Reconnaissance | 1.002 | 0.261 | 0.703 | 1.184 | 0.169 | 10 | + +Findings (descriptive): + +- **Every class moves** for `beta > 0` in every seed, because every class has an + unequal support distribution across contributors (no class has equal `N_ij`). +- The **rank order of displacement is seed-sensitive**: seed 0's largest is + `Weaponization`, seed 1's is `RDOS`, seed 2's is `Lateral_movement`. `Exploitation` + and `Reconnaissance` are consistently smaller. +- **High-support-imbalance classes tend to move more** (e.g. `Weaponization`, + `RDOS`, `Tampering`, `Lateral_movement` have large max/min support ratios), but the + relationship is not exact and is seed-dependent. +- **Displacement is a direct-aggregation measure only** and is not interpreted as + improved or degraded detection. The end-to-end effect (§F) is reported separately. + +Outputs: `artifacts/analysis/xiiotid__alpha0.5__seed{0,1,2}__support_weighted__beta0__prototype_snapshot__prototype_effect.csv` +and `...__prototype_weights.csv`; cross-seed aggregate +`artifacts/analysis/xiiotid__alpha0.5__multiseed__prototype_effect_pooled.csv`. + +--- + +## H. Client-level effect (validation, relative to beta = 0) + +Full per-client table: `artifacts/analysis/xiiotid__alpha0.5__multiseed__client_validation_detail.csv`. +Summary: `...__client_validation_summary.csv`. + +| seed | beta | improve | degrade | unchanged | mean Δ | min Δ | max Δ | +|---|---|---|---|---|---|---|---| +| 0 | 0.25 | 9 | 1 | 0 | +0.0144 | −0.0048 | +0.0593 | +| 0 | 0.50 | 8 | 2 | 0 | +0.0284 | −0.0286 | +0.0926 | +| 0 | 0.75 | 7 | 3 | 0 | +0.0207 | −0.0358 | +0.0926 | +| 0 | 1.00 | 4 | 6 | 0 | −0.0059 | −0.0760 | +0.0414 | +| 1 | 0.25 | 6 | 2 | 2 | +0.0076 | −0.0045 | +0.0240 | +| 1 | 0.50 | 7 | 2 | 1 | +0.0166 | −0.0054 | +0.0424 | +| 1 | 0.75 | 8 | 1 | 1 | +0.0265 | −0.0030 | +0.0872 | +| 1 | 1.00 | 9 | 0 | 1 | +0.0545 | 0.0000 | +0.1927 | +| 2 | 0.25 | 7 | 1 | 2 | +0.0112 | −0.0007 | +0.0295 | +| 2 | 0.50 | 8 | 0 | 2 | +0.0239 | 0.0000 | +0.0697 | +| 2 | 0.75 | 8 | 1 | 1 | +0.0578 | −0.0019 | +0.1771 | +| 2 | 1.00 | 6 | 3 | 1 | +0.0388 | −0.4028 | +0.2676 | + +Descriptive observations: + +- At `beta = 0.75` the majority of eligible clients improve in every seed + (7/10, 8/10, 8/10), and the mean client delta is positive in all three seeds. +- The improvement fraction is **seed-dependent**: at `beta = 1.00` seed 0 degrades + (4 improve / 6 degrade) while seed 1 improves for 9/10 clients. +- **`min positive attack N_ij` vs ΔF1 is non-monotonic.** At `beta = 0.75`, clients + with `N_ij = 1` show deltas of −0.021, +0.029, +0.041; clients with `N_ij ≥ 130` + show +0.004 to +0.096; the largest gain (+0.177) is a client with `N_ij = 10`. Low + support neither systematically benefits nor systematically degrades. +- One extreme loss appears at `beta = 1.00`, seed 2 (client with `N_ij = 1`, + Δ = −0.403), showing that high beta can be unstable on specific clients. + +**Client ids are different realisations across seeds** and are not treated as the +same statistical population. This analysis is descriptive and non-causal. + +--- + +## I. Seed stability + +- **Did the seed-specific preferred beta vary?** Yes: 0.50 (seed 0), 1.00 (seed 1), + 0.75 (seed 2). +- **Was the pooled selection clear?** Arithmetically yes — `beta = 0.75` has the + highest pooled mean and the lowest across-seed std, with no tie. Statistically it + is **weak**: pooled means span only 0.544→0.579 (range ≈ 0.035), while the + within-beta across-seed std is 0.075–0.100. Between-seed variance dominates the + between-beta differences, so the pooled ordering is not strongly separated. +- **Are effect directions stable?** Partly. The pooled validation rare-class + macro-F1 rises from `beta = 0` to `0.75`, and the mean per-client delta is positive + for `0.25–0.75` in all seeds. But the *shape* and the best beta differ by seed + (seed 0 peaks at 0.50 and declines by 1.00; seed 1 increases monotonically; seed 2 + peaks at 0.75). Pooled validation rare-class recall actually decreases with beta. +- **Is variance large relative to mean differences?** Yes. With only three seeds the + standard error of the pooled mean is ≈ `std/√3` ≈ 0.043–0.058, of the same order as + the total spread between beta candidates. The apparent "preference" for 0.75 is + therefore not robustly distinguishable from noise in this condition. + +--- + +## J. Runtime and resources + +Measured with `/usr/bin/time -v`, CPU only, no GPU, no swap, exit status 0 for every +run. + +| seed | run | wall-clock | CPU % | peak RSS | +|---|---|---|---|---| +| 0 | `original_protean` (reused) | 42:06.78 | 740 % | 6.21 GiB | +| 0 | beta 0.00 | 41:17 | 740 % | 6.21 GiB | +| 0 | beta 0.25 | 42:11 | 738 % | 6.21 GiB | +| 0 | beta 0.50 | 42:57 | 737 % | 6.21 GiB | +| 0 | beta 0.75 | 43:45 | 733 % | 6.21 GiB | +| 0 | beta 1.00 | 42:08 | 737 % | 6.21 GiB | +| 1 | `original_protean` | 41:21.16 | 738 % | 7.48 GiB | +| 1 | beta 0.00 | 40:38.89 | 744 % | 7.48 GiB | +| 1 | beta 0.25 | 40:32.74 | 744 % | 7.48 GiB | +| 1 | beta 0.50 | 40:43.99 | 743 % | 7.48 GiB | +| 1 | beta 0.75 | 40:41.36 | 745 % | 7.48 GiB | +| 1 | beta 1.00 | 41:08.86 | 744 % | 7.48 GiB | +| 2 | `original_protean` | 41:09.70 | 743 % | 6.45 GiB | +| 2 | beta 0.00 | 40:40.14 | 742 % | 6.45 GiB | +| 2 | beta 0.25 | 41:25.22 | 740 % | 6.45 GiB | +| 2 | beta 0.50 | 40:23.42 | 742 % | 6.45 GiB | +| 2 | beta 0.75 | 41:05.82 | 740 % | 6.45 GiB | +| 2 | beta 1.00 | 42:20.66 | 741 % | 6.45 GiB | + +- Seed-0 runs were reused from Session 5 and were not re-run. +- New compute this session: 12 runs, ≈ 8 h 11 m wall-clock (seed-1 chain ≈ 4 h 04 m, + seed-2 chain ≈ 4 h 07 m), single-run sequential. +- RSS rises to ≈ 7.48 GiB for seed 1; this is dataset/partition dependent, not + beta dependent, and well within the 29 GiB host. +- Preprocessing/load remains negligible next to training. No warnings, no client + failures, 9/9 global prototypes in every round, no retries. + +--- + +## K. Limitations + +- **Only X-IIoTID**, **only alpha = 0.50**, **only three project-defined seeds** + (`{0,1,2}`). No cross-alpha and no cross-dataset conclusion. +- The `S_j` participant/denominator rule + (`S_j = {i | N_ij > 0 and valid C_ij}`, equal weights in the baseline) remains a + **REPRODUCTION ASSUMPTION**, material in 53/162 class-condition cells (32.7 %). + The `beta = 0` comparison against `original_protean` is conditional on both modes + sharing this rule and does **not** prove `beta = 0` reproduces original PROTEAN. +- Train/validation/test adaptation (70/10/20 stratified) differs from the paper's + literal 80/20; the validation eval partition is a project decision. +- **No global optimum claim.** `beta = 0.75` is only the best among the five tested + candidates under pooled validation for this one condition, and the between-seed + variance is comparable to the between-beta spread. +- The selected-beta test numbers are single-condition and must not be read as + generalisation; no test metric was used for selection. +- The union of rare classes (and hence global rare support) differs by seed because + partitions differ; only the per-client-mean metric is directly comparable across + seeds. +- No 5G-NIDD beta sweep was run. + +--- + +## L. `beta = 0` equivalence vs `original_protean` (seeds 1 and 2) + +| check | seed 1 | seed 2 | +|---|---|---| +| same manifest / split / hashes | yes | yes | +| round-0 snapshot byte-identical | yes (`b7f61c99…`) | yes (`5b749f41…`) | +| `train_support_Nij.csv`, `characterization.csv`, `rare_unseen.csv` identical | yes | yes | +| aggregation arithmetic | mathematically equivalent (`vec/count` vs `vec·(1/count)`) | same | +| test / history / per-client / per-class | differ slightly (float ordering fed back through round≥1) | same | + +Classified as **mathematically equivalent + floating-point numerical divergence**; +**no semantic discrepancy** was found in either seed. Mechanism is identical to +Session 5: round 0 is byte-identical, the ~1 ULP ordering difference enters the +round-1 prototype-alignment training and trajectories diverge slowly. + +--- + +## M. Analysis tooling + +Read-only, no retraining: + +- `scripts/analyze_multiseed_beta.py` — pooled validation tables, per-seed + diagnostic selection, client validation summary/detail, cross-seed prototype-effect + aggregation, test descriptive table, and `...__selection_summary.json`. +- `scripts/analyze_prototype_effect.py` — per-seed direct prototype-effect CSVs from + the fixed round-0 snapshot. + +Outputs under `artifacts/analysis/xiiotid__alpha0.5__multiseed__*` and the per-seed +`...__prototype_snapshot__prototype_effect.csv` / `...__prototype_weights.csv`. diff --git a/docs/research/SESSION6_AUDIT.md b/docs/research/SESSION6_AUDIT.md new file mode 100644 index 0000000..aba06e4 --- /dev/null +++ b/docs/research/SESSION6_AUDIT.md @@ -0,0 +1,319 @@ +# Session 6 Post-Experiment Audit + +Status: **PASS WITH MINOR CORRECTIONS.** Independent recomputation from machine-readable +artifacts confirms the Session 6 claims. No discovered issue changes `beta_star` or the +scientific interpretation. One provenance gap was closed (runtime CSV) and one handoff +wording nuance was clarified. **No training was run, no aggregation semantics changed, no +test-based selection occurred.** + +Audit performed: 2026-09-25. Audited HEAD: `50f2da72166e4f609077d65b4916f745da5e8e3b`. + +Companion documents: `docs/research/MULTISEED_BETA_VALIDATION.md`, +`docs/research/EXPERIMENT_PROTOCOL.md`, `docs/research/PAIRED_BETA_VALIDATION.md`, +`docs/baseline/BASELINE_VALIDATION.md`, `docs/handoff/SESSION_HANDOFF.md`. + +--- + +## A. Audit scope + +Independently verify the Session 6 claim for: + +- dataset `xiiotid`, alpha `0.50`, seeds `{0,1,2}`, beta `{0, 0.25, 0.5, 0.75, 1.0}`; +- PRIMARY pooled-validation selection `beta_star = 0.75`, using validation per-client + rare-class macro-F1 averaged within seed then across seeds. + +The audit recomputed metrics from run artifacts and re-derived the selection; it did not +repeat the report. No new condition was run. + +--- + +## B. Run inventory + +18 full-data runs were enumerated from `results/*/metadata.json` (15 `support_weighted` + +3 `original_protean`). Every expected run is present, none duplicated, none partial. + +| seed | runs present | metadata fields | manifest reload | smoke | missing artifacts | +|---|---|---|---|---|---| +| 0 | original + beta {0,.25,.5,.75,1} | all correct | true | false | none | +| 1 | original + beta {0,.25,.5,.75,1} | all correct | true | false | none | +| 2 | original + beta {0,.25,.5,.75,1} | all correct | true | false | none | + +Verified per run: dataset=xiiotid, alpha=0.5, seed, aggregation mode, beta, clients=10, +rounds=10, local_epochs=3, batch=256, lr=1e-3, mu=0.1, lambda=1.0, `manifest_reload_ok=true`, +`smoke=false`. No run exists for any alpha ≠ 0.50 and no `5gnidd...support_weighted` run +exists. + +Informational: the seed-0 `original_protean` `metadata.json` lacks the `beta` key because +it predates the Session-5 validation-artifact change; the other 17 runs carry `beta` +(`null` for `original_protean`). No scientific impact. + +--- + +## C. Pairing / manifest integrity + +Within each seed, all six runs share one manifest content hash and the deterministic +artifacts are byte-identical: + +| seed | manifest content hash | round-0 snapshot SHA-256 (all 6 runs) | support/characterization/rare_unseen identical | +|---|---|---|---| +| 0 | `e81a6538…` | `e27f41b3a62eb83c…` | yes | +| 1 | `54ea1df1…` | `b7f61c995e97474c…` | yes | +| 2 | `be881ac6…` | `5b749f4185e11fc7…` | yes | + +`train_support_Nij.csv` equals the manifest support matrix for each seed. `history.json` +and `test_global_metrics.json` differ across runs of a seed, exactly as expected because +beta/original changes training. Pairing is proven by hashes, not filenames. + +Reproducing each manifest from the raw dataset (`prepare_dataset` → `reproduce_train_assignments`): +all three return `reproduce_ok=True`, `support_match=True`, sizes 569,909 / 81,414 / 162,831, +10 clients; train/validation/test are **pairwise disjoint** and their union is the whole +cleaned dataset (814,154 rows). + +--- + +## D. Validation metric recomputation + +Recomputed from raw artifacts (no report numbers copied): + +- seed-level primary score = mean over eligible clients of `val_per_client.csv` + `rare_class_macro_f1`; **equals `val_selection.json` per-client-mean exactly (diff 0.0)**; +- global validation accuracy / macro-F1 / weighted-F1 / rare macro-F1 / rare recall + recomputed **independently from `val_confusion_matrix.json`**; **equal to + `val_global_metrics.json` exactly (diff 0.0)**. + +| seed | beta | eligible | recomputed seed mean | report claim | +|---|---|---|---|---| +| 0 | 0.00 / 0.25 / 0.50 / 0.75 / 1.00 | 10 each | 0.557340 / 0.571787 / 0.585762 / 0.578003 / 0.551443 | same | +| 1 | 0.00 / 0.25 / 0.50 / 0.75 / 1.00 | 10 each | 0.628624 / 0.636205 / 0.645189 / 0.655160 / 0.683117 | same | +| 2 | 0.00 / 0.25 / 0.50 / 0.75 / 1.00 | 10 each | 0.447118 / 0.458310 / 0.471015 / 0.504937 / 0.485877 | same | + +All recomputed values agree with the report to ≤ 4e-7 (six-decimal formatting precision). + +Limitation (unavoidable, documented): the *per-client predictions* cannot be recomputed +because the trained model is not persisted and retraining is forbidden by the audit. The +per-client metric is therefore verified via (a) its defining source code +(`evaluation/metrics.py::rare_class_metrics` over each client's training-derived rare ids), +(b) exact equality of its mean with `val_selection.json`, and (c) exact equality of the +global union-rare metric recomputed from the confusion matrix. This does not affect +`beta_star`, which depends only on the seed-level means. + +--- + +## E. Rare-class integrity + +Per-client rare classes recomputed from training `N_ij` (two smallest positive-support +attack classes, tie by class id) and compared to `rare_unseen.csv` flags: + +- **0 mismatches** across all 30 client-seed cells. +- Benign class (`Normal`) never flagged rare/unseen; rare and unseen never overlap. +- Unseen = attack class with `N_ij = 0`; union of rare classes derived from training equals + both `val_global_metrics.evaluated_rare_classes` and + `test_global_metrics.evaluated_rare_classes` for every seed — i.e. validation/test labels + did **not** influence rare-class identity. +- Eligible clients = **10 in every seed** (no client excluded); every client has exactly 2 + rare classes; no client has only 1 attack class. +- Union rare classes: seed 0 = `{C&C, Exfiltration, Exploitation, Lateral_movement, + Tampering, Weaponization}`; seeds 1 and 2 additionally include `RDOS`. Hence global test + rare support 25,454 (seed 0) vs 53,706 (seeds 1,2). + +--- + +## F. Pooled-selection recomputation + +Computed from the independently recomputed seed-level scores (not from the report). + +| beta | pooled mean | sample std (ddof=1) | pop std (ddof=0) | min | max | +|---|---|---|---|---|---| +| 0.00 | 0.544361 | 0.091447 | 0.074666 | 0.447118 | 0.628624 | +| 0.25 | 0.555434 | 0.090068 | 0.073540 | 0.458310 | 0.636205 | +| 0.50 | 0.567322 | 0.088539 | 0.072292 | 0.471015 | 0.645189 | +| **0.75** | **0.579367** | **0.075121** | 0.061336 | 0.504937 | 0.655160 | +| 1.00 | 0.573479 | 0.100450 | 0.082017 | 0.485877 | 0.683117 | + +- Recompute **matches the report** (mean and sample std) to ≤ 4e-7. +- `argmax` of the pooled validation score = **0.75** (no tie). +- Because every seed has the same eligible count (10), the required + "average within seed, then across seeds" equals a flat average over all clients; the + flat-pool sensitivity check is numerically identical here. The required hierarchy was + still the implemented procedure. +- **Std convention:** the report uses **sample** std (ddof=1), explicitly labelled + `std (sample)` in the report table. Other std uses in the repository + (`characterization.py`, `preprocessing.py`) are population std (ddof=0) for support + statistics. No prior multi-seed std convention existed, and the label is present, so no + convention was silently changed. Recommendation: keep the "(sample)" label wherever the + pooled std is quoted. + +--- + +## G. Validation/test leakage audit + +- `run_baseline.py`: the `val_selection` payload (lines 297–313) is built **only** from + `val_rare`, `val_metrics`, `val_per_client_rows`. Test variables appear in a separate + block and feed only test artifacts. `test_used_for_selection: false` and + `selection_split: "validation"` are recorded for all 15 `support_weighted` runs. +- `analyze_multiseed_beta.py`: the PRIMARY selection block (lines 176–208) consumes only + `per_seed_rows`, which are built from `val_selection.json`. `test_rows` are read in a + later block (line 348+) and are never referenced by the selection logic or tie-break. + The pooled argmax is reproduced by rerunning this read-only script. +- No code path sorts/filters candidates by a test field. + +Conclusion: selection is **programmatic and validation-only**; the boolean flag is a +literal but is corroborated by the independent recomputation of `beta_star` from validation +artifacts alone. `test_used_for_selection = false` is upheld. + +--- + +## H. Test metric recomputation + +`beta_star = 0.75` test metrics recomputed independently from `test_confusion_matrix.json` +(max diff vs the JSON = 0.0), then averaged across seeds with sample std: + +| metric | recomputed mean ± std | report claim | +|---|---|---| +| rare-class macro-F1 | 0.665503 ± 0.026685 | 0.665503 ± 0.026685 | +| rare-class recall | 0.862583 ± 0.013921 | 0.862583 ± 0.013921 | +| macro-F1 | 0.731169 ± 0.006234 | 0.731169 ± 0.006234 | +| accuracy | 0.906392 ± 0.018450 | 0.906392 ± 0.018450 | +| weighted-F1 | 0.933632 ± 0.011230 | 0.933632 ± 0.011230 | + +`original_protean` recomputed identically: rare macro-F1 0.642520 ± 0.028683; rare recall +0.876957 ± 0.010771; macro-F1 0.708965 ± 0.001172; accuracy 0.899454 ± 0.008398; +weighted-F1 0.924733 ± 0.006245 — all match the report. No superiority claim is made. + +--- + +## I. beta = 0 equivalence audit + +For seeds 0, 1 and 2, `original_protean` vs `support_weighted beta=0`: + +| property | seed 0 | seed 1 | seed 2 | +|---|---|---|---| +| same manifest/split | yes | yes | yes | +| round-0 snapshot byte-identical | yes | yes | yes | +| support/characterization/rare_unseen identical | yes | yes | yes | +| independent aggregation diff (`ΣC/count` vs `Σ(1/count)C`) | ≤2.2e-16 | ≤2.2e-16 | ≤4.4e-16 | +| test/history/per-client | differ slightly | differ slightly | differ slightly | +| classification | **B** | **B** | **B** | + +History divergence begins after round 0 (first-round rare macro-F1 abs diff 0.0 at round 0, +growing to ~2e-4–3.6e-3 by round 9, max 6e-3–1.8e-2): floating-point ordering feedback, not +a semantic change. Class **B** (semantically equivalent + floating-point divergence). +Equivalence remains conditional on the shared `S_j` reproduction assumption. + +--- + +## J. Direct-effect audit + +Recomputed independently from the fixed round-0 snapshots (own `tensordot` implementation): + +- per `(class, beta)` displacement matches `...__prototype_effect.csv` **exactly** + (all classes, all betas, all seeds); rank order per seed matches; +- normalised weights sum to 1 for every beta; `support_min/max` match the CSV; +- the same snapshot is reused across beta candidates (byte-identical hashes, §C). + +Cross-seed mean `||Cbar(1) − Cbar(0)||`: Lateral_movement 2.724 (std 1.035), RDOS 2.631 +(1.165), Weaponization 2.561 (1.228), Tampering 2.354 (1.165), Exploitation 2.344 (0.412), +Exfiltration 2.020 (0.502), C&C 1.694 (0.505), Normal 1.557 (0.548), Reconnaissance 1.002 +(0.261). All match the report. Rank order is seed-sensitive (seed 0 top = Weaponization, +seed 1 = RDOS, seed 2 = Lateral_movement). Displacement is a direct-aggregation measure +only; no detection causality is inferred. + +--- + +## K. Client-level audit + +Recomputed from `val_per_client.csv` deltas vs `beta = 0`: + +| seed | beta | improve | degrade | unchanged (ties) | total | report table | +|---|---|---|---|---|---|---| +| 0 | 0.75 | 7 | 3 | 0 | 10 | 7/3/0 | +| 1 | 0.75 | 8 | 1 | 1 (client 7) | 10 | 8/1/1 | +| 2 | 0.75 | 8 | 1 | 1 (client 4) | 10 | 8/1/1 | +| 0 | 1.00 | 4 | 6 | 0 | 10 | 4/6/0 | +| 1 | 1.00 | 9 | 0 | 1 (client 7) | 10 | 9/0/1 | + +The report's §H table already carries an explicit `unchanged` column that accounts for the +ties; only the handoff §9 shorthand ("8/1") omitted them (clarified, §N). Mean Δ / min / max +recomputed match report §H for all six shown cells. `client_validation_detail.csv` deltas +match recomputation (0 mismatches). `min positive N_ij` vs ΔF1 is confirmed **non-monotonic** +(e.g. seed 0 N=1 → −0.021, N=2 → +0.093, N=10 → −0.036; seed 2 N=7 → +0.163, N=10 → +0.177, +N=135 → +0.096). Client ids are treated as per-seed realisations, not one population. + +--- + +## L. Artifact provenance + +Every reported quantity traces to a machine-readable artifact or a deterministic +recomputation. + +| Reported quantity | Source artifact(s) | Derivation | Verified | +|---|---|---|---| +| run existence / config | `results//metadata.json` | direct read | yes | +| manifest content/partition/split hashes | `artifacts/partitions/xiiotid__alpha0.5__seed{0,1,2}.json` | module hash + `reproduce_train_assignments` | yes | +| pairing proof | `artifacts/analysis/...__prototype_snapshot.json` | sha256 across 6 runs/seed | yes | +| seed-level validation score | `results/.../val_per_client.csv` + `val_selection.json` | mean of per-client rare macro-F1 | yes | +| validation global metrics | `results/.../val_confusion_matrix.json` | independent numpy recompute | yes | +| pooled mean/std, `beta_star` | recomputed seed scores | mean/`stdev` over seeds, argmax | yes | +| test metrics (beta 0.75, original) | `results/.../test_confusion_matrix.json` | independent recompute + mean/std | yes | +| beta=0 equivalence | original vs beta0 artifacts/snapshots | sha256 + independent aggregation | yes | +| direct-effect displacement | snapshots + `...__prototype_effect.csv` | independent `tensordot` recompute | yes | +| client improve/degrade | `results/.../val_per_client.csv` | deltas vs beta0 | yes | +| rare-class identity | `artifacts/analysis/...__rare_unseen.csv` + manifest | recompute definition | yes | +| runtime / RSS | `artifacts/analysis/xiiotid__alpha0.5__multiseed__runtime.csv` (added by this audit; seed 0 from Session-5 docs, seeds 1–2 from `/usr/bin/time -v` logs) | parse logs | yes | + +--- + +## M. Source / data integrity + +- HEAD `50f2da72166e4f609077d65b4916f745da5e8e3b`; working-tree source blobs for + `prototype_aggregation.py` (`c298967a…`) and `scripts/run_baseline.py` (`2b469e35…`) + equal their HEAD blobs. Only docs/analysis files are modified/untracked. +- `planningawal/` unchanged (`git status`/`git diff` empty). +- Raw dataset SHA-256 unchanged: X-IIoTID `7b929005…`, 5G-NIDD `fa36f808…`. +- `artifacts/partitions/` unmodified (tracked, no diff); manifests reproduced from raw data. +- `equal_available` mode: absent. Beta candidates: `(0.0, 0.25, 0.50, 0.75, 1.00)` unchanged. +- No alpha ≠ 0.50 result directory; no 5G-NIDD `support_weighted` run. + +--- + +## N. Deviations / corrections found + +1. **Provenance gap (corrected).** Runtime/RSS numbers existed only in Markdown and + ephemeral `/tmp` logs. Added machine-readable + `artifacts/analysis/xiiotid__alpha0.5__multiseed__runtime.csv` (seed 0 from Session-5 + docs; seeds 1–2 parsed from `/usr/bin/time -v` logs). No scientific value changed. +2. **Handoff wording (clarified).** Handoff §9 wrote "8/1 (seed 1), 8/1 (seed 2)" omitting + the one unchanged client each; corrected to "7/3/0, 8/1/1, 8/1/1" to match the report + §H table, which already showed the `unchanged` column. +3. **Std convention (documented, not changed).** Pooled std is sample (ddof=1), labelled + "(sample)" in the report; other repo std uses are population. No prior multi-seed + convention existed. Kept sample, labelled. +4. **Metadata schema (informational).** Seed-0 `original_protean` lacks the `beta` key + (older code); no impact. +5. **Audit-side only:** two first-pass audit scripts had class-list/order bugs + (client-0 class union; support-ordered vs id-ordered comparison); corrected within the + audit. These were audit-script defects, not repository defects. + +No arithmetic error, mislabeled metric, or incorrect derived CSV was found in the Session 6 +report or analysis artifacts. + +--- + +## O. Final audit verdict + +**PASS WITH MINOR CORRECTIONS.** + +- Run inventory, pairing, manifest/split/data integrity: **PASS**. +- Validation metric recomputation (seed-level and global): **PASS** (exact). +- Rare-class integrity: **PASS** (0 mismatches; training-derived; no val/test influence). +- Pooled selection: **PASS**; independent `beta_star = 0.75` using validation only. +- Validation/test leakage: **PASS**; selection reads validation only and is reproducible. +- Test summaries (selected beta and `original_protean`): **PASS**(exact). +- beta=0 equivalence: **PASS**, class **B** (semantically equivalent + float divergence). +- Direct-effect and client-level analyses: **PASS** (independent recompute exact). +- Source/data integrity, pytest (75 passed, 0 failed, 0 skipped, 0 warnings): **PASS**. + +The corrections were provenance/wording only and do not affect `beta_star` or the +scientific interpretation. Session 6 is **safe to checkpoint** with the documented +limitations (single dataset, single alpha, three project-defined seeds, weak effect +relative to seed variance, `S_j` reproduction assumption, no global-optimum claim). diff --git a/scripts/analyze_multiseed_beta.py b/scripts/analyze_multiseed_beta.py new file mode 100644 index 0000000..2f78aca --- /dev/null +++ b/scripts/analyze_multiseed_beta.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python +"""Pooled multi-seed validation analysis for the beta aggregation study. + +Reads the per-run artifacts already written by ``scripts/run_baseline.py`` for +one ``(dataset, alpha)`` condition across several seeds and beta candidates and +produces: + +- a per-(seed, beta) validation table (the candidate-evaluation record); +- a pooled-across-seed validation table (the PRIMARY selection rule); +- per-client rare-class deltas relative to ``beta = 0`` for each seed; +- a machine-readable selection summary. + +This script only reads existing artifacts. It never trains and never uses test +metrics for selection; test columns, where emitted, are clearly labelled as +descriptive only. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import statistics +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from protean_research.aggregation import BETA_CANDIDATES # noqa: E402 + + +def beta_tag(beta: float) -> str: + return f"{beta:g}" + + +def run_name(dataset: str, alpha: float, seed: int, beta: float) -> str: + return f"{dataset}__alpha{alpha:g}__seed{seed}__support_weighted__beta{beta_tag(beta)}" + + +def original_run_name(dataset: str, alpha: float, seed: int) -> str: + return f"{dataset}__alpha{alpha:g}__seed{seed}__original_protean" + + +def read_json(path: Path): + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def read_csv(path: Path) -> list[dict]: + with path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def parse_args(argv=None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset", default="xiiotid") + parser.add_argument("--alpha", type=float, default=0.5) + parser.add_argument("--seeds", type=int, nargs="+", default=[0, 1, 2]) + parser.add_argument( + "--betas", type=float, nargs="+", default=list(BETA_CANDIDATES) + ) + parser.add_argument("--results-dir", default="results") + parser.add_argument("--analysis-dir", default="artifacts/analysis") + parser.add_argument( + "--tag", + default=None, + help="output stem suffix (default: __alpha__multiseed)", + ) + return parser.parse_args(argv) + + +def mean_or_none(values): + values = [float(v) for v in values if v is not None] + return statistics.fmean(values) if values else None + + +def std_or_none(values, ddof: int = 1): + values = [float(v) for v in values if v is not None] + if len(values) < 2: + return None + return statistics.stdev(values) if ddof == 1 else statistics.pstdev(values) + + +def main(argv=None) -> int: + args = parse_args(argv) + results_root = Path(args.results_dir).expanduser() + if not results_root.is_absolute(): + results_root = REPO_ROOT / results_root + analysis_dir = Path(args.analysis_dir).expanduser() + if not analysis_dir.is_absolute(): + analysis_dir = REPO_ROOT / analysis_dir + analysis_dir.mkdir(parents=True, exist_ok=True) + + tag = args.tag or f"{args.dataset}__alpha{args.alpha:g}__multiseed" + + per_seed_rows: list[dict] = [] + client_rows: list[dict] = [] + missing: list[str] = [] + + for seed in args.seeds: + for beta in args.betas: + name = run_name(args.dataset, args.alpha, seed, beta) + run_dir = results_root / name + selection_path = run_dir / "val_selection.json" + if not selection_path.exists(): + missing.append(str(selection_path)) + continue + selection = read_json(selection_path) + metadata = read_json(run_dir / "metadata.json") + row = { + "seed": seed, + "beta": beta, + "eligible_clients": selection["val_n_eval_clients"], + "val_rare_macro_f1_per_client_mean": selection[ + "val_rare_class_macro_f1_per_client_mean" + ], + "val_rare_macro_f1_global": selection["val_rare_class_macro_f1_global"], + "val_rare_recall_per_client_mean": selection[ + "val_rare_class_recall_per_client_mean" + ], + "val_rare_recall_global": selection["val_rare_class_recall_global"], + "val_macro_f1_global": selection["val_macro_f1_global"], + "val_accuracy_global": selection["val_accuracy_global"], + "val_weighted_f1_global": selection["val_weighted_f1_global"], + "manifest_content_hash": metadata.get("manifest_content_hash"), + "manifest_reload_ok": metadata.get("manifest_reload_ok"), + } + per_seed_rows.append(row) + + # per-client validation metrics relative to beta=0 + per_client_path = run_dir / "val_per_client.csv" + if per_client_path.exists(): + for entry in read_csv(per_client_path): + client_rows.append({ + "seed": seed, + "beta": beta, + "client_id": int(entry["client_id"]), + "n_samples": int(entry["n_samples"]), + "val_rare_macro_f1": ( + float(entry["rare_class_macro_f1"]) + if entry["rare_class_macro_f1"] not in ("", "None") + else None + ), + "val_rare_recall": ( + float(entry["rare_class_recall"]) + if entry["rare_class_recall"] not in ("", "None") + else None + ), + }) + + if missing: + print("MISSING ARTIFACTS (analysis incomplete):", file=sys.stderr) + for path in missing: + print(" " + path, file=sys.stderr) + + # ---- pooled table ----------------------------------------------------- + pooled_rows: list[dict] = [] + for beta in args.betas: + rows = [r for r in per_seed_rows if r["beta"] == beta] + scores = [r["val_rare_macro_f1_per_client_mean"] for r in rows] + recalls_pc = [r["val_rare_recall_per_client_mean"] for r in rows] + recalls_g = [r["val_rare_recall_global"] for r in rows] + pooled_rows.append({ + "beta": beta, + "n_seeds": len(rows), + "pooled_val_rare_macro_f1_per_client_mean": mean_or_none(scores), + "std_sample_val_rare_macro_f1": std_or_none(scores, ddof=1), + "std_pop_val_rare_macro_f1": std_or_none(scores, ddof=0), + "min_val_rare_macro_f1": min(scores) if scores else None, + "max_val_rare_macro_f1": max(scores) if scores else None, + "pooled_val_rare_recall_per_client_mean": mean_or_none(recalls_pc), + "pooled_val_rare_recall_global": mean_or_none(recalls_g), + }) + + # ---- PRIMARY pooled selection ---------------------------------------- + selectable = [r for r in pooled_rows if r["n_seeds"] == len(args.seeds)] + selected_beta = None + selection_criterion = None + tie_break_used = False + if selectable: + best_score = max( + r["pooled_val_rare_macro_f1_per_client_mean"] for r in selectable + ) + finalists = [ + r for r in selectable + if abs(r["pooled_val_rare_macro_f1_per_client_mean"] - best_score) <= 1e-12 + ] + if len(finalists) > 1: + tie_break_used = True + best_recall = max( + r["pooled_val_rare_recall_per_client_mean"] for r in finalists + ) + finalists = [ + r for r in finalists + if abs(r["pooled_val_rare_recall_per_client_mean"] - best_recall) <= 1e-12 + ] + if len(finalists) > 1: + min_std = min(r["std_sample_val_rare_macro_f1"] for r in finalists) + finalists = [ + r for r in finalists + if abs(r["std_sample_val_rare_macro_f1"] - min_std) <= 1e-12 + ] + selected_beta = min(r["beta"] for r in finalists) + selection_criterion = ( + "max pooled validation per-client-mean rare-class macro-F1 " + "(tie-break: recall, then lower std, then smaller beta)" + ) + + # ---- per-seed diagnostic selection ----------------------------------- + per_seed_selection = [] + for seed in args.seeds: + rows = [r for r in per_seed_rows if r["seed"] == seed] + if not rows: + continue + best = max(r["val_rare_macro_f1_per_client_mean"] for r in rows) + finalists = [ + r for r in rows + if abs(r["val_rare_macro_f1_per_client_mean"] - best) <= 1e-12 + ] + if len(finalists) > 1: + best_recall = max(r["val_rare_recall_per_client_mean"] for r in finalists) + finalists = [ + r for r in finalists + if abs(r["val_rare_recall_per_client_mean"] - best_recall) <= 1e-12 + ] + selected = min(r["beta"] for r in finalists) + per_seed_selection.append({ + "seed": seed, + "selected_beta": selected, + "val_rare_macro_f1_per_client_mean": best, + "test_used_for_selection": False, + }) + + # ---- support lookup + client-level summary (validation, vs beta = 0) -- + def support_lookup(seed: int): + meta_path = ( + results_root / run_name(args.dataset, args.alpha, seed, args.betas[0]) + / "metadata.json" + ) + benign = read_json(meta_path).get("benign_class") if meta_path.exists() else None + support_path = ( + results_root / run_name(args.dataset, args.alpha, seed, args.betas[0]) + / "train_support_Nij.csv" + ) + support: dict[int, dict[str, int]] = {} + if support_path.exists(): + for entry in read_csv(support_path): + cid = int(entry["client_id"]) + support[cid] = {k: int(v) for k, v in entry.items() if k != "client_id"} + return support, benign + + client_summary_rows: list[dict] = [] + client_detail_rows: list[dict] = [] + for seed in args.seeds: + support, benign = support_lookup(seed) + min_pos = {} + for cid, vals in support.items(): + attacks = [v for k, v in vals.items() if k != benign and v > 0] + min_pos[cid] = min(attacks) if attacks else None + base = { + r["client_id"]: r["val_rare_macro_f1"] + for r in client_rows + if r["seed"] == seed and r["beta"] == args.betas[0] + } + for beta in args.betas: + deltas = [] + for r in client_rows: + if r["seed"] != seed or r["beta"] != beta: + continue + if r["val_rare_macro_f1"] is None or base.get(r["client_id"]) is None: + continue + delta = r["val_rare_macro_f1"] - base[r["client_id"]] + deltas.append(delta) + client_detail_rows.append({ + "seed": seed, + "beta": beta, + "client_id": r["client_id"], + "min_positive_attack_Nij": min_pos.get(r["client_id"]), + "val_rare_macro_f1": r["val_rare_macro_f1"], + "val_rare_recall": r["val_rare_recall"], + "delta_vs_beta0": delta, + }) + n_improve = sum(1 for d in deltas if d > 1e-9) + n_degrade = sum(1 for d in deltas if d < -1e-9) + client_summary_rows.append({ + "seed": seed, + "beta": beta, + "n_eligible": len(deltas), + "n_improve_vs_beta0": n_improve, + "n_degrade_vs_beta0": n_degrade, + "n_unchanged": len(deltas) - n_improve - n_degrade, + "mean_delta_vs_beta0": mean_or_none(deltas), + "median_delta_vs_beta0": statistics.median(deltas) if deltas else None, + "min_delta_vs_beta0": min(deltas) if deltas else None, + "max_delta_vs_beta0": max(deltas) if deltas else None, + }) + + # ---- direct prototype-effect cross-seed summary ---------------------- + effect_raw: list[dict] = [] + for seed in args.seeds: + path = analysis_dir / ( + f"{args.dataset}__alpha{args.alpha:g}__seed{seed}__support_weighted__beta0" + "__prototype_snapshot__prototype_effect.csv" + ) + if not path.exists(): + continue + for r in read_csv(path): + effect_raw.append({ + "seed": seed, + "class_id": int(r["class_id"]), + "class_name": r["class_name"], + "beta": float(r["beta"]), + "n_contributors": int(r["n_contributors"]), + "support_min": int(r["support_min"]), + "support_max": int(r["support_max"]), + "prototype_norm": float(r["prototype_norm"]), + "displacement_from_beta0": float(r["displacement_from_beta0"]), + "relative_displacement": float(r["relative_displacement"]), + }) + effect_pooled_rows: list[dict] = [] + for class_id in sorted({r["class_id"] for r in effect_raw}): + class_name = next( + r["class_name"] for r in effect_raw if r["class_id"] == class_id + ) + for beta in args.betas: + subset = [ + r for r in effect_raw + if r["class_id"] == class_id and abs(r["beta"] - beta) <= 1e-9 + ] + if not subset: + continue + disps = [r["displacement_from_beta0"] for r in subset] + rel = [r["relative_displacement"] for r in subset] + effect_pooled_rows.append({ + "class_id": class_id, + "class_name": class_name, + "beta": beta, + "n_seeds": len(subset), + "n_contributors_seed0": subset[0]["n_contributors"], + "mean_displacement": mean_or_none(disps), + "std_sample_displacement": std_or_none(disps, ddof=1), + "min_displacement": min(disps), + "max_displacement": max(disps), + "mean_relative_displacement": mean_or_none(rel), + }) + + # ---- test metrics for the selected beta + original baseline ---------- + test_rows: list[dict] = [] + for seed in args.seeds: + for beta in args.betas: + path = results_root / run_name(args.dataset, args.alpha, seed, beta) / "test_global_metrics.json" + if not path.exists(): + continue + metrics = read_json(path) + test_rows.append({ + "seed": seed, + "aggregation": f"support_weighted_beta{beta_tag(beta)}", + "beta": beta, + "test_rare_macro_f1": metrics["rare_class_macro_f1"], + "test_rare_recall": metrics["rare_class_recall"], + "test_macro_f1": metrics["macro_f1"], + "test_accuracy": metrics["accuracy"], + "test_weighted_f1": metrics["weighted_f1"], + "test_rare_support": metrics.get("rare_support"), + }) + original_path = results_root / original_run_name(args.dataset, args.alpha, seed) / "test_global_metrics.json" + if original_path.exists(): + metrics = read_json(original_path) + test_rows.append({ + "seed": seed, + "aggregation": "original_protean", + "beta": None, + "test_rare_macro_f1": metrics["rare_class_macro_f1"], + "test_rare_recall": metrics["rare_class_recall"], + "test_macro_f1": metrics["macro_f1"], + "test_accuracy": metrics["accuracy"], + "test_weighted_f1": metrics["weighted_f1"], + "test_rare_support": metrics.get("rare_support"), + }) + + # ---- write artifacts -------------------------------------------------- + def write_csv(path: Path, rows: list[dict]) -> None: + if not rows: + path.write_text("", encoding="utf-8") + return + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + per_seed_path = analysis_dir / f"{tag}__validation_per_seed.csv" + pooled_path = analysis_dir / f"{tag}__validation_pooled.csv" + client_path = analysis_dir / f"{tag}__client_validation_deltas.csv" + test_path = analysis_dir / f"{tag}__test_descriptive.csv" + client_summary_path = analysis_dir / f"{tag}__client_validation_summary.csv" + client_detail_path = analysis_dir / f"{tag}__client_validation_detail.csv" + effect_path = analysis_dir / f"{tag}__prototype_effect_pooled.csv" + summary_path = analysis_dir / f"{tag}__selection_summary.json" + + write_csv(per_seed_path, per_seed_rows) + write_csv(pooled_path, pooled_rows) + write_csv(client_path, client_rows) + write_csv(test_path, test_rows) + write_csv(client_summary_path, client_summary_rows) + write_csv(client_detail_path, client_detail_rows) + write_csv(effect_path, effect_pooled_rows) + + summary = { + "dataset": args.dataset, + "alpha": args.alpha, + "seeds": args.seeds, + "betas": args.betas, + "primary_selection": { + "rule": selection_criterion, + "selected_beta": selected_beta, + "tie_break_used": tie_break_used, + "selection_split": "validation", + "test_used_for_selection": False, + }, + "per_seed_diagnostic": per_seed_selection, + "pooled_table": pooled_rows, + "n_missing_artifacts": len(missing), + "missing_artifacts": missing, + "outputs": { + "per_seed": str(per_seed_path), + "pooled": str(pooled_path), + "client_deltas": str(client_path), + "client_validation_summary": str(client_summary_path), + "client_validation_detail": str(client_detail_path), + "test_descriptive": str(test_path), + "prototype_effect_pooled": str(effect_path), + }, + } + with summary_path.open("w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) + handle.write("\n") + + print(json.dumps({ + "pooled_selected_beta": selected_beta, + "tie_break_used": tie_break_used, + "per_seed_diagnostic": per_seed_selection, + "n_missing_artifacts": len(missing), + "summary": str(summary_path), + }, indent=2, sort_keys=True)) + return 0 if not missing else 1 + + +if __name__ == "__main__": + raise SystemExit(main())