eol_token [3072] -1.0325466394424438
pad_token [3072] -0.19644945859909058
x_embedder.weight [3072,16] 2.6272177696228027
x_embedder.bias [3072] -3.4384851455688477
t_embedder.mlp.0.weight [1024,256] -400.48297119140625
t_embedder.mlp.0.bias [1024] -2.558306932449341
t_embedder.mlp.2.weight [1024,1024] 2099.714111328125
t_embedder.mlp.2.bias [1024] -87.74635314941406
cap_embedder.0.weight [4096] 1913.120849609375
cap_embedder.0.bias [4096] -8.790785789489746
cap_embedder.1.weight [1024,4096] 105.6129150390625
cap_embedder.1.bias [1024] -87.74635314941406
layers.0.attention.gate [32] -0.23894305527210236
layers.0.attention.wq.weight [3072,3072] -39.35104751586914
layers.0.attention.wk.weight [3072,3072] -89.17790222167969
layers.0.attention.wv.weight [3072,3072] -20.645753860473633
layers.0.attention.wk_y.weight [3072,4096] 190.0828094482422
layers.0.attention.wv_y.weight [3072,4096] -96.9395751953125
layers.0.attention.wo.weight [3072,3072] -9.372066497802734
layers.0.attention.q_norm.weight [3072] 2892.91455078125
layers.0.attention.q_norm.bias [3072] -11.59660530090332
layers.0.attention.k_norm.weight [3072] 2894.311767578125
layers.0.attention.k_norm.bias [3072] -10.11361026763916
layers.0.attention.ky_norm.weight [3072] 3063.945556640625
layers.0.attention.ky_norm.bias [3072] -0.011474681086838245
layers.0.feed_forward.w1.weight [8192,3072] -1099.5635986328125
layers.0.feed_forward.w2.weight [3072,8192] 78.66285705566406
layers.0.feed_forward.w3.weight [8192,3072] 11.34511947631836
layers.0.attention_norm.weight [3072] 2975.3125
layers.0.ffn_norm.weight [3072] 2950.692138671875
layers.0.adaLN_modulation.1.weight [18432,1024] 13849.119140625
layers.0.adaLN_modulation.1.bias [18432] -69.28019714355469
layers.0.attention_y_norm.weight [4096] 4157.97607421875
layers.1.attention.gate [32] 0.05398716777563095
layers.1.attention.wq.weight [3072,3072] 21.618366241455078
layers.1.attention.wk.weight [3072,3072] -114.5734634399414
layers.1.attention.wv.weight [3072,3072] 13.230636596679688
layers.1.attention.wk_y.weight [3072,4096] 12.02648639678955
layers.1.attention.wv_y.weight [3072,4096] 12.750207901000977
layers.1.attention.wo.weight [3072,3072] 27.462081909179688
layers.1.attention.q_norm.weight [3072] 3007.773681640625
layers.1.attention.q_norm.bias [3072] -0.11417929828166962
layers.1.attention.k_norm.weight [3072] 3008.679443359375
layers.1.attention.k_norm.bias [3072] 0.20688214898109436
layers.1.attention.ky_norm.weight [3072] 3072.005615234375
layers.1.attention.ky_norm.bias [3072] -3.925985947716981e-05
layers.1.feed_forward.w1.weight [8192,3072] -120.26490783691406
layers.1.feed_forward.w2.weight [3072,8192] 32.61935806274414
layers.1.feed_forward.w3.weight [8192,3072] 57.32780456542969
layers.1.attention_norm.weight [3072] 3038.548095703125
layers.1.ffn_norm.weight [3072] 3046.478759765625
layers.1.adaLN_modulation.1.weight [18432,1024] 2870.427734375
layers.1.adaLN_modulation.1.bias [18432] -41.343116760253906
layers.1.attention_y_norm.weight [4096] 4095.635498046875
layers.2.attention.gate [32] 0.01702398620545864
layers.2.attention.wq.weight [3072,3072] 80.4625015258789
layers.2.attention.wk.weight [3072,3072] -79.031005859375
layers.2.attention.wv.weight [3072,3072] 56.150634765625
layers.2.attention.wk_y.weight [3072,4096] -82.40980529785156
layers.2.attention.wv_y.weight [3072,4096] -3.2464599609375
layers.2.attention.wo.weight [3072,3072] -67.74993896484375
layers.2.attention.q_norm.weight [3072] 2987.19140625
layers.2.attention.q_norm.bias [3072] -1.2148432731628418
layers.2.attention.k_norm.weight [3072] 2987.6943359375
layers.2.attention.k_norm.bias [3072] 0.22290560603141785
layers.2.attention.ky_norm.weight [3072] 3071.9580078125
layers.2.attention.ky_norm.bias [3072] 3.954765634262003e-05
layers.2.feed_forward.w1.weight [8192,3072] 166.37966918945312
layers.2.feed_forward.w2.weight [3072,8192] -200.70860290527344
layers.2.feed_forward.w3.weight [8192,3072] -100.52227783203125
layers.2.attention_norm.weight [3072] 3036.0888671875
layers.2.ffn_norm.weight [3072] 3047.217041015625
layers.2.adaLN_modulation.1.weight [18432,1024] 3933.6630859375
layers.2.adaLN_modulation.1.bias [18432] -36.592166900634766
layers.2.attention_y_norm.weight [4096] 4095.60986328125
layers.3.attention.gate [32] -0.03346005082130432
layers.3.attention.wq.weight [3072,3072] 21.118162155151367
layers.3.attention.wk.weight [3072,3072] 6.076943874359131
layers.3.attention.wv.weight [3072,3072] 38.26424026489258
layers.3.attention.wk_y.weight [3072,4096] -51.09260177612305
layers.3.attention.wv_y.weight [3072,4096] -58.693336486816406
layers.3.attention.wo.weight [3072,3072] 34.71654510498047
layers.3.attention.q_norm.weight [3072] 3021.00341796875
layers.3.attention.q_norm.bias [3072] -0.11786049604415894
layers.3.attention.k_norm.weight [3072] 3021.9580078125
layers.3.attention.k_norm.bias [3072] 0.3359159231185913
layers.3.attention.ky_norm.weight [3072] 3072.02001953125
layers.3.attention.ky_norm.bias [3072] 3.8860143831698224e-05
layers.3.feed_forward.w1.weight [8192,3072] -193.73635864257812
layers.3.feed_forward.w2.weight [3072,8192] -8.395387649536133
layers.3.feed_forward.w3.weight [8192,3072] -44.06329345703125
layers.3.attention_norm.weight [3072] 3004.2099609375
layers.3.ffn_norm.weight [3072] 3054.364501953125
layers.3.adaLN_modulation.1.weight [18432,1024] 3780.938232421875
layers.3.adaLN_modulation.1.bias [18432] -50.4470100402832
layers.3.attention_y_norm.weight [4096] 4095.462646484375
layers.4.attention.gate [32] -0.08687646687030792
layers.4.attention.wq.weight [3072,3072] 74.03779602050781
layers.4.attention.wk.weight [3072,3072] -23.970577239990234
layers.4.attention.wv.weight [3072,3072] 25.737144470214844
layers.4.attention.wk_y.weight [3072,4096] 41.61859893798828
layers.4.attention.wv_y.weight [3072,4096] -79.94941711425781
layers.4.attention.wo.weight [3072,3072] -65.76936340332031
layers.4.attention.q_norm.weight [3072] 2993.53173828125
layers.4.attention.q_norm.bias [3072] -2.4829561710357666
layers.4.attention.k_norm.weight [3072] 2994.48486328125
layers.4.attention.k_norm.bias [3072] -1.0826486349105835
layers.4.attention.ky_norm.weight [3072] 3072.0869140625
layers.4.attention.ky_norm.bias [3072] -5.461290857056156e-05
layers.4.feed_forward.w1.weight [8192,3072] -187.58587646484375
layers.4.feed_forward.w2.weight [3072,8192] -16.679698944091797
layers.4.feed_forward.w3.weight [8192,3072] -44.093780517578125
layers.4.attention_norm.weight [3072] 3016.871337890625
layers.4.ffn_norm.weight [3072] 3035.125244140625
layers.4.adaLN_modulation.1.weight [18432,1024] 3604.148193359375
layers.4.adaLN_modulation.1.bias [18432] -35.95985412597656
layers.4.attention_y_norm.weight [4096] 4095.566162109375
layers.5.attention.gate [32] -0.015579883940517902
layers.5.attention.wq.weight [3072,3072] -47.77400207519531
layers.5.attention.wk.weight [3072,3072] 55.452056884765625
layers.5.attention.wv.weight [3072,3072] -15.54096794128418
layers.5.attention.wk_y.weight [3072,4096] 9.09926986694336
layers.5.attention.wv_y.weight [3072,4096] -21.658100128173828
layers.5.attention.wo.weight [3072,3072] -3.191923141479492
layers.5.attention.q_norm.weight [3072] 2978.672119140625
layers.5.attention.q_norm.bias [3072] 0.844097375869751
layers.5.attention.k_norm.weight [3072] 2979.16015625
layers.5.attention.k_norm.bias [3072] 1.5903466939926147
layers.5.attention.ky_norm.weight [3072] 3072.01513671875
layers.5.attention.ky_norm.bias [3072] 6.433688395190984e-05
layers.5.feed_forward.w1.weight [8192,3072] -493.45184326171875
layers.5.feed_forward.w2.weight [3072,8192] 33.80913543701172
layers.5.feed_forward.w3.weight [8192,3072] -119.64804077148438
layers.5.attention_norm.weight [3072] 3028.444091796875
layers.5.ffn_norm.weight [3072] 3048.370361328125
layers.5.adaLN_modulation.1.weight [18432,1024] 3663.1435546875
layers.5.adaLN_modulation.1.bias [18432] -21.38986587524414
layers.5.attention_y_norm.weight [4096] 4095.36572265625
layers.6.attention.gate [32] 0.0017092330381274223
layers.6.attention.wq.weight [3072,3072] -57.05885314941406
layers.6.attention.wk.weight [3072,3072] -124.88363647460938
layers.6.attention.wv.weight [3072,3072] 117.13907623291016
layers.6.attention.wk_y.weight [3072,4096] -17.165081024169922
layers.6.attention.wv_y.weight [3072,4096] -16.49935531616211
layers.6.attention.wo.weight [3072,3072] -186.33645629882812
layers.6.attention.q_norm.weight [3072] 2969.322509765625
layers.6.attention.q_norm.bias [3072] 0.2132672816514969
layers.6.attention.k_norm.weight [3072] 2969.732421875
layers.6.attention.k_norm.bias [3072] 1.7181894779205322
layers.6.attention.ky_norm.weight [3072] 3071.85009765625
layers.6.attention.ky_norm.bias [3072] -7.439008913934231e-05
layers.6.feed_forward.w1.weight [8192,3072] -161.91812133789062
layers.6.feed_forward.w2.weight [3072,8192] 16.046649932861328
layers.6.feed_forward.w3.weight [8192,3072] 56.12321472167969
layers.6.attention_norm.weight [3072] 3019.05224609375
layers.6.ffn_norm.weight [3072] 3062.233154296875
layers.6.adaLN_modulation.1.weight [18432,1024] 4069.689453125
layers.6.adaLN_modulation.1.bias [18432] 2.369978427886963
layers.6.attention_y_norm.weight [4096] 4094.580322265625
layers.7.attention.gate [32] 0.014872916042804718
layers.7.attention.wq.weight [3072,3072] 13.645149230957031
layers.7.attention.wk.weight [3072,3072] -98.79708862304688
layers.7.attention.wv.weight [3072,3072] 46.834678649902344
layers.7.attention.wk_y.weight [3072,4096] 39.77424240112305
layers.7.attention.wv_y.weight [3072,4096] -12.367530822753906
layers.7.attention.wo.weight [3072,3072] 5.777314186096191
layers.7.attention.q_norm.weight [3072] 2982.542724609375
layers.7.attention.q_norm.bias [3072] 0.07141581177711487
layers.7.attention.k_norm.weight [3072] 2984.382568359375
layers.7.attention.k_norm.bias [3072] -1.1735128164291382
layers.7.attention.ky_norm.weight [3072] 3069.626708984375
layers.7.attention.ky_norm.bias [3072] -0.0011913726339116693
layers.7.feed_forward.w1.weight [8192,3072] 124.79920196533203
layers.7.feed_forward.w2.weight [3072,8192] -43.91264343261719
layers.7.feed_forward.w3.weight [8192,3072] -98.5946273803711
layers.7.attention_norm.weight [3072] 3038.01708984375
layers.7.ffn_norm.weight [3072] 3046.095458984375
layers.7.adaLN_modulation.1.weight [18432,1024] 4130.634765625
layers.7.adaLN_modulation.1.bias [18432] 6.680734634399414
layers.7.attention_y_norm.weight [4096] 4093.17431640625
layers.8.attention.gate [32] -0.01121686864644289
layers.8.attention.wq.weight [3072,3072] -37.67837142944336
layers.8.attention.wk.weight [3072,3072] -18.989913940429688
layers.8.attention.wv.weight [3072,3072] 111.23761749267578
layers.8.attention.wk_y.weight [3072,4096] 13.811660766601562
layers.8.attention.wv_y.weight [3072,4096] 42.801727294921875
layers.8.attention.wo.weight [3072,3072] -45.730072021484375
layers.8.attention.q_norm.weight [3072] 3007.490234375
layers.8.attention.q_norm.bias [3072] 1.4651389122009277
layers.8.attention.k_norm.weight [3072] 3016.03857421875
layers.8.attention.k_norm.bias [3072] -3.029041051864624
layers.8.attention.ky_norm.weight [3072] 3045.24951171875
layers.8.attention.ky_norm.bias [3072] -0.0020398912020027637
layers.8.feed_forward.w1.weight [8192,3072] -485.4872131347656
layers.8.feed_forward.w2.weight [3072,8192] -10.426300048828125
layers.8.feed_forward.w3.weight [8192,3072] -49.361053466796875
layers.8.attention_norm.weight [3072] 3047.21044921875
layers.8.ffn_norm.weight [3072] 3046.864990234375
layers.8.adaLN_modulation.1.weight [18432,1024] 3084.468017578125
layers.8.adaLN_modulation.1.bias [18432] 57.78226089477539
layers.8.attention_y_norm.weight [4096] 4107.97119140625
layers.9.attention.gate [32] 0.012306184507906437
layers.9.attention.wq.weight [3072,3072] 15.961026191711426
layers.9.attention.wk.weight [3072,3072] 65.44038391113281
layers.9.attention.wv.weight [3072,3072] -9.263336181640625
layers.9.attention.wk_y.weight [3072,4096] 70.1529541015625
layers.9.attention.wv_y.weight [3072,4096] -14.750406265258789
layers.9.attention.wo.weight [3072,3072] -12.780290603637695
layers.9.attention.q_norm.weight [3072] 2995.48681640625
layers.9.attention.q_norm.bias [3072] -0.23327499628067017
layers.9.attention.k_norm.weight [3072] 2997.305908203125
layers.9.attention.k_norm.bias [3072] 0.7548366189002991
layers.9.attention.ky_norm.weight [3072] 3068.716796875
layers.9.attention.ky_norm.bias [3072] 0.0006274020997807384
layers.9.feed_forward.w1.weight [8192,3072] -1842.455078125
layers.9.feed_forward.w2.weight [3072,8192] -9.157768249511719
layers.9.feed_forward.w3.weight [8192,3072] -14.311078071594238
layers.9.attention_norm.weight [3072] 3061.135498046875
layers.9.ffn_norm.weight [3072] 3062.936279296875
layers.9.adaLN_modulation.1.weight [18432,1024] 3939.8603515625
layers.9.adaLN_modulation.1.bias [18432] 67.203125
layers.9.attention_y_norm.weight [4096] 4092.83544921875
layers.10.attention.gate [32] 0.04029601067304611
layers.10.attention.wq.weight [3072,3072] 38.86644744873047
layers.10.attention.wk.weight [3072,3072] -146.41436767578125
layers.10.attention.wv.weight [3072,3072] -15.050546646118164
layers.10.attention.wk_y.weight [3072,4096] -19.586706161499023
layers.10.attention.wv_y.weight [3072,4096] 11.6649169921875
layers.10.attention.wo.weight [3072,3072] -76.11978149414062
layers.10.attention.q_norm.weight [3072] 2946.590576171875
layers.10.attention.q_norm.bias [3072] 0.9757489562034607
layers.10.attention.k_norm.weight [3072] 2951.70654296875
layers.10.attention.k_norm.bias [3072] -0.4007686376571655
layers.10.attention.ky_norm.weight [3072] 3062.026611328125
layers.10.attention.ky_norm.bias [3072] 0.0037135514430701733
layers.10.feed_forward.w1.weight [8192,3072] -504.6271057128906
layers.10.feed_forward.w2.weight [3072,8192] -185.80174255371094
layers.10.feed_forward.w3.weight [8192,3072] 143.99978637695312
layers.10.attention_norm.weight [3072] 3059.58544921875
layers.10.ffn_norm.weight [3072] 3089.393798828125
layers.10.adaLN_modulation.1.weight [18432,1024] 3745.93603515625
layers.10.adaLN_modulation.1.bias [18432] 79.93937683105469
layers.10.attention_y_norm.weight [4096] 4099.947265625
layers.11.attention.gate [32] -0.01198449358344078
layers.11.attention.wq.weight [3072,3072] 63.14850997924805
layers.11.attention.wk.weight [3072,3072] 143.75637817382812
layers.11.attention.wv.weight [3072,3072] 46.21250915527344
layers.11.attention.wk_y.weight [3072,4096] 23.84990119934082
layers.11.attention.wv_y.weight [3072,4096] 92.25910949707031
layers.11.attention.wo.weight [3072,3072] -62.00811004638672
layers.11.attention.q_norm.weight [3072] 3019.99658203125
layers.11.attention.q_norm.bias [3072] 0.8490341305732727
layers.11.attention.k_norm.weight [3072] 3020.435302734375
layers.11.attention.k_norm.bias [3072] 0.8896084427833557
layers.11.attention.ky_norm.weight [3072] 3072.01904296875
layers.11.attention.ky_norm.bias [3072] -2.768600097624585e-06
layers.11.feed_forward.w1.weight [8192,3072] 1253.9071044921875
layers.11.feed_forward.w2.weight [3072,8192] -7.5883331298828125
layers.11.feed_forward.w3.weight [8192,3072] 2.7933826446533203
layers.11.attention_norm.weight [3072] 3070.736572265625
layers.11.ffn_norm.weight [3072] 3095.813232421875
layers.11.adaLN_modulation.1.weight [18432,1024] 3233.74951171875
layers.11.adaLN_modulation.1.bias [18432] 71.24488830566406
layers.11.attention_y_norm.weight [4096] 4095.555419921875
layers.12.attention.gate [32] -0.007618741597980261
layers.12.attention.wq.weight [3072,3072] 33.432884216308594
layers.12.attention.wk.weight [3072,3072] 98.3436279296875
layers.12.attention.wv.weight [3072,3072] -48.924705505371094
layers.12.attention.wk_y.weight [3072,4096] -57.52131652832031
layers.12.attention.wv_y.weight [3072,4096] 74.3033218383789
layers.12.attention.wo.weight [3072,3072] 0.3465995788574219
layers.12.attention.q_norm.weight [3072] 2954.002685546875
layers.12.attention.q_norm.bias [3072] 2.3720955848693848
layers.12.attention.k_norm.weight [3072] 2957.90966796875
layers.12.attention.k_norm.bias [3072] -2.5865988731384277
layers.12.attention.ky_norm.weight [3072] 3071.710205078125
layers.12.attention.ky_norm.bias [3072] -0.0002978214470203966
layers.12.feed_forward.w1.weight [8192,3072] -214.1547393798828
layers.12.feed_forward.w2.weight [3072,8192] 54.76542663574219
layers.12.feed_forward.w3.weight [8192,3072] 98.478515625
layers.12.attention_norm.weight [3072] 3116.89990234375
layers.12.ffn_norm.weight [3072] 3124.07763671875
layers.12.adaLN_modulation.1.weight [18432,1024] 3140.50341796875
layers.12.adaLN_modulation.1.bias [18432] 160.7002410888672
layers.12.attention_y_norm.weight [4096] 4092.88818359375
layers.13.attention.gate [32] -0.03315815329551697
layers.13.attention.wq.weight [3072,3072] -22.661317825317383
layers.13.attention.wk.weight [3072,3072] 11.8341064453125
layers.13.attention.wv.weight [3072,3072] -9.14779281616211
layers.13.attention.wk_y.weight [3072,4096] 66.34749603271484
layers.13.attention.wv_y.weight [3072,4096] 3.579252243041992
layers.13.attention.wo.weight [3072,3072] -61.694175720214844
layers.13.attention.q_norm.weight [3072] 3028.61474609375
layers.13.attention.q_norm.bias [3072] 1.2185944318771362
layers.13.attention.k_norm.weight [3072] 3030.0087890625
layers.13.attention.k_norm.bias [3072] -0.8221495151519775
layers.13.attention.ky_norm.weight [3072] 3072.18017578125
layers.13.attention.ky_norm.bias [3072] -0.0013712140498682857
layers.13.feed_forward.w1.weight [8192,3072] -955.824462890625
layers.13.feed_forward.w2.weight [3072,8192] -211.47750854492188
layers.13.feed_forward.w3.weight [8192,3072] 17.22570037841797
layers.13.attention_norm.weight [3072] 3100.14111328125
layers.13.ffn_norm.weight [3072] 3107.3359375
layers.13.adaLN_modulation.1.weight [18432,1024] 4191.0283203125
layers.13.adaLN_modulation.1.bias [18432] 107.70501708984375
layers.13.attention_y_norm.weight [4096] 4094.821044921875
layers.14.attention.gate [32] 0.014552398584783077
layers.14.attention.wq.weight [3072,3072] -33.400760650634766
layers.14.attention.wk.weight [3072,3072] 85.48472595214844
layers.14.attention.wv.weight [3072,3072] 147.46282958984375
layers.14.attention.wk_y.weight [3072,4096] 46.765472412109375
layers.14.attention.wv_y.weight [3072,4096] -117.26578521728516
layers.14.attention.wo.weight [3072,3072] -67.44631958007812
layers.14.attention.q_norm.weight [3072] 3027.458984375
layers.14.attention.q_norm.bias [3072] -1.36245596408844
layers.14.attention.k_norm.weight [3072] 3028.90234375
layers.14.attention.k_norm.bias [3072] 1.1499375104904175
layers.14.attention.ky_norm.weight [3072] 3071.199462890625
layers.14.attention.ky_norm.bias [3072] -0.00021015049424022436
layers.14.feed_forward.w1.weight [8192,3072] -885.6415405273438
layers.14.feed_forward.w2.weight [3072,8192] 140.91741943359375
layers.14.feed_forward.w3.weight [8192,3072] 47.07232666015625
layers.14.attention_norm.weight [3072] 3102.81494140625
layers.14.ffn_norm.weight [3072] 3154.388916015625
layers.14.adaLN_modulation.1.weight [18432,1024] 4061.25537109375
layers.14.adaLN_modulation.1.bias [18432] 147.67689514160156
layers.14.attention_y_norm.weight [4096] 4095.15478515625
layers.15.attention.gate [32] -0.0038218656554818153
layers.15.attention.wq.weight [3072,3072] 5.81072998046875
layers.15.attention.wk.weight [3072,3072] -18.721271514892578
layers.15.attention.wv.weight [3072,3072] -62.68359375
layers.15.attention.wk_y.weight [3072,4096] 10.82135009765625
layers.15.attention.wv_y.weight [3072,4096] 9.509391784667969
layers.15.attention.wo.weight [3072,3072] -9.801557540893555
layers.15.attention.q_norm.weight [3072] 3043.85498046875
layers.15.attention.q_norm.bias [3072] -1.826299786567688
layers.15.attention.k_norm.weight [3072] 3050.7705078125
layers.15.attention.k_norm.bias [3072] 5.323702335357666
layers.15.attention.ky_norm.weight [3072] 3053.9814453125
layers.15.attention.ky_norm.bias [3072] -0.00020967214368283749
layers.15.feed_forward.w1.weight [8192,3072] -1640.3089599609375
layers.15.feed_forward.w2.weight [3072,8192] 24.164220809936523
layers.15.feed_forward.w3.weight [8192,3072] -176.226318359375
layers.15.attention_norm.weight [3072] 3131.90087890625
layers.15.ffn_norm.weight [3072] 3108.34033203125
layers.15.adaLN_modulation.1.weight [18432,1024] 3182.173583984375
layers.15.adaLN_modulation.1.bias [18432] 185.0066680908203
layers.15.attention_y_norm.weight [4096] 4092.14306640625
layers.16.attention.gate [32] -0.025445424020290375
layers.16.attention.wq.weight [3072,3072] 131.5052032470703
layers.16.attention.wk.weight [3072,3072] -112.08161926269531
layers.16.attention.wv.weight [3072,3072] 110.10245513916016
layers.16.attention.wk_y.weight [3072,4096] -86.7544174194336
layers.16.attention.wv_y.weight [3072,4096] -21.646686553955078
layers.16.attention.wo.weight [3072,3072] -74.89533233642578
layers.16.attention.q_norm.weight [3072] 2927.330078125
layers.16.attention.q_norm.bias [3072] 0.10248535871505737
layers.16.attention.k_norm.weight [3072] 2933.65283203125
layers.16.attention.k_norm.bias [3072] -1.7414463758468628
layers.16.attention.ky_norm.weight [3072] 3064.706787109375
layers.16.attention.ky_norm.bias [3072] 0.00431065121665597
layers.16.feed_forward.w1.weight [8192,3072] -2018.5386962890625
layers.16.feed_forward.w2.weight [3072,8192] 77.16911315917969
layers.16.feed_forward.w3.weight [8192,3072] 80.67045593261719
layers.16.attention_norm.weight [3072] 3131.244140625
layers.16.ffn_norm.weight [3072] 3055.95947265625
layers.16.adaLN_modulation.1.weight [18432,1024] 614.5826416015625
layers.16.adaLN_modulation.1.bias [18432] 155.30259704589844
layers.16.attention_y_norm.weight [4096] 4106.484375
layers.17.attention.gate [32] 0.0464252270758152
layers.17.attention.wq.weight [3072,3072] -70.33844757080078
layers.17.attention.wk.weight [3072,3072] -24.124074935913086
layers.17.attention.wv.weight [3072,3072] -11.825756072998047
layers.17.attention.wk_y.weight [3072,4096] -52.56653594970703
layers.17.attention.wv_y.weight [3072,4096] 0.6748561859130859
layers.17.attention.wo.weight [3072,3072] 99.63531494140625
layers.17.attention.q_norm.weight [3072] 3044.56494140625
layers.17.attention.q_norm.bias [3072] 3.0524942874908447
layers.17.attention.k_norm.weight [3072] 3050.87548828125
layers.17.attention.k_norm.bias [3072] -1.237405776977539
layers.17.attention.ky_norm.weight [3072] 3069.18896484375
layers.17.attention.ky_norm.bias [3072] 0.002125290222465992
layers.17.feed_forward.w1.weight [8192,3072] -3232.49951171875
layers.17.feed_forward.w2.weight [3072,8192] 124.21507263183594
layers.17.feed_forward.w3.weight [8192,3072] 1.7152252197265625
layers.17.attention_norm.weight [3072] 3068.261962890625
layers.17.ffn_norm.weight [3072] 3070.30517578125
layers.17.adaLN_modulation.1.weight [18432,1024] 2111.0537109375
layers.17.adaLN_modulation.1.bias [18432] 134.48922729492188
layers.17.attention_y_norm.weight [4096] 4136.7578125
layers.18.attention.gate [32] 0.08387154340744019
layers.18.attention.wq.weight [3072,3072] 11.327713012695312
layers.18.attention.wk.weight [3072,3072] 11.674558639526367
layers.18.attention.wv.weight [3072,3072] -54.94795608520508
layers.18.attention.wk_y.weight [3072,4096] -52.60209655761719
layers.18.attention.wv_y.weight [3072,4096] -8.594970703125
layers.18.attention.wo.weight [3072,3072] -133.90725708007812
layers.18.attention.q_norm.weight [3072] 3070.612548828125
layers.18.attention.q_norm.bias [3072] -2.529588222503662
layers.18.attention.k_norm.weight [3072] 3070.044921875
layers.18.attention.k_norm.bias [3072] -2.209380865097046
layers.18.attention.ky_norm.weight [3072] 3074.084716796875
layers.18.attention.ky_norm.bias [3072] -6.738098454661667e-05
layers.18.feed_forward.w1.weight [8192,3072] -1076.1514892578125
layers.18.feed_forward.w2.weight [3072,8192] 30.47028350830078
layers.18.feed_forward.w3.weight [8192,3072] 170.23670959472656
layers.18.attention_norm.weight [3072] 3068.975830078125
layers.18.ffn_norm.weight [3072] 3061.591796875
layers.18.adaLN_modulation.1.weight [18432,1024] 1968.1068115234375
layers.18.adaLN_modulation.1.bias [18432] 124.23133850097656
layers.18.attention_y_norm.weight [4096] 4098.11181640625
layers.19.attention.gate [32] 0.17188625037670135
layers.19.attention.wq.weight [3072,3072] 117.72798156738281
layers.19.attention.wk.weight [3072,3072] 103.63431549072266
layers.19.attention.wv.weight [3072,3072] -95.07378387451172
layers.19.attention.wk_y.weight [3072,4096] 52.24602508544922
layers.19.attention.wv_y.weight [3072,4096] 18.882774353027344
layers.19.attention.wo.weight [3072,3072] 66.77247619628906
layers.19.attention.q_norm.weight [3072] 3170.20263671875
layers.19.attention.q_norm.bias [3072] 2.2199411392211914
layers.19.attention.k_norm.weight [3072] 3185.52880859375
layers.19.attention.k_norm.bias [3072] -3.6901142597198486
layers.19.attention.ky_norm.weight [3072] 3048.489013671875
layers.19.attention.ky_norm.bias [3072] -0.03251801058650017
layers.19.feed_forward.w1.weight [8192,3072] -1652.0145263671875
layers.19.feed_forward.w2.weight [3072,8192] -186.74960327148438
layers.19.feed_forward.w3.weight [8192,3072] -11.902843475341797
layers.19.attention_norm.weight [3072] 3159.30224609375
layers.19.ffn_norm.weight [3072] 3000.66650390625
layers.19.adaLN_modulation.1.weight [18432,1024] -5254.0078125
layers.19.adaLN_modulation.1.bias [18432] 158.53675842285156
layers.19.attention_y_norm.weight [4096] 4189.46484375
layers.20.attention.gate [32] -1.5953177213668823
layers.20.attention.wq.weight [3072,3072] 144.7916259765625
layers.20.attention.wk.weight [3072,3072] 92.65377807617188
layers.20.attention.wv.weight [3072,3072] 206.8223419189453
layers.20.attention.wk_y.weight [3072,4096] -437.47711181640625
layers.20.attention.wv_y.weight [3072,4096] -51.4007682800293
layers.20.attention.wo.weight [3072,3072] -71.60449981689453
layers.20.attention.q_norm.weight [3072] 3110.75732421875
layers.20.attention.q_norm.bias [3072] -1.0422128438949585
layers.20.attention.k_norm.weight [3072] 3157.789794921875
layers.20.attention.k_norm.bias [3072] 6.680957794189453
layers.20.attention.ky_norm.weight [3072] 2991.18701171875
layers.20.attention.ky_norm.bias [3072] 0.11590077728033066
layers.20.feed_forward.w1.weight [8192,3072] -2556.183837890625
layers.20.feed_forward.w2.weight [3072,8192] -68.1128921508789
layers.20.feed_forward.w3.weight [8192,3072] -77.56266021728516
layers.20.attention_norm.weight [3072] 3164.392578125
layers.20.ffn_norm.weight [3072] 2951.37255859375
layers.20.adaLN_modulation.1.weight [18432,1024] -6961.55224609375
layers.20.adaLN_modulation.1.bias [18432] 128.227294921875
layers.20.attention_y_norm.weight [4096] 4332.23046875
layers.21.attention.gate [32] 0.3935057818889618
layers.21.attention.wq.weight [3072,3072] -87.36233520507812
layers.21.attention.wk.weight [3072,3072] -15.280412673950195
layers.21.attention.wv.weight [3072,3072] -67.25896453857422
layers.21.attention.wk_y.weight [3072,4096] 40.6765251159668
layers.21.attention.wv_y.weight [3072,4096] 4.294679641723633
layers.21.attention.wo.weight [3072,3072] 84.40620422363281
layers.21.attention.q_norm.weight [3072] 3188.112548828125
layers.21.attention.q_norm.bias [3072] 1.9379830360412598
layers.21.attention.k_norm.weight [3072] 3192.55078125
layers.21.attention.k_norm.bias [3072] -3.9968056678771973
layers.21.attention.ky_norm.weight [3072] 3052.492431640625
layers.21.attention.ky_norm.bias [3072] -0.003833170747384429
layers.21.feed_forward.w1.weight [8192,3072] -1081.25537109375
layers.21.feed_forward.w2.weight [3072,8192] 3.0426559448242188
layers.21.feed_forward.w3.weight [8192,3072] 204.11849975585938
layers.21.attention_norm.weight [3072] 3093.47900390625
layers.21.ffn_norm.weight [3072] 2938.971923828125
layers.21.adaLN_modulation.1.weight [18432,1024] -3367.383056640625
layers.21.adaLN_modulation.1.bias [18432] 84.77622985839844
layers.21.attention_y_norm.weight [4096] 4163.63623046875
layers.22.attention.gate [32] 0.14358212053775787
layers.22.attention.wq.weight [3072,3072] -8.050209045410156
layers.22.attention.wk.weight [3072,3072] 61.84921646118164
layers.22.attention.wv.weight [3072,3072] -147.03427124023438
layers.22.attention.wk_y.weight [3072,4096] -46.90061950683594
layers.22.attention.wv_y.weight [3072,4096] 153.3000946044922
layers.22.attention.wo.weight [3072,3072] 160.7344207763672
layers.22.attention.q_norm.weight [3072] 3119.335205078125
layers.22.attention.q_norm.bias [3072] -2.683124542236328
layers.22.attention.k_norm.weight [3072] 3127.46435546875
layers.22.attention.k_norm.bias [3072] 3.678471088409424
layers.22.attention.ky_norm.weight [3072] 3039.71826171875
layers.22.attention.ky_norm.bias [3072] 0.002608638256788254
layers.22.feed_forward.w1.weight [8192,3072] -2517.890380859375
layers.22.feed_forward.w2.weight [3072,8192] 130.9670867919922
layers.22.feed_forward.w3.weight [8192,3072] -67.2720947265625
layers.22.attention_norm.weight [3072] 3140.771484375
layers.22.ffn_norm.weight [3072] 2921.1396484375
layers.22.adaLN_modulation.1.weight [18432,1024] -5785.93115234375
layers.22.adaLN_modulation.1.bias [18432] 84.46720886230469
layers.22.attention_y_norm.weight [4096] 4130.8642578125
layers.23.attention.gate [32] -0.002392765134572983
layers.23.attention.wq.weight [3072,3072] 219.03675842285156
layers.23.attention.wk.weight [3072,3072] -4.637319564819336
layers.23.attention.wv.weight [3072,3072] -56.83465576171875
layers.23.attention.wk_y.weight [3072,4096] -96.41117095947266
layers.23.attention.wv_y.weight [3072,4096] 14.194984436035156
layers.23.attention.wo.weight [3072,3072] 99.3251953125
layers.23.attention.q_norm.weight [3072] 3248.3740234375
layers.23.attention.q_norm.bias [3072] 5.431629180908203
layers.23.attention.k_norm.weight [3072] 3256.32275390625
layers.23.attention.k_norm.bias [3072] 2.7839131355285645
layers.23.attention.ky_norm.weight [3072] 3007.111572265625
layers.23.attention.ky_norm.bias [3072] 0.004978652112185955
layers.23.feed_forward.w1.weight [8192,3072] -5671.43408203125
layers.23.feed_forward.w2.weight [3072,8192] 19.224151611328125
layers.23.feed_forward.w3.weight [8192,3072] -0.8238067626953125
layers.23.attention_norm.weight [3072] 3166.041748046875
layers.23.ffn_norm.weight [3072] 2893.22314453125
layers.23.adaLN_modulation.1.weight [18432,1024] -8372.5546875
layers.23.adaLN_modulation.1.bias [18432] 81.4506607055664
layers.23.attention_y_norm.weight [4096] 4125.078125
layers.24.attention.gate [32] 0.013135295361280441
layers.24.attention.wq.weight [3072,3072] 130.8161163330078
layers.24.attention.wk.weight [3072,3072] 21.15003204345703
layers.24.attention.wv.weight [3072,3072] -31.553049087524414
layers.24.attention.wk_y.weight [3072,4096] -101.0406265258789
layers.24.attention.wv_y.weight [3072,4096] 23.723058700561523
layers.24.attention.wo.weight [3072,3072] -38.196678161621094
layers.24.attention.q_norm.weight [3072] 3155.390380859375
layers.24.attention.q_norm.bias [3072] 3.4366486072540283
layers.24.attention.k_norm.weight [3072] 3156.375244140625
layers.24.attention.k_norm.bias [3072] 4.5202317237854
layers.24.attention.ky_norm.weight [3072] 3070.053955078125
layers.24.attention.ky_norm.bias [3072] -0.0001328116632066667
layers.24.feed_forward.w1.weight [8192,3072] -2434.389404296875
layers.24.feed_forward.w2.weight [3072,8192] 304.4960632324219
layers.24.feed_forward.w3.weight [8192,3072] -134.15628051757812
layers.24.attention_norm.weight [3072] 3176.412841796875
layers.24.ffn_norm.weight [3072] 2897.515380859375
layers.24.adaLN_modulation.1.weight [18432,1024] -2769.151123046875
layers.24.adaLN_modulation.1.bias [18432] 98.36761474609375
layers.24.attention_y_norm.weight [4096] 4094.14794921875
layers.25.attention.gate [32] -0.2465812712907791
layers.25.attention.wq.weight [3072,3072] -16.641672134399414
layers.25.attention.wk.weight [3072,3072] -46.90250778198242
layers.25.attention.wv.weight [3072,3072] -101.754150390625
layers.25.attention.wk_y.weight [3072,4096] 23.654953002929688
layers.25.attention.wv_y.weight [3072,4096] 40.800445556640625
layers.25.attention.wo.weight [3072,3072] 48.517608642578125
layers.25.attention.q_norm.weight [3072] 3089.47412109375
layers.25.attention.q_norm.bias [3072] 1.6239193677902222
layers.25.attention.k_norm.weight [3072] 3090.643310546875
layers.25.attention.k_norm.bias [3072] 3.5273795127868652
layers.25.attention.ky_norm.weight [3072] 3068.446533203125
layers.25.attention.ky_norm.bias [3072] -0.0014377831248566508
layers.25.feed_forward.w1.weight [8192,3072] -1247.4962158203125
layers.25.feed_forward.w2.weight [3072,8192] 81.568603515625
layers.25.feed_forward.w3.weight [8192,3072] 39.595802307128906
layers.25.attention_norm.weight [3072] 3143.8056640625
layers.25.ffn_norm.weight [3072] 2957.525146484375
layers.25.adaLN_modulation.1.weight [18432,1024] 2085.119140625
layers.25.adaLN_modulation.1.bias [18432] 50.94136047363281
layers.25.attention_y_norm.weight [4096] 4097.025390625
layers.26.attention.gate [32] 0.06776668131351471
layers.26.attention.wq.weight [3072,3072] -114.65640258789062
layers.26.attention.wk.weight [3072,3072] 35.947418212890625
layers.26.attention.wv.weight [3072,3072] -71.32766723632812
layers.26.attention.wk_y.weight [3072,4096] 98.2721939086914
layers.26.attention.wv_y.weight [3072,4096] 16.36252212524414
layers.26.attention.wo.weight [3072,3072] 91.789306640625
layers.26.attention.q_norm.weight [3072] 3124.747314453125
layers.26.attention.q_norm.bias [3072] -2.1321656703948975
layers.26.attention.k_norm.weight [3072] 3127.651611328125
layers.26.attention.k_norm.bias [3072] 2.0264811515808105
layers.26.attention.ky_norm.weight [3072] 3066.502197265625
layers.26.attention.ky_norm.bias [3072] -0.00292210653424263
layers.26.feed_forward.w1.weight [8192,3072] -2005.973876953125
layers.26.feed_forward.w2.weight [3072,8192] -68.9849853515625
layers.26.feed_forward.w3.weight [8192,3072] 140.1864776611328
layers.26.attention_norm.weight [3072] 3274.666748046875
layers.26.ffn_norm.weight [3072] 2943.43603515625
layers.26.adaLN_modulation.1.weight [18432,1024] -10638.904296875
layers.26.adaLN_modulation.1.bias [18432] 169.11045837402344
layers.26.attention_y_norm.weight [4096] 4092.880126953125
layers.27.attention.gate [32] 0.23680511116981506
layers.27.attention.wq.weight [3072,3072] -111.7753677368164
layers.27.attention.wk.weight [3072,3072] 29.898605346679688
layers.27.attention.wv.weight [3072,3072] -106.8389892578125
layers.27.attention.wk_y.weight [3072,4096] 49.35763931274414
layers.27.attention.wv_y.weight [3072,4096] 80.67949676513672
layers.27.attention.wo.weight [3072,3072] 219.8512725830078
layers.27.attention.q_norm.weight [3072] 3011.391357421875
layers.27.attention.q_norm.bias [3072] -3.666907787322998
layers.27.attention.k_norm.weight [3072] 3020.61962890625
layers.27.attention.k_norm.bias [3072] -4.364409923553467
layers.27.attention.ky_norm.weight [3072] 3016.0146484375
layers.27.attention.ky_norm.bias [3072] 0.0004845573566854
layers.27.feed_forward.w1.weight [8192,3072] -2012.774658203125
layers.27.feed_forward.w2.weight [3072,8192] 264.86895751953125
layers.27.feed_forward.w3.weight [8192,3072] -185.6049346923828
layers.27.attention_norm.weight [3072] 3217.038330078125
layers.27.ffn_norm.weight [3072] 2952.324951171875
layers.27.adaLN_modulation.1.weight [18432,1024] -6907.4052734375
layers.27.adaLN_modulation.1.bias [18432] 122.84993743896484
layers.27.attention_y_norm.weight [4096] 4158.11328125
layers.28.attention.gate [32] 0.07415129989385605
layers.28.attention.wq.weight [3072,3072] -57.30853271484375
layers.28.attention.wk.weight [3072,3072] -101.21186828613281
layers.28.attention.wv.weight [3072,3072] -58.32230758666992
layers.28.attention.wk_y.weight [3072,4096] -53.361759185791016
layers.28.attention.wv_y.weight [3072,4096] 63.823814392089844
layers.28.attention.wo.weight [3072,3072] 103.74847412109375
layers.28.attention.q_norm.weight [3072] 3239.63818359375
layers.28.attention.q_norm.bias [3072] -3.9089338779449463
layers.28.attention.k_norm.weight [3072] 3245.327392578125
layers.28.attention.k_norm.bias [3072] -4.821537494659424
layers.28.attention.ky_norm.weight [3072] 3003.415283203125
layers.28.attention.ky_norm.bias [3072] -0.03735707700252533
layers.28.feed_forward.w1.weight [8192,3072] -2887.3330078125
layers.28.feed_forward.w2.weight [3072,8192] 73.66928100585938
layers.28.feed_forward.w3.weight [8192,3072] 277.9718017578125
layers.28.attention_norm.weight [3072] 3211.492431640625
layers.28.ffn_norm.weight [3072] 2956.84033203125
layers.28.adaLN_modulation.1.weight [18432,1024] -11667.16796875
layers.28.adaLN_modulation.1.bias [18432] 139.17074584960938
layers.28.attention_y_norm.weight [4096] 4112.95458984375
layers.29.attention.gate [32] -0.07671057432889938
layers.29.attention.wq.weight [3072,3072] -68.83955383300781
layers.29.attention.wk.weight [3072,3072] -0.9065589904785156
layers.29.attention.wv.weight [3072,3072] 28.281841278076172
layers.29.attention.wk_y.weight [3072,4096] -7.9993896484375
layers.29.attention.wv_y.weight [3072,4096] 15.681102752685547
layers.29.attention.wo.weight [3072,3072] 58.18054962158203
layers.29.attention.q_norm.weight [3072] 3099.19580078125
layers.29.attention.q_norm.bias [3072] 4.845227241516113
layers.29.attention.k_norm.weight [3072] 3100.7880859375
layers.29.attention.k_norm.bias [3072] 3.324284553527832
layers.29.attention.ky_norm.weight [3072] 3070.992431640625
layers.29.attention.ky_norm.bias [3072] -5.29188837390393e-05
layers.29.feed_forward.w1.weight [8192,3072] -1088.1844482421875
layers.29.feed_forward.w2.weight [3072,8192] -63.90240478515625
layers.29.feed_forward.w3.weight [8192,3072] 255.48028564453125
layers.29.attention_norm.weight [3072] 3301.2177734375
layers.29.ffn_norm.weight [3072] 2959.95263671875
layers.29.adaLN_modulation.1.weight [18432,1024] -14894.068359375
layers.29.adaLN_modulation.1.bias [18432] 199.5867156982422
layers.29.attention_y_norm.weight [4096] 4094.56201171875
layers.30.attention.gate [32] -0.017847031354904175
layers.30.attention.wq.weight [3072,3072] -17.817123413085938
layers.30.attention.wk.weight [3072,3072] -194.7149658203125
layers.30.attention.wv.weight [3072,3072] 88.34908294677734
layers.30.attention.wk_y.weight [3072,4096] -11.472770690917969
layers.30.attention.wv_y.weight [3072,4096] -5.1739501953125
layers.30.attention.wo.weight [3072,3072] 74.96720886230469
layers.30.attention.q_norm.weight [3072] 3157.85107421875
layers.30.attention.q_norm.bias [3072] -0.26245981454849243
layers.30.attention.k_norm.weight [3072] 3157.635498046875
layers.30.attention.k_norm.bias [3072] 3.0958383083343506
layers.30.attention.ky_norm.weight [3072] 3069.7412109375
layers.30.attention.ky_norm.bias [3072] 0.001112552359700203
layers.30.feed_forward.w1.weight [8192,3072] -1377.74267578125
layers.30.feed_forward.w2.weight [3072,8192] -82.40560913085938
layers.30.feed_forward.w3.weight [8192,3072] -30.384689331054688
layers.30.attention_norm.weight [3072] 3242.50732421875
layers.30.ffn_norm.weight [3072] 2975.627197265625
layers.30.adaLN_modulation.1.weight [18432,1024] -11811.19921875
layers.30.adaLN_modulation.1.bias [18432] 168.41806030273438
layers.30.attention_y_norm.weight [4096] 4094.72412109375
layers.31.attention.gate [32] -0.01899895630776882
layers.31.attention.wq.weight [3072,3072] 1.1386985778808594
layers.31.attention.wk.weight [3072,3072] 83.11277770996094
layers.31.attention.wv.weight [3072,3072] 129.1820526123047
layers.31.attention.wk_y.weight [3072,4096] 129.71629333496094
layers.31.attention.wv_y.weight [3072,4096] 28.900983810424805
layers.31.attention.wo.weight [3072,3072] -18.090412139892578
layers.31.attention.q_norm.weight [3072] 3148.16455078125
layers.31.attention.q_norm.bias [3072] 1.7190591096878052
layers.31.attention.k_norm.weight [3072] 3150.99853515625
layers.31.attention.k_norm.bias [3072] -1.90182626247406
layers.31.attention.ky_norm.weight [3072] 3072.027587890625
layers.31.attention.ky_norm.bias [3072] -5.545430030906573e-05
layers.31.feed_forward.w1.weight [8192,3072] -987.2952270507812
layers.31.feed_forward.w2.weight [3072,8192] -125.24247741699219
layers.31.feed_forward.w3.weight [8192,3072] 14.822467803955078
layers.31.attention_norm.weight [3072] 3137.328125
layers.31.ffn_norm.weight [3072] 2949.994384765625
layers.31.adaLN_modulation.1.weight [18432,1024] -3916.6396484375
layers.31.adaLN_modulation.1.bias [18432] 52.58620071411133
layers.31.attention_y_norm.weight [4096] 4095.30322265625
final_layer.linear.weight [32,3072] 18.008819580078125
final_layer.linear.bias [32] 0.39991268515586853
final_layer.adaLN_modulation.1.weight [6144,1024] 10919.576171875
final_layer.adaLN_modulation.1.bias [6144] -84.26333618164062