diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-1.5B-Instruct.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-1.5B-Instruct.json new file mode 100644 index 000000000..99168b7b2 --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-1.5B-Instruct.json @@ -0,0 +1,1022 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 70.50235748291016, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 134.4214630126953, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 235.74794006347656, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 73.25755310058594, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 27.22879981994629, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 17.5551815032959, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 54.210426330566406, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 38.808937072753906, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 29.799747467041016, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 10.296355247497559, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 8.86428165435791, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 6.43813943862915, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 7.0912184715271, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 3.285884141921997, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 6.073758125305176, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 5.325990676879883, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 4.591946601867676, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 7.021907329559326, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 6.392782211303711, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 210.51983642578125, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 7.1035943031311035, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 18.701711654663086, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 14.842622756958008, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 10.50004768371582, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 7.225146770477295, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 7.463952541351318, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 15.226134300231934, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 105.4173355102539, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.5021594166755676, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 34.75935363769531, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 22.855531692504883, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 25.09166717529297, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 28.533172607421875, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 18.625717163085938, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 39.77565383911133, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 24.77678680419922, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 11.854388236999512, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 20.372356414794922, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 14.639552116394043, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 9.82955551147461, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 13.942151069641113, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 12.524999618530273, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 8.19681167602539, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 8.561081886291504, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 6.421900749206543, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 5.568161964416504, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 10.090147972106934, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 5.6181230545043945, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 5.173826694488525, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 5.663441181182861, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 6.824708461761475, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 4.724992275238037, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 6.829834938049316, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 9.968582153320312, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 14.35350513458252, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 20.121768951416016, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 1.9020992517471313, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 46.9393424987793, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 76.04901123046875, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 104.08525848388672, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 77.74343872070312, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 104.15605926513672, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 105.16349792480469, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 78.4150390625, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 57.51069641113281, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 50.26409912109375, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 50.36701965332031, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 56.66413497924805, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 62.384559631347656, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 44.97883987426758, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 69.7376480102539, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 35.93111801147461, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 33.63168716430664, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 37.695919036865234, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 43.516517639160156, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 30.479318618774414, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 12.495409965515137, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 19.616689682006836, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 18.42948341369629, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 10.799560546875, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 14.167623519897461, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 14.938597679138184, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 8.896568298339844, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 25.774547576904297, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 1.8306859731674194, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.896544337272644, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 2.345759868621826, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 2.0610744953155518, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 2.3658556938171387, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 1.6586917638778687, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 1.7613047361373901, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 1.325312852859497, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 1.458108901977539, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 1.4319790601730347, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 0.9579543471336365, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 0.8787619471549988, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 1.0447536706924438, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 0.9157310724258423, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.7528730630874634, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 0.9293556213378906, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 0.8057093620300293, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 1.2973601818084717, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 1.1357901096343994, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 1.3661632537841797, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 0.8829066753387451, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 0.9105398654937744, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 2.086926221847534, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 1.0393351316452026, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 1.114574670791626, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 2.599745035171509, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 1.1256712675094604, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 1.1784162521362305, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.8094121813774109, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.22000817954540253, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.21972468495368958, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.22064059972763062, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.22308556735515594, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.22396250069141388, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.228360116481781, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.2306283563375473, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.2430228292942047, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.2115175724029541, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.18226943910121918, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.144245907664299, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.21965907514095306, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.1797526627779007, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.26513636112213135, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.19463808834552765, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.22129350900650024, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.22545330226421356, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.25302645564079285, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.26326504349708557, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.15203869342803955, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.22418837249279022, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.23777326941490173, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.18076598644256592, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.19919466972351074, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.11310968548059464, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.08452697843313217, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.1029304787516594, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.03922705352306366, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.1410205066204071, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.18240582942962646, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.1702580451965332, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.19508686661720276, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.21549257636070251, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.22021502256393433, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.2044307142496109, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.22745060920715332, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.23825915157794952, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.2181481122970581, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.23490090668201447, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.2379382699728012, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.19233369827270508, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.2587313652038574, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.07332809269428253, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.22992204129695892, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.2537729740142822, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.2389948070049286, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.20716068148612976, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.2575169503688812, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.22347678244113922, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.18831054866313934, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.19853907823562622, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.16343259811401367, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.1583252102136612, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.254446804523468, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.23828543722629547, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 856.5148315429688, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 48.941104888916016, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": 70.25466918945312, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": 370.885986328125, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 75.51139831542969, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 52.004058837890625, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": 641.026611328125, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": 323.4858093261719, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": 2.1745388507843018, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": 3.0791690349578857, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": 2.029968023300171, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-1.5B.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-1.5B.json new file mode 100644 index 000000000..41cde8e6b --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-1.5B.json @@ -0,0 +1,1022 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 70.4939193725586, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 134.2310028076172, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 235.44140625, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 73.19381713867188, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 27.216264724731445, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 17.544504165649414, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 54.17462158203125, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 38.78171920776367, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 29.777149200439453, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 10.289377212524414, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 8.858332633972168, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 6.433396816253662, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 7.085702419281006, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 3.323948383331299, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 6.204164505004883, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 5.321533203125, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 4.588479995727539, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 7.01450252532959, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 6.386813163757324, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 210.38458251953125, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 7.096683979034424, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 18.68245506286621, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 14.824685096740723, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 10.491303443908691, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 7.2194437980651855, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 7.458613872528076, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 15.222760200500488, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 105.41569519042969, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.5017311573028564, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 34.71562576293945, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 22.82915496826172, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 25.0699520111084, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 28.508079528808594, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 18.608009338378906, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 39.732391357421875, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 24.760026931762695, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 11.842738151550293, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 20.35906982421875, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 14.627532958984375, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 9.821962356567383, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 13.930404663085938, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 12.509871482849121, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 8.187695503234863, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 8.553187370300293, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 6.414614200592041, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 5.561778545379639, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 10.078697204589844, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 5.61345100402832, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 5.265484809875488, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 5.659949779510498, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 6.8203511238098145, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 4.721294403076172, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 6.82572603225708, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 9.963521003723145, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 14.342291831970215, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 20.092098236083984, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 1.901187777519226, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 46.9141731262207, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 76.07878112792969, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 103.9194564819336, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 77.62561798095703, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 104.01624298095703, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 105.0235366821289, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 78.33445739746094, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 57.44070816040039, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 50.20344924926758, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 50.32845687866211, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 56.6197624206543, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 62.338096618652344, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 44.92917251586914, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 69.69624328613281, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 35.90705108642578, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 33.610374450683594, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 37.67365646362305, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 43.488929748535156, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 30.451993942260742, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 12.480182647705078, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 19.595102310180664, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 19.067970275878906, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 10.786394119262695, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 14.150126457214355, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 14.927021026611328, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 8.891448020935059, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 25.74305534362793, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 1.7818864583969116, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.8955822587013245, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 2.344149351119995, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 2.0597119331359863, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 2.36411714553833, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 1.6570613384246826, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 1.7604507207870483, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 1.3245182037353516, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 1.4567548036575317, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 1.4310829639434814, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 0.95713210105896, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 0.8781776428222656, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 1.0438013076782227, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 0.9315219521522522, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.7521569728851318, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 0.9286947250366211, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 0.8047553896903992, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 1.2965552806854248, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 1.134974479675293, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 1.3648872375488281, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 0.8667459487915039, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 0.9100639224052429, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 2.127535820007324, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 1.0382369756698608, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 1.113753318786621, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 2.597890853881836, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 1.1248247623443604, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 1.1984941959381104, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.8139898777008057, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.21965594589710236, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.219479501247406, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.22144284844398499, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.22390463948249817, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.22383669018745422, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.22818723320960999, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.23134392499923706, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.24275101721286774, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.21139128506183624, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.18210072815418243, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.14415481686592102, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.21947966516017914, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.17875106632709503, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.264996200799942, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.19353187084197998, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.22111012041568756, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.2242278754711151, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.2527434229850769, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.26184532046318054, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.1519661247730255, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.22386522591114044, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.2386160045862198, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.18057651817798615, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.1989467740058899, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.11306505650281906, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.08449216932058334, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.10287519544363022, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.039204664528369904, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.14075909554958344, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.18212397396564484, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.1700422316789627, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.1948907971382141, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.2153141051530838, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.21998055279254913, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.20416118204593658, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.2272879034280777, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.23795834183692932, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.21887299418449402, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.23469635844230652, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.23774078488349915, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.1920779049396515, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.2584812641143799, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.07330238074064255, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.23073157668113708, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.2523840367794037, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.23874858021736145, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.20698708295822144, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.25723400712013245, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.223300039768219, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.18824049830436707, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.19840741157531738, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.16326843202114105, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.1581888198852539, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.25306230783462524, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.23808495700359344, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 864.8881225585938, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 48.853694915771484, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": 70.18457794189453, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": 371.1153259277344, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 75.41203308105469, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 51.92624282836914, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": 642.9313354492188, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": 323.5724182128906, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": 2.1736748218536377, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": 3.1729259490966797, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": 2.024953842163086, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-3B-Instruct.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-3B-Instruct.json new file mode 100644 index 000000000..6a67b14b3 --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-3B-Instruct.json @@ -0,0 +1,1310 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.28.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.29.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.30.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.31.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.32.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.33.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.34.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.35.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 20.964319229125977, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 0.11561352014541626, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 0.14991413056850433, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 0.3673713207244873, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 0.5076134204864502, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 33.89468002319336, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 45.08732986450195, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 33.234222412109375, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 29.3447322845459, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 26.664169311523438, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 22.323949813842773, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 18.259737014770508, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 14.422037124633789, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 22.172054290771484, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 27.363698959350586, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 28.474334716796875, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 10.4143648147583, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 10.719133377075195, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 8.6494722366333, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 5.69321870803833, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 23.889677047729492, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 11.59121036529541, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 5.997435569763184, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 19.415578842163086, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 8.241704940795898, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 12.993823051452637, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 36.26508712768555, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 19.957971572875977, + "type": "mlp.down_proj" + }, + "model.layers.28.mlp.down_proj": { + "snr": 6.067765235900879, + "type": "mlp.down_proj" + }, + "model.layers.29.mlp.down_proj": { + "snr": 5.369481086730957, + "type": "mlp.down_proj" + }, + "model.layers.30.mlp.down_proj": { + "snr": 7.358774662017822, + "type": "mlp.down_proj" + }, + "model.layers.31.mlp.down_proj": { + "snr": 7.8687238693237305, + "type": "mlp.down_proj" + }, + "model.layers.32.mlp.down_proj": { + "snr": 8.713484764099121, + "type": "mlp.down_proj" + }, + "model.layers.33.mlp.down_proj": { + "snr": 21.233531951904297, + "type": "mlp.down_proj" + }, + "model.layers.34.mlp.down_proj": { + "snr": 32.37357711791992, + "type": "mlp.down_proj" + }, + "model.layers.35.mlp.down_proj": { + "snr": 179.8053741455078, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.24989914894104004, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 0.11613649874925613, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 0.16354432702064514, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 0.36216047406196594, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 0.3485107719898224, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 2.6546616554260254, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 8.362885475158691, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 7.38665246963501, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 13.016111373901367, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 14.94902515411377, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 20.92418670654297, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 15.954015731811523, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 8.980009078979492, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 17.59958267211914, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 17.23070526123047, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 23.725330352783203, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 17.000444412231445, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 18.293012619018555, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 12.644190788269043, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 16.278690338134766, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 7.407368183135986, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 6.109912395477295, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 5.3692426681518555, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 9.354235649108887, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 7.655010223388672, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 6.252986431121826, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 14.26718521118164, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 7.705836772918701, + "type": "mlp.gate_proj" + }, + "model.layers.28.mlp.gate_proj": { + "snr": 5.998677730560303, + "type": "mlp.gate_proj" + }, + "model.layers.29.mlp.gate_proj": { + "snr": 6.044872760772705, + "type": "mlp.gate_proj" + }, + "model.layers.30.mlp.gate_proj": { + "snr": 9.027137756347656, + "type": "mlp.gate_proj" + }, + "model.layers.31.mlp.gate_proj": { + "snr": 5.449969291687012, + "type": "mlp.gate_proj" + }, + "model.layers.32.mlp.gate_proj": { + "snr": 4.206825256347656, + "type": "mlp.gate_proj" + }, + "model.layers.33.mlp.gate_proj": { + "snr": 5.22825288772583, + "type": "mlp.gate_proj" + }, + "model.layers.34.mlp.gate_proj": { + "snr": 43.71927261352539, + "type": "mlp.gate_proj" + }, + "model.layers.35.mlp.gate_proj": { + "snr": 45.37385177612305, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 0.7069714665412903, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 0.17766596376895905, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 0.28577035665512085, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 0.6763099431991577, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 0.8340913653373718, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 3.946547031402588, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 19.56715202331543, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 36.21149826049805, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 44.28759002685547, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 45.47198486328125, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 79.00128936767578, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 52.28038787841797, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 48.08102035522461, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 56.071285247802734, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 72.24358367919922, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 54.818233489990234, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 47.251495361328125, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 51.585636138916016, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 43.47938919067383, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 38.132469177246094, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 21.78435707092285, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 22.261096954345703, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 30.751861572265625, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 28.61063575744629, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 20.21415901184082, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 20.759052276611328, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 33.80818557739258, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 17.274362564086914, + "type": "mlp.up_proj" + }, + "model.layers.28.mlp.up_proj": { + "snr": 13.943653106689453, + "type": "mlp.up_proj" + }, + "model.layers.29.mlp.up_proj": { + "snr": 16.202186584472656, + "type": "mlp.up_proj" + }, + "model.layers.30.mlp.up_proj": { + "snr": 24.25114631652832, + "type": "mlp.up_proj" + }, + "model.layers.31.mlp.up_proj": { + "snr": 10.68645191192627, + "type": "mlp.up_proj" + }, + "model.layers.32.mlp.up_proj": { + "snr": 5.7449774742126465, + "type": "mlp.up_proj" + }, + "model.layers.33.mlp.up_proj": { + "snr": 11.879876136779785, + "type": "mlp.up_proj" + }, + "model.layers.34.mlp.up_proj": { + "snr": 25.948715209960938, + "type": "mlp.up_proj" + }, + "model.layers.35.mlp.up_proj": { + "snr": 38.63526153564453, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.28.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.29.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.30.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.31.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.32.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.33.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.34.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.35.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 12.243099212646484, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.6446183323860168, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 0.7159711718559265, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 5.5100932121276855, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 3.0802414417266846, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 1.0472767353057861, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 3.576918601989746, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 3.3793225288391113, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 2.9598212242126465, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 6.102792263031006, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 2.231630325317383, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 2.176372766494751, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 1.3229435682296753, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 2.6183862686157227, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 2.608288526535034, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 1.5090984106063843, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 1.284422516822815, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 0.8903945088386536, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 1.8880385160446167, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 0.8905735015869141, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 0.9060881733894348, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 0.7572551965713501, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 0.940827488899231, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 3.7776191234588623, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 1.328923225402832, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 1.3986345529556274, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 1.2436336278915405, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 0.7737217545509338, + "type": "self_attn.k_proj" + }, + "model.layers.28.self_attn.k_proj": { + "snr": 2.6027626991271973, + "type": "self_attn.k_proj" + }, + "model.layers.29.self_attn.k_proj": { + "snr": 2.2332751750946045, + "type": "self_attn.k_proj" + }, + "model.layers.30.self_attn.k_proj": { + "snr": 2.476585626602173, + "type": "self_attn.k_proj" + }, + "model.layers.31.self_attn.k_proj": { + "snr": 1.1115432977676392, + "type": "self_attn.k_proj" + }, + "model.layers.32.self_attn.k_proj": { + "snr": 0.8251476287841797, + "type": "self_attn.k_proj" + }, + "model.layers.33.self_attn.k_proj": { + "snr": 0.9331105947494507, + "type": "self_attn.k_proj" + }, + "model.layers.34.self_attn.k_proj": { + "snr": 6.602395534515381, + "type": "self_attn.k_proj" + }, + "model.layers.35.self_attn.k_proj": { + "snr": 10.151693344116211, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.3661542534828186, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.19571374356746674, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.2244851142168045, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.2593664526939392, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.2569783926010132, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.2564302980899811, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.18539844453334808, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.2328651398420334, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.22055882215499878, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.21800543367862701, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.22867777943611145, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.23986175656318665, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.17598563432693481, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.20469218492507935, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.21040217578411102, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.23787625133991241, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.16339677572250366, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.2070712298154831, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.1826934814453125, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.19459959864616394, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.2668156027793884, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.16906610131263733, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.18790249526500702, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.18883933126926422, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.1793188899755478, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.1800570785999298, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.17790433764457703, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.2029498964548111, + "type": "self_attn.o_proj" + }, + "model.layers.28.self_attn.o_proj": { + "snr": 0.17044201493263245, + "type": "self_attn.o_proj" + }, + "model.layers.29.self_attn.o_proj": { + "snr": 0.19938386976718903, + "type": "self_attn.o_proj" + }, + "model.layers.30.self_attn.o_proj": { + "snr": 0.23108959197998047, + "type": "self_attn.o_proj" + }, + "model.layers.31.self_attn.o_proj": { + "snr": 0.16427059471607208, + "type": "self_attn.o_proj" + }, + "model.layers.32.self_attn.o_proj": { + "snr": 0.10631092637777328, + "type": "self_attn.o_proj" + }, + "model.layers.33.self_attn.o_proj": { + "snr": 0.09417019784450531, + "type": "self_attn.o_proj" + }, + "model.layers.34.self_attn.o_proj": { + "snr": 0.1324978619813919, + "type": "self_attn.o_proj" + }, + "model.layers.35.self_attn.o_proj": { + "snr": 0.11784011125564575, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.05565479397773743, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.138458251953125, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.12992437183856964, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.15362468361854553, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.1563446819782257, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.15544593334197998, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.15956827998161316, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.17549948394298553, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.16668449342250824, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.15626586973667145, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.18318884074687958, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.171547532081604, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.18164905905723572, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.2091975212097168, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.17431670427322388, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.20902502536773682, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.15439842641353607, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.1945274919271469, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.18916545808315277, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.20778712630271912, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.20866931974887848, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.1900305300951004, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.18200653791427612, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.2070988416671753, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.1845332235097885, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.20868781208992004, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.19242744147777557, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.15225112438201904, + "type": "self_attn.q_proj" + }, + "model.layers.28.self_attn.q_proj": { + "snr": 0.20065009593963623, + "type": "self_attn.q_proj" + }, + "model.layers.29.self_attn.q_proj": { + "snr": 0.19390477240085602, + "type": "self_attn.q_proj" + }, + "model.layers.30.self_attn.q_proj": { + "snr": 0.18538697063922882, + "type": "self_attn.q_proj" + }, + "model.layers.31.self_attn.q_proj": { + "snr": 0.18954339623451233, + "type": "self_attn.q_proj" + }, + "model.layers.32.self_attn.q_proj": { + "snr": 0.20089596509933472, + "type": "self_attn.q_proj" + }, + "model.layers.33.self_attn.q_proj": { + "snr": 0.19814996421337128, + "type": "self_attn.q_proj" + }, + "model.layers.34.self_attn.q_proj": { + "snr": 0.17733213305473328, + "type": "self_attn.q_proj" + }, + "model.layers.35.self_attn.q_proj": { + "snr": 0.14075976610183716, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 845.8053588867188, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 83.97241973876953, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": 213.70960998535156, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": 18.950267791748047, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": 435.8339538574219, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.28.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.29.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.30.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.31.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.32.self_attn.v_proj": { + "snr": 1.2341279983520508, + "type": "self_attn.v_proj" + }, + "model.layers.33.self_attn.v_proj": { + "snr": 0.6158654689788818, + "type": "self_attn.v_proj" + }, + "model.layers.34.self_attn.v_proj": { + "snr": 509.3221130371094, + "type": "self_attn.v_proj" + }, + "model.layers.35.self_attn.v_proj": { + "snr": 538.6658325195312, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-3B.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-3B.json new file mode 100644 index 000000000..93b5cfec6 --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-3B.json @@ -0,0 +1,1310 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.28.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.29.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.30.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.31.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.32.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.33.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.34.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.35.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 20.942785263061523, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 0.11550866067409515, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 0.14981402456760406, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 0.36719316244125366, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 0.5072987079620361, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 33.86688232421875, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 45.066246032714844, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 33.20981979370117, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 29.310104370117188, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 26.638381958007812, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 22.302486419677734, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 18.249290466308594, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 14.057564735412598, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 22.154281616210938, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 27.348575592041016, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 28.447378158569336, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 10.405216217041016, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 10.71042251586914, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 8.642854690551758, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 5.690433979034424, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 23.869070053100586, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 11.584356307983398, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 5.992950916290283, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 18.495361328125, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 8.233827590942383, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 12.626734733581543, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 36.21802520751953, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 19.932941436767578, + "type": "mlp.down_proj" + }, + "model.layers.28.mlp.down_proj": { + "snr": 6.0616455078125, + "type": "mlp.down_proj" + }, + "model.layers.29.mlp.down_proj": { + "snr": 5.363720417022705, + "type": "mlp.down_proj" + }, + "model.layers.30.mlp.down_proj": { + "snr": 7.455615520477295, + "type": "mlp.down_proj" + }, + "model.layers.31.mlp.down_proj": { + "snr": 7.8631815910339355, + "type": "mlp.down_proj" + }, + "model.layers.32.mlp.down_proj": { + "snr": 8.706913948059082, + "type": "mlp.down_proj" + }, + "model.layers.33.mlp.down_proj": { + "snr": 21.220134735107422, + "type": "mlp.down_proj" + }, + "model.layers.34.mlp.down_proj": { + "snr": 32.33852005004883, + "type": "mlp.down_proj" + }, + "model.layers.35.mlp.down_proj": { + "snr": 179.8906707763672, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.24970805644989014, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 0.11607512086629868, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 0.16310769319534302, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 0.3621424436569214, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 0.3482637107372284, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 2.6533455848693848, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 8.359040260314941, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 7.382037162780762, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 13.00683879852295, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 14.936161994934082, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 20.907283782958984, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 15.941497802734375, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 8.97419548034668, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 17.585100173950195, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 17.21462059020996, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 23.703285217285156, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 16.986576080322266, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 18.27729606628418, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 12.63351058959961, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 16.2633113861084, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 7.399787902832031, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 6.10424280166626, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 5.363350868225098, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 9.344535827636719, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 7.647364616394043, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 6.143579959869385, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 14.254817008972168, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 7.7000861167907715, + "type": "mlp.gate_proj" + }, + "model.layers.28.mlp.gate_proj": { + "snr": 5.994422435760498, + "type": "mlp.gate_proj" + }, + "model.layers.29.mlp.gate_proj": { + "snr": 6.041909694671631, + "type": "mlp.gate_proj" + }, + "model.layers.30.mlp.gate_proj": { + "snr": 9.027522087097168, + "type": "mlp.gate_proj" + }, + "model.layers.31.mlp.gate_proj": { + "snr": 5.450753211975098, + "type": "mlp.gate_proj" + }, + "model.layers.32.mlp.gate_proj": { + "snr": 4.149200439453125, + "type": "mlp.gate_proj" + }, + "model.layers.33.mlp.gate_proj": { + "snr": 5.223763942718506, + "type": "mlp.gate_proj" + }, + "model.layers.34.mlp.gate_proj": { + "snr": 43.65521240234375, + "type": "mlp.gate_proj" + }, + "model.layers.35.mlp.gate_proj": { + "snr": 45.312774658203125, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 0.7065013647079468, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 0.17752516269683838, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 0.2847473919391632, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 0.6757690906524658, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 0.8353318572044373, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 3.940711736679077, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 19.556047439575195, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 36.19340515136719, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 44.2518424987793, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 45.418025970458984, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 78.90928649902344, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 52.24648666381836, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 48.02030563354492, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 56.016239166259766, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 72.16619873046875, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 54.75283432006836, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 47.204097747802734, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 51.549312591552734, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 43.43872833251953, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 38.09785461425781, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 21.767858505249023, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 22.243661880493164, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 30.71843147277832, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 28.5756778717041, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 20.186717987060547, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 20.742860794067383, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 33.777984619140625, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 17.254213333129883, + "type": "mlp.up_proj" + }, + "model.layers.28.mlp.up_proj": { + "snr": 13.930026054382324, + "type": "mlp.up_proj" + }, + "model.layers.29.mlp.up_proj": { + "snr": 16.17984390258789, + "type": "mlp.up_proj" + }, + "model.layers.30.mlp.up_proj": { + "snr": 24.236648559570312, + "type": "mlp.up_proj" + }, + "model.layers.31.mlp.up_proj": { + "snr": 10.665648460388184, + "type": "mlp.up_proj" + }, + "model.layers.32.mlp.up_proj": { + "snr": 5.735939025878906, + "type": "mlp.up_proj" + }, + "model.layers.33.mlp.up_proj": { + "snr": 11.592061042785645, + "type": "mlp.up_proj" + }, + "model.layers.34.mlp.up_proj": { + "snr": 25.923419952392578, + "type": "mlp.up_proj" + }, + "model.layers.35.mlp.up_proj": { + "snr": 38.579349517822266, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.28.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.29.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.30.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.31.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.32.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.33.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.34.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.35.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 12.24727725982666, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.6436238288879395, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 0.7156716585159302, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 5.505439758300781, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 3.0760715007781982, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 1.0453941822052002, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 3.57472562789917, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 3.3765170574188232, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 2.8859639167785645, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 6.09852409362793, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 2.229580879211426, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 2.173879623413086, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 1.3220131397247314, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 2.61668062210083, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 2.606799840927124, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 1.5080311298370361, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 1.2841484546661377, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 0.8896433115005493, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 1.8873414993286133, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 0.8897770643234253, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 0.9051405787467957, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 0.7568970322608948, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 0.9403582811355591, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 3.777062177658081, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 1.3280683755874634, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 1.3980307579040527, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 1.2435240745544434, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 0.7732619047164917, + "type": "self_attn.k_proj" + }, + "model.layers.28.self_attn.k_proj": { + "snr": 2.6010243892669678, + "type": "self_attn.k_proj" + }, + "model.layers.29.self_attn.k_proj": { + "snr": 2.232773780822754, + "type": "self_attn.k_proj" + }, + "model.layers.30.self_attn.k_proj": { + "snr": 2.4743099212646484, + "type": "self_attn.k_proj" + }, + "model.layers.31.self_attn.k_proj": { + "snr": 1.11082923412323, + "type": "self_attn.k_proj" + }, + "model.layers.32.self_attn.k_proj": { + "snr": 0.8243986368179321, + "type": "self_attn.k_proj" + }, + "model.layers.33.self_attn.k_proj": { + "snr": 0.932928204536438, + "type": "self_attn.k_proj" + }, + "model.layers.34.self_attn.k_proj": { + "snr": 6.608611583709717, + "type": "self_attn.k_proj" + }, + "model.layers.35.self_attn.k_proj": { + "snr": 10.160987854003906, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.36662933230400085, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.1955128312110901, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.22419843077659607, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.25902292132377625, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.2567676901817322, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.2560890316963196, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.18518221378326416, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.23254290223121643, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.2203962802886963, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.217017263174057, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.22843335568904877, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.23816843330860138, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.17585325241088867, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.20451271533966064, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.2095799297094345, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.23767071962356567, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.16328400373458862, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.20690056681632996, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.18191492557525635, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.1945018619298935, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.26658856868743896, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.16897724568843842, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.18773262202739716, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.18808405101299286, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.17919476330280304, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.1793426126241684, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.1777871698141098, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.20279864966869354, + "type": "self_attn.o_proj" + }, + "model.layers.28.self_attn.o_proj": { + "snr": 0.17030371725559235, + "type": "self_attn.o_proj" + }, + "model.layers.29.self_attn.o_proj": { + "snr": 0.1992504596710205, + "type": "self_attn.o_proj" + }, + "model.layers.30.self_attn.o_proj": { + "snr": 0.23085352778434753, + "type": "self_attn.o_proj" + }, + "model.layers.31.self_attn.o_proj": { + "snr": 0.1641533523797989, + "type": "self_attn.o_proj" + }, + "model.layers.32.self_attn.o_proj": { + "snr": 0.10621391236782074, + "type": "self_attn.o_proj" + }, + "model.layers.33.self_attn.o_proj": { + "snr": 0.09411631524562836, + "type": "self_attn.o_proj" + }, + "model.layers.34.self_attn.o_proj": { + "snr": 0.13239727914333344, + "type": "self_attn.o_proj" + }, + "model.layers.35.self_attn.o_proj": { + "snr": 0.11740171164274216, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.055595725774765015, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.13823610544204712, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.1297825127840042, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.15291297435760498, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.15615035593509674, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.15535500645637512, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.15993140637874603, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.1753682643175125, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.1664913445711136, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.15656901895999908, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.18300014734268188, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.1713649481534958, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.1809009313583374, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.20895132422447205, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.17413195967674255, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.20878490805625916, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.1547088772058487, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.1943129003047943, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.1889297217130661, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.207680344581604, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.20839959383010864, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.18989044427871704, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.18180623650550842, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.2069384753704071, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.1842993050813675, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.2078687846660614, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.19224946200847626, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.15170617401599884, + "type": "self_attn.q_proj" + }, + "model.layers.28.self_attn.q_proj": { + "snr": 0.20116600394248962, + "type": "self_attn.q_proj" + }, + "model.layers.29.self_attn.q_proj": { + "snr": 0.19373668730258942, + "type": "self_attn.q_proj" + }, + "model.layers.30.self_attn.q_proj": { + "snr": 0.18462225794792175, + "type": "self_attn.q_proj" + }, + "model.layers.31.self_attn.q_proj": { + "snr": 0.18939673900604248, + "type": "self_attn.q_proj" + }, + "model.layers.32.self_attn.q_proj": { + "snr": 0.20071947574615479, + "type": "self_attn.q_proj" + }, + "model.layers.33.self_attn.q_proj": { + "snr": 0.19740056991577148, + "type": "self_attn.q_proj" + }, + "model.layers.34.self_attn.q_proj": { + "snr": 0.17658494412899017, + "type": "self_attn.q_proj" + }, + "model.layers.35.self_attn.q_proj": { + "snr": 0.1407373696565628, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 846.30126953125, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 83.83415222167969, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": 213.51316833496094, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": 18.92746925354004, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": 433.9771728515625, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.28.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.29.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.30.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.31.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.32.self_attn.v_proj": { + "snr": 1.2332282066345215, + "type": "self_attn.v_proj" + }, + "model.layers.33.self_attn.v_proj": { + "snr": 0.6151890158653259, + "type": "self_attn.v_proj" + }, + "model.layers.34.self_attn.v_proj": { + "snr": 509.7169189453125, + "type": "self_attn.v_proj" + }, + "model.layers.35.self_attn.v_proj": { + "snr": 536.0748901367188, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-7B-Instruct.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-7B-Instruct.json new file mode 100644 index 000000000..0e18bd386 --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-7B-Instruct.json @@ -0,0 +1,1022 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 10.283808708190918, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 1.2089825868606567, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 19.309062957763672, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 50.174461364746094, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 114.28582763671875, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 215.5762176513672, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 204.5117950439453, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 182.5479278564453, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 74.92950439453125, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 16.482666015625, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 55.33920669555664, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 16.851062774658203, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 58.65230178833008, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 11.150161743164062, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 65.32643127441406, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 46.736305236816406, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 14.288785934448242, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 23.40110206604004, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 86.34363555908203, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 49.14613342285156, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 1276.84814453125, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 51.803409576416016, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 143.0666046142578, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 35.14984893798828, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 21.41700553894043, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 10.651569366455078, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 21.635149002075195, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 1446.2774658203125, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.04497330263257027, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 0.16888172924518585, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 0.33653727173805237, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 3.1445391178131104, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 9.107144355773926, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 15.909018516540527, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 60.9138069152832, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 57.570281982421875, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 65.82791137695312, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 10.455283164978027, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 26.970706939697266, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 31.139820098876953, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 43.987159729003906, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 20.704849243164062, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 21.191452026367188, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 42.66447830200195, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 22.136825561523438, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 22.60980987548828, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 81.80574035644531, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 20.88619613647461, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 58.3524055480957, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 22.786706924438477, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 16.932226181030273, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 16.819862365722656, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 19.76348304748535, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 28.98714256286621, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 36.7071533203125, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 51.81539535522461, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 0.2243107706308365, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 0.4464716613292694, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 1.7838181257247925, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 17.912736892700195, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 47.45841979980469, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 56.3084602355957, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 173.33717346191406, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 148.22750854492188, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 133.63565063476562, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 83.65129852294922, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 117.94369506835938, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 94.52413940429688, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 130.43333435058594, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 76.11975860595703, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 158.75192260742188, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 143.72706604003906, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 84.28279876708984, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 116.65055084228516, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 177.1201934814453, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 82.4564437866211, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 137.73019409179688, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 89.97538757324219, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 86.30876159667969, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 61.53449249267578, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 45.22392654418945, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 60.3155517578125, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 40.06092071533203, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 48.12322998046875, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": 0.08805440366268158, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 4.771554470062256, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.46674421429634094, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 1.6167784929275513, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 2.0980119705200195, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 1.4339035749435425, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 1.7446703910827637, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 1.2829725742340088, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 2.2314982414245605, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 1.5125916004180908, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 1.2817912101745605, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 3.3553454875946045, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 1.591347336769104, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 1.1114169359207153, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 1.1536189317703247, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.994098424911499, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 1.484580636024475, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 1.2999093532562256, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 2.1628623008728027, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 1.3842225074768066, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 1.440075159072876, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 1.7816450595855713, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 1.746536135673523, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 1.318993091583252, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 1.7234206199645996, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 2.586996555328369, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 1.6486897468566895, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 1.3349357843399048, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 0.9039687514305115, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.10605750232934952, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.2503393292427063, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.21453581750392914, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.20600366592407227, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.22004099190235138, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.2267625778913498, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.1736888736486435, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.2314220815896988, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.24031606316566467, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.13458871841430664, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.20170633494853973, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.19507651031017303, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.1862162947654724, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.15117767453193665, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.1857745349407196, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.2064860314130783, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.15419450402259827, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.17895667254924774, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.18284623324871063, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.17497135698795319, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.178844153881073, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.16190896928310394, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.19371949136257172, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.14116843044757843, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.14100700616836548, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.14792074263095856, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.11953117698431015, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.06241385638713837, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.02127065323293209, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.14693336188793182, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.16316214203834534, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.1218630000948906, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.13916714489459991, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.155359148979187, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.1590007096529007, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.1958903819322586, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.22448301315307617, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.20126597583293915, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.1980895698070526, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.2289486974477768, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.22922305762767792, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.21452386677265167, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.24151542782783508, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.21893717348575592, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.2321016639471054, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.24078059196472168, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.22774985432624817, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.20914016664028168, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.22847522795200348, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.2500442862510681, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.2353251725435257, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.20365388691425323, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.21967172622680664, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.2122868150472641, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.2415798157453537, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.12347634881734848, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 230.88636779785156, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": 22.38136100769043, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 246.59597778320312, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 499.61761474609375, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": 69.18345642089844, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 984.9320068359375, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 64.06214141845703, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": 28.43911361694336, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": 725.1439819335938, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": 63.43681716918945, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": 238.4695587158203, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": 111.88697814941406, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": 686.2830200195312, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": 566.2647705078125, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": 4.070064544677734, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": 4.3411664962768555, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-7B.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-7B.json new file mode 100644 index 000000000..af13ecf9e --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_Qwen-Qwen2.5-7B.json @@ -0,0 +1,1022 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 10.277782440185547, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 1.2050706148147583, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 19.284534454345703, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 50.16513442993164, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 114.24882507324219, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 215.48194885253906, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 204.39431762695312, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 182.5116729736328, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 74.9266128540039, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 16.474102020263672, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 55.30583572387695, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 16.84047508239746, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 58.62131118774414, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 11.144298553466797, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 65.28057098388672, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 46.701290130615234, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 14.278325080871582, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 23.382247924804688, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 93.8782958984375, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 49.10498809814453, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 1277.5101318359375, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 51.7880859375, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 143.03504943847656, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 35.123931884765625, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 21.403743743896484, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 10.551352500915527, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 21.62333869934082, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 1541.98681640625, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.04497644677758217, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 0.16878646612167358, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 0.336302250623703, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 3.141293525695801, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 9.098686218261719, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 15.89354419708252, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 60.85503387451172, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 57.53098678588867, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 65.77096557617188, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 10.453179359436035, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 26.94801139831543, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 31.111093521118164, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 43.963191986083984, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 20.690765380859375, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 20.47557258605957, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 42.63906478881836, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 22.11542320251465, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 22.590566635131836, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 81.74773406982422, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 20.872997283935547, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 58.32197952270508, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 22.784095764160156, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 16.935768127441406, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 16.830224990844727, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 19.774564743041992, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 27.770675659179688, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 36.714595794677734, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 51.81637191772461, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 0.22425401210784912, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 0.4456978142261505, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 1.7769725322723389, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 17.8966121673584, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 47.43608856201172, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 56.2298698425293, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 173.1498260498047, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 148.02874755859375, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 133.5174560546875, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 83.45183563232422, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 117.88772583007812, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 94.41156768798828, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 130.3107452392578, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 76.04458618164062, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 158.59634399414062, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 143.59596252441406, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 84.2161636352539, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 116.55204010009766, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 176.95449829101562, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 82.37284088134766, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 137.5695343017578, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 89.87335205078125, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 86.1510238647461, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 61.37428665161133, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 45.10757064819336, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 60.16519546508789, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 39.96969223022461, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 48.04258346557617, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": 0.08800078183412552, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 4.764852046966553, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.46627077460289, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 1.6155915260314941, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 2.096365451812744, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 1.431254267692566, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 1.7440669536590576, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 1.2815033197402954, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 2.2301025390625, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 1.5116536617279053, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 1.2699830532073975, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 3.3086464405059814, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 1.59111487865448, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 1.1007944345474243, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 1.163416862487793, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.9935113787651062, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 1.483581304550171, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 1.2992271184921265, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 2.162485122680664, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 1.3841017484664917, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 1.453418493270874, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 1.781678557395935, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 1.7460925579071045, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 1.3188031911849976, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 1.723441243171692, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 2.585094928741455, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 1.6478856801986694, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 1.3221096992492676, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 0.9034463167190552, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.10636883229017258, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.24971255660057068, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.21437697112560272, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.2058248072862625, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.21978946030139923, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.2269466072320938, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.17318543791770935, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.23159846663475037, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.2400084286928177, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.134766086935997, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.20152011513710022, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.19492347538471222, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.18607021868228912, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.15107683837413788, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.18565276265144348, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.20626339316368103, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.1541011780500412, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.1784645915031433, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.18307389318943024, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.17449897527694702, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.1787375956773758, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.161802276968956, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.1931520402431488, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.14108893275260925, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.14064815640449524, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.14790543913841248, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.11950570344924927, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.062389008700847626, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.02138795144855976, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.14676862955093384, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.16297142207622528, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.12198334187269211, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.13921146094799042, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.15567339956760406, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.1589033454656601, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.195299431681633, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.22430908679962158, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.2011336237192154, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.1982448250055313, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.22880099713802338, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.22898294031620026, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.21394900977611542, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.24130398035049438, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.21905161440372467, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.2319282442331314, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.24004821479320526, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.22754515707492828, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.2086794078350067, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.2290779948234558, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.250373899936676, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.23474709689617157, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.20302507281303406, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.21992310881614685, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.2120121270418167, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.24161922931671143, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.12337693572044373, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 231.07347106933594, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": 22.34870719909668, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 246.30386352539062, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 499.5611572265625, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": 69.09609985351562, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 983.3341674804688, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 64.04925537109375, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": 28.41021728515625, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": 724.2736206054688, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": 63.35670852661133, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": Infinity, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": 238.2569122314453, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": 111.78319549560547, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": 687.0054931640625, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": 565.3272705078125, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": 4.064513683319092, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": 4.335177421569824, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_google-gemma-2-2b.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_google-gemma-2-2b.json new file mode 100644 index 000000000..a2d6f0457 --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_google-gemma-2-2b.json @@ -0,0 +1,1158 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": 4.538210391998291, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 7.746472358703613, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 4.3358893394470215, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 26.88057518005371, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 8.699942588806152, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 32.808380126953125, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 10.831522941589355, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 18.843679428100586, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 9.348078727722168, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 7.061270236968994, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 5.454320907592773, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 7.386133193969727, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 6.648562908172607, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 5.853652477264404, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 8.570493698120117, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 13.120837211608887, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 14.780969619750977, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 6.953134059906006, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 12.589436531066895, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 8.844094276428223, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 7.598869800567627, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 11.293925285339355, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 9.384604454040527, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 12.12533187866211, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 11.217570304870605, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 14.197714805603027, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 12.449926376342773, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 16.885862350463867, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 23.410266876220703, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 22.57662582397461, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 17.29996681213379, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 11.718637466430664, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 6.376136779785156, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 6.794021129608154, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 3.2425343990325928, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 2.368421792984009, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 3.3193087577819824, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 3.9515960216522217, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 3.2761318683624268, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 4.026322841644287, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 3.415473699569702, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 3.3418092727661133, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 3.6233012676239014, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 3.2199010848999023, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 3.6848936080932617, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 3.4439642429351807, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 3.7366604804992676, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 4.262336254119873, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 4.333253860473633, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 3.640247344970703, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 4.2978034019470215, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 4.339972496032715, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 3.8502564430236816, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 28.129924774169922, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 41.49960708618164, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 125.47801971435547, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 119.93355560302734, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 162.62631225585938, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 32.36909484863281, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 49.10078430175781, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 28.541580200195312, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 14.764090538024902, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 16.5697078704834, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 19.26059913635254, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 15.082040786743164, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 15.5792875289917, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 9.84595012664795, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 11.506875991821289, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 21.507600784301758, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 15.110466957092285, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 27.062183380126953, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 16.40383529663086, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 13.117464065551758, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 11.393353462219238, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 10.791608810424805, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 7.512388706207275, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 9.889434814453125, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 7.587779521942139, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 4.561068058013916, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": 4.538210391998291, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.1.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.2.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.3.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.4.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.5.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.6.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.7.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.8.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.9.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.10.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.11.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.12.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.13.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.14.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.15.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.16.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.17.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.18.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.19.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.20.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.21.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.22.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.23.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.24.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.25.post_feedforward_layernorm": { + "snr": Infinity, + "type": "post_feedforward_layernorm" + }, + "model.layers.0.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.1.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.2.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.3.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.4.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.5.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.6.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.7.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.8.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.9.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.10.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.11.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.12.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.13.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.14.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.15.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.16.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.17.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.18.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.19.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.20.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.21.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.22.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.23.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.24.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.25.pre_feedforward_layernorm": { + "snr": Infinity, + "type": "pre_feedforward_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 0.5685535073280334, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 1.060130000114441, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 1.0735561847686768, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 1.0217311382293701, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 0.9687430262565613, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 0.8411160111427307, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 0.936741054058075, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 0.7236003279685974, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 0.9032857418060303, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 0.7513307929039001, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 0.6875415444374084, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 0.6611058712005615, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 0.8023670315742493, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 0.7188767194747925, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.7930117249488831, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 0.9076258540153503, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 0.7295113801956177, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 0.898467481136322, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 0.9652048945426941, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 0.9855819344520569, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 1.2863355875015259, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 1.116607904434204, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 0.7438228130340576, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 0.8499895334243774, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 0.7764042019844055, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 0.7127887606620789, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.2556447386741638, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.2930974066257477, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.27571651339530945, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.280631959438324, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.2958097755908966, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.3072899580001831, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.31374114751815796, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.2903076410293579, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.2625811696052551, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.2306082546710968, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.24869701266288757, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.2556127905845642, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.28926730155944824, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.25355643033981323, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.23122912645339966, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.28772857785224915, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.22682352364063263, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.2558597922325134, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.1773315966129303, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.2106105089187622, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.2008877396583557, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.1973956972360611, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.25533634424209595, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.20066529512405396, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.18342143297195435, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.3224162459373474, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.2074502408504486, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.33233126997947693, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.3586291968822479, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.2850974202156067, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.37816473841667175, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.31616899371147156, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.4988365173339844, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.4238639175891876, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.2674674689769745, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.34524214267730713, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.4472109377384186, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.41363632678985596, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.44623735547065735, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.4404333531856537, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.5200268626213074, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.4320363700389862, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.46235284209251404, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.47477203607559204, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.4001321494579315, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.42365774512290955, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.37057873606681824, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.3990235924720764, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.35094162821769714, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.35721710324287415, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.2812618315219879, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.19463211297988892, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": 1.3365743160247803, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 2.402009963989258, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": 3.8695859909057617, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": 4.117948055267334, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": 5.651231288909912, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 2.720799446105957, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 1.4446897506713867, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": 4.497112274169922, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": 1.7241870164871216, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": 1.7104988098144531, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": 1.4231206178665161, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": 2.1643989086151123, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 1.5254249572753906, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": 2.3788745403289795, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 3.4155967235565186, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": 4.623549938201904, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": 1.5291141271591187, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": 3.9934189319610596, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": 9.035382270812988, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": 5.8578925132751465, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": 3.759958505630493, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": 4.558528900146484, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": 0.9163281917572021, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": 2.564377546310425, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": 3.689103841781616, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": 5.6444854736328125, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-1B-Instruct.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-1B-Instruct.json new file mode 100644 index 000000000..c5b5ed6ed --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-1B-Instruct.json @@ -0,0 +1,590 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 70.0594253540039, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 11.135851860046387, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 7.035482883453369, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 6.422532081604004, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 5.748020172119141, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 3.885556697845459, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 3.4336745738983154, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 2.791595935821533, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 5.36277961730957, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 4.459208011627197, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 6.272170066833496, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 5.264761447906494, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 4.324735641479492, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 3.878648042678833, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 2.9773054122924805, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 4.471445560455322, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 25.227100372314453, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 6.58299446105957, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 3.4688243865966797, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 1.555246114730835, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 0.7770601511001587, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 0.6239906549453735, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 0.6440379023551941, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 0.5120116472244263, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 0.6544050574302673, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 0.5381016731262207, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 0.622873842716217, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 0.9361700415611267, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 1.475605845451355, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 1.608325719833374, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 1.0720024108886719, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 0.7111338973045349, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 28.431896209716797, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 15.546019554138184, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 23.048023223876953, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 25.790977478027344, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 18.552549362182617, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 8.85106372833252, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 10.653799057006836, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 7.365357875823975, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 11.98373794555664, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 8.04493236541748, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 8.523039817810059, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 5.381742477416992, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 3.9845118522644043, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 3.4893221855163574, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 1.764201045036316, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 0.9730708599090576, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 0.11727584153413773, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.24786807596683502, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 0.36378130316734314, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 0.2983120381832123, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 0.33789733052253723, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 0.29155924916267395, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 0.2537297010421753, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 0.28204113245010376, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 0.2776711583137512, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 0.2927376627922058, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 0.31486213207244873, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 0.32363659143447876, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 0.31382912397384644, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 0.4635234773159027, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.25379249453544617, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 0.2628238797187805, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.27602291107177734, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.2149604707956314, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.2540294826030731, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.27978822588920593, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.3121289908885956, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.35037684440612793, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.366205096244812, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.3692712187767029, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.3301038146018982, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.3003396987915039, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.30804169178009033, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.28501132130622864, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.2171541005373001, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.19183959066867828, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.19215913116931915, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.25486502051353455, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.03850084915757179, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.0713055431842804, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.07948919385671616, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.08047746121883392, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.0852593332529068, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.09794823825359344, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.09627152234315872, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.11065381020307541, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.12031875550746918, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.09804573655128479, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.10897502303123474, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.09267337620258331, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.08803492039442062, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.0902542844414711, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.10154066979885101, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.09083802253007889, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": 2.842210054397583, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 10.59461498260498, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": 8.993025779724121, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": 62.567787170410156, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": 23.80082893371582, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 7.957369804382324, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 12.01815414428711, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": 5.095500469207764, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": 11.719332695007324, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": 555.0869750976562, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": 22.95538330078125, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": 30.042158126831055, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 9.577271461486816, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": 18.176361083984375, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 1.5695856809616089, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": 2.7235565185546875, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-1B.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-1B.json new file mode 100644 index 000000000..b84594fd7 --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-1B.json @@ -0,0 +1,590 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 57.09797286987305, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 9.538983345031738, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 6.227016925811768, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 5.660686492919922, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 5.178432464599609, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 3.5638349056243896, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 3.0918056964874268, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 2.456392288208008, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 4.525328636169434, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 3.9409055709838867, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 5.447249412536621, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 4.807600975036621, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 3.915374517440796, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 3.4820363521575928, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 2.6045074462890625, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 3.7237701416015625, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 22.160131454467773, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 6.072206020355225, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 3.2467362880706787, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 1.4111896753311157, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 0.7405938506126404, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 0.5916463136672974, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 0.6149423718452454, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 0.48369669914245605, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 0.6047574877738953, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 0.5092479586601257, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 0.5999670624732971, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 0.8980127573013306, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 1.4252448081970215, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 1.509937047958374, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 1.0066585540771484, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 0.6413647532463074, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 26.08852195739746, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 13.382951736450195, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 20.088768005371094, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 23.0632381439209, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 16.07433319091797, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 8.00507640838623, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 9.538354873657227, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 6.286602973937988, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 10.092820167541504, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 7.193963527679443, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 7.320116996765137, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 4.8728532791137695, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 3.596583366394043, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 3.166161298751831, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 1.5600818395614624, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 0.8726214170455933, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 0.1154392883181572, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.24299409985542297, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 0.3624322712421417, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 0.29509487748146057, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 0.32953736186027527, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 0.2908833622932434, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 0.2488437294960022, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 0.27847856283187866, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 0.27143892645835876, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 0.28804272413253784, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 0.31197959184646606, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 0.3203586935997009, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 0.30905747413635254, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 0.46828722953796387, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.24205778539180756, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 0.2559327781200409, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.2638678550720215, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.21109595894813538, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.24751724302768707, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.2728094160556793, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.3001374304294586, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.33903488516807556, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.3530929982662201, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.36753255128860474, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.3373180329799652, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.2970578670501709, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.3076324760913849, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.2766900658607483, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.20973259210586548, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.18185566365718842, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.18329747021198273, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.2437991499900818, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.038040731102228165, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.0707998052239418, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.0787411704659462, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.08089710026979446, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.08591937273740768, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.09852176159620285, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.09690654277801514, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.11181341856718063, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.12042108923196793, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.09799323976039886, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.10901063680648804, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.09307146072387695, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.0880950540304184, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.08886399120092392, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.09955056011676788, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.08929339051246643, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": 2.5501928329467773, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 9.449499130249023, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": 7.9920830726623535, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": 50.69462585449219, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": 19.083511352539062, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 7.21597146987915, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 11.27744197845459, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": 4.579711437225342, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": 10.940719604492188, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": 553.4417724609375, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": 20.59434700012207, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": 26.636865615844727, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 8.614749908447266, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": 17.722007751464844, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 1.48500657081604, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": 2.5776851177215576, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-3B-Instruct.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-3B-Instruct.json new file mode 100644 index 000000000..7764eecd9 --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-3B-Instruct.json @@ -0,0 +1,1022 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 2.306217670440674, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 2.2327167987823486, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 1.4501516819000244, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 1.363667607307434, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 1.4520279169082642, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 1.4664665460586548, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 1.4122329950332642, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 1.0504299402236938, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 0.9837537407875061, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 0.8659006357192993, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 0.7936406135559082, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 0.9000886678695679, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 1.1559213399887085, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 1.3054672479629517, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 1.196791410446167, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 1.3163655996322632, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 1.3388997316360474, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 1.592497706413269, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 1.5399079322814941, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 1.5683293342590332, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 1.4739630222320557, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 1.2608393430709839, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 1.2087301015853882, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 1.1851829290390015, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 1.0537594556808472, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 1.1649317741394043, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 1.2376821041107178, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 1.147771954536438, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.9385462999343872, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 0.8528683185577393, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 0.761657178401947, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 0.6598325371742249, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 0.44578588008880615, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 0.4053060710430145, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 0.3588462769985199, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 0.35667839646339417, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 0.3106202781200409, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 0.2821919322013855, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 0.29143741726875305, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 0.29830989241600037, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 0.2862427532672882, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 0.2797018587589264, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 0.2679217755794525, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 0.2782425880432129, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 0.3503592610359192, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 0.3968559205532074, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 0.4318574070930481, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 0.4693693220615387, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 0.5051979422569275, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 0.5675955414772034, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 0.5861843824386597, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 0.4759417772293091, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 0.38529056310653687, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 0.3180919587612152, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 0.2695689797401428, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 0.21765239536762238, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 1.4919718503952026, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 1.7983858585357666, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 2.1709094047546387, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 2.751326560974121, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 3.063521385192871, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 2.4026951789855957, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 2.3890223503112793, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 2.3861353397369385, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 2.0745043754577637, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 1.8550645112991333, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 1.6184496879577637, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 1.9287559986114502, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 1.7427546977996826, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 1.9872609376907349, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 2.0224087238311768, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 1.7851638793945312, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 1.7160604000091553, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 1.6870195865631104, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 1.6585396528244019, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 1.5509096384048462, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 1.4310423135757446, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 1.5009464025497437, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 1.4866929054260254, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 1.332513689994812, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 1.073512077331543, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 0.7472100257873535, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 0.4880162179470062, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 0.2527681589126587, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 0.08262510597705841, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.1441459059715271, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 0.21418076753616333, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 0.22496014833450317, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 0.23101305961608887, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 0.23644132912158966, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 0.23666173219680786, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 0.19791515171527863, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 0.22062039375305176, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 0.21218444406986237, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 0.24218571186065674, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 0.21870514750480652, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 0.22160987555980682, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 0.22726823389530182, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.20256873965263367, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 0.24100735783576965, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 0.23794010281562805, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 0.2913324534893036, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 0.28093472123146057, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 0.31062793731689453, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 0.2942160367965698, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 0.28014805912971497, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 0.3512437045574188, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 0.2837671637535095, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 0.2960015535354614, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 0.5086414813995361, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 0.24054698646068573, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 0.247616246342659, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.18390265107154846, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.14759540557861328, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.15726515650749207, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.16903570294380188, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.17953157424926758, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.2351229190826416, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.22804339230060577, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.24786025285720825, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.21847976744174957, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.2092437595129013, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.23278094828128815, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.20468176901340485, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.2353818416595459, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.2702614367008209, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.19177420437335968, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.18293911218643188, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.20286045968532562, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.20763878524303436, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.190629780292511, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.22044304013252258, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.21491236984729767, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.23289704322814941, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.21457163989543915, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.1949365884065628, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.1606779545545578, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.13892440497875214, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.1407029926776886, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.16027599573135376, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.0534212663769722, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.06873775273561478, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.07522258907556534, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.06616844981908798, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.06809444725513458, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.0758095383644104, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.07800278812646866, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.07535763084888458, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.09488166123628616, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.09709945321083069, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.09381720423698425, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.08205580711364746, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.10723169893026352, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.10166660696268082, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.08822792023420334, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.0814041867852211, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.07586681097745895, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.07040166854858398, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.0728282704949379, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.06912193447351456, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.06646180897951126, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.06960278004407883, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.06566876918077469, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.07412787526845932, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.07131384313106537, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.07768437266349792, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.0809575766324997, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.06796683371067047, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": 1.4029983282089233, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 3.123720169067383, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": 2.4177253246307373, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": 5.588768005371094, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": 4.395562648773193, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 3.2982685565948486, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 3.2798449993133545, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": 2.109200954437256, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": 3.229325532913208, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": 1.7349927425384521, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": 1.5926740169525146, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": 1.9097802639007568, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 2.5654332637786865, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": 3.536489963531494, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 8.366667747497559, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": 7.348303318023682, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": 2.815748691558838, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": 4.048776149749756, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": 4.426101207733154, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": 7.098501682281494, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": 3.700288772583008, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": 2.1859049797058105, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": 3.6953284740448, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": 11.148802757263184, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": 2.4171905517578125, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": 4.404144287109375, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": 2.340604782104492, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": 3.284160614013672, + "type": "self_attn.v_proj" + } +} diff --git a/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-3B.json b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-3B.json new file mode 100644 index 000000000..131a5b79b --- /dev/null +++ b/src/axolotl/integrations/spectrum/model_snr_results/snr_results_meta-llama-Llama-3.2-3B.json @@ -0,0 +1,1022 @@ +{ + "model.layers.0.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.1.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.2.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.3.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.4.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.5.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.6.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.7.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.8.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.9.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.10.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.11.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.12.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.13.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.14.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.15.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.16.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.17.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.18.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.19.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.20.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.21.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.22.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.23.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.24.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.25.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.26.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "model.layers.27.input_layernorm": { + "snr": Infinity, + "type": "input_layernorm" + }, + "lm_head": { + "snr": Infinity, + "type": "lm_head" + }, + "model.layers.0.mlp.down_proj": { + "snr": 2.364603281021118, + "type": "mlp.down_proj" + }, + "model.layers.1.mlp.down_proj": { + "snr": 2.229910373687744, + "type": "mlp.down_proj" + }, + "model.layers.2.mlp.down_proj": { + "snr": 1.4312117099761963, + "type": "mlp.down_proj" + }, + "model.layers.3.mlp.down_proj": { + "snr": 1.3216407299041748, + "type": "mlp.down_proj" + }, + "model.layers.4.mlp.down_proj": { + "snr": 1.4183496236801147, + "type": "mlp.down_proj" + }, + "model.layers.5.mlp.down_proj": { + "snr": 1.4453660249710083, + "type": "mlp.down_proj" + }, + "model.layers.6.mlp.down_proj": { + "snr": 1.4030662775039673, + "type": "mlp.down_proj" + }, + "model.layers.7.mlp.down_proj": { + "snr": 1.042332649230957, + "type": "mlp.down_proj" + }, + "model.layers.8.mlp.down_proj": { + "snr": 0.9530982375144958, + "type": "mlp.down_proj" + }, + "model.layers.9.mlp.down_proj": { + "snr": 0.849862277507782, + "type": "mlp.down_proj" + }, + "model.layers.10.mlp.down_proj": { + "snr": 0.7704945206642151, + "type": "mlp.down_proj" + }, + "model.layers.11.mlp.down_proj": { + "snr": 0.8871145844459534, + "type": "mlp.down_proj" + }, + "model.layers.12.mlp.down_proj": { + "snr": 1.1408143043518066, + "type": "mlp.down_proj" + }, + "model.layers.13.mlp.down_proj": { + "snr": 1.2769343852996826, + "type": "mlp.down_proj" + }, + "model.layers.14.mlp.down_proj": { + "snr": 1.1703068017959595, + "type": "mlp.down_proj" + }, + "model.layers.15.mlp.down_proj": { + "snr": 1.2794467210769653, + "type": "mlp.down_proj" + }, + "model.layers.16.mlp.down_proj": { + "snr": 1.3154453039169312, + "type": "mlp.down_proj" + }, + "model.layers.17.mlp.down_proj": { + "snr": 1.5596749782562256, + "type": "mlp.down_proj" + }, + "model.layers.18.mlp.down_proj": { + "snr": 1.4949405193328857, + "type": "mlp.down_proj" + }, + "model.layers.19.mlp.down_proj": { + "snr": 1.5329173803329468, + "type": "mlp.down_proj" + }, + "model.layers.20.mlp.down_proj": { + "snr": 1.4396660327911377, + "type": "mlp.down_proj" + }, + "model.layers.21.mlp.down_proj": { + "snr": 1.217085838317871, + "type": "mlp.down_proj" + }, + "model.layers.22.mlp.down_proj": { + "snr": 1.150472640991211, + "type": "mlp.down_proj" + }, + "model.layers.23.mlp.down_proj": { + "snr": 1.1166225671768188, + "type": "mlp.down_proj" + }, + "model.layers.24.mlp.down_proj": { + "snr": 0.9966591000556946, + "type": "mlp.down_proj" + }, + "model.layers.25.mlp.down_proj": { + "snr": 1.0938347578048706, + "type": "mlp.down_proj" + }, + "model.layers.26.mlp.down_proj": { + "snr": 1.1505423784255981, + "type": "mlp.down_proj" + }, + "model.layers.27.mlp.down_proj": { + "snr": 1.1156749725341797, + "type": "mlp.down_proj" + }, + "model.layers.0.mlp.gate_proj": { + "snr": 0.9329171776771545, + "type": "mlp.gate_proj" + }, + "model.layers.1.mlp.gate_proj": { + "snr": 0.8513413667678833, + "type": "mlp.gate_proj" + }, + "model.layers.2.mlp.gate_proj": { + "snr": 0.7584061026573181, + "type": "mlp.gate_proj" + }, + "model.layers.3.mlp.gate_proj": { + "snr": 0.65835040807724, + "type": "mlp.gate_proj" + }, + "model.layers.4.mlp.gate_proj": { + "snr": 0.436420738697052, + "type": "mlp.gate_proj" + }, + "model.layers.5.mlp.gate_proj": { + "snr": 0.39712461829185486, + "type": "mlp.gate_proj" + }, + "model.layers.6.mlp.gate_proj": { + "snr": 0.3530206084251404, + "type": "mlp.gate_proj" + }, + "model.layers.7.mlp.gate_proj": { + "snr": 0.34982794523239136, + "type": "mlp.gate_proj" + }, + "model.layers.8.mlp.gate_proj": { + "snr": 0.30338960886001587, + "type": "mlp.gate_proj" + }, + "model.layers.9.mlp.gate_proj": { + "snr": 0.27569833397865295, + "type": "mlp.gate_proj" + }, + "model.layers.10.mlp.gate_proj": { + "snr": 0.28934162855148315, + "type": "mlp.gate_proj" + }, + "model.layers.11.mlp.gate_proj": { + "snr": 0.2929173707962036, + "type": "mlp.gate_proj" + }, + "model.layers.12.mlp.gate_proj": { + "snr": 0.28263387084007263, + "type": "mlp.gate_proj" + }, + "model.layers.13.mlp.gate_proj": { + "snr": 0.27778616547584534, + "type": "mlp.gate_proj" + }, + "model.layers.14.mlp.gate_proj": { + "snr": 0.26527827978134155, + "type": "mlp.gate_proj" + }, + "model.layers.15.mlp.gate_proj": { + "snr": 0.27635642886161804, + "type": "mlp.gate_proj" + }, + "model.layers.16.mlp.gate_proj": { + "snr": 0.35072311758995056, + "type": "mlp.gate_proj" + }, + "model.layers.17.mlp.gate_proj": { + "snr": 0.4002636671066284, + "type": "mlp.gate_proj" + }, + "model.layers.18.mlp.gate_proj": { + "snr": 0.4319891333580017, + "type": "mlp.gate_proj" + }, + "model.layers.19.mlp.gate_proj": { + "snr": 0.47527065873146057, + "type": "mlp.gate_proj" + }, + "model.layers.20.mlp.gate_proj": { + "snr": 0.5112077593803406, + "type": "mlp.gate_proj" + }, + "model.layers.21.mlp.gate_proj": { + "snr": 0.5749644637107849, + "type": "mlp.gate_proj" + }, + "model.layers.22.mlp.gate_proj": { + "snr": 0.5967603921890259, + "type": "mlp.gate_proj" + }, + "model.layers.23.mlp.gate_proj": { + "snr": 0.48045310378074646, + "type": "mlp.gate_proj" + }, + "model.layers.24.mlp.gate_proj": { + "snr": 0.3838970363140106, + "type": "mlp.gate_proj" + }, + "model.layers.25.mlp.gate_proj": { + "snr": 0.3108249604701996, + "type": "mlp.gate_proj" + }, + "model.layers.26.mlp.gate_proj": { + "snr": 0.26704445481300354, + "type": "mlp.gate_proj" + }, + "model.layers.27.mlp.gate_proj": { + "snr": 0.20953254401683807, + "type": "mlp.gate_proj" + }, + "model.layers.0.mlp.up_proj": { + "snr": 1.5084924697875977, + "type": "mlp.up_proj" + }, + "model.layers.1.mlp.up_proj": { + "snr": 1.7789595127105713, + "type": "mlp.up_proj" + }, + "model.layers.2.mlp.up_proj": { + "snr": 2.1431775093078613, + "type": "mlp.up_proj" + }, + "model.layers.3.mlp.up_proj": { + "snr": 2.762744903564453, + "type": "mlp.up_proj" + }, + "model.layers.4.mlp.up_proj": { + "snr": 3.0324745178222656, + "type": "mlp.up_proj" + }, + "model.layers.5.mlp.up_proj": { + "snr": 2.3884809017181396, + "type": "mlp.up_proj" + }, + "model.layers.6.mlp.up_proj": { + "snr": 2.388005256652832, + "type": "mlp.up_proj" + }, + "model.layers.7.mlp.up_proj": { + "snr": 2.339340925216675, + "type": "mlp.up_proj" + }, + "model.layers.8.mlp.up_proj": { + "snr": 2.0497021675109863, + "type": "mlp.up_proj" + }, + "model.layers.9.mlp.up_proj": { + "snr": 1.822119116783142, + "type": "mlp.up_proj" + }, + "model.layers.10.mlp.up_proj": { + "snr": 1.600373387336731, + "type": "mlp.up_proj" + }, + "model.layers.11.mlp.up_proj": { + "snr": 1.9298171997070312, + "type": "mlp.up_proj" + }, + "model.layers.12.mlp.up_proj": { + "snr": 1.728783369064331, + "type": "mlp.up_proj" + }, + "model.layers.13.mlp.up_proj": { + "snr": 1.965298056602478, + "type": "mlp.up_proj" + }, + "model.layers.14.mlp.up_proj": { + "snr": 2.023681640625, + "type": "mlp.up_proj" + }, + "model.layers.15.mlp.up_proj": { + "snr": 1.7721818685531616, + "type": "mlp.up_proj" + }, + "model.layers.16.mlp.up_proj": { + "snr": 1.7068361043930054, + "type": "mlp.up_proj" + }, + "model.layers.17.mlp.up_proj": { + "snr": 1.6673219203948975, + "type": "mlp.up_proj" + }, + "model.layers.18.mlp.up_proj": { + "snr": 1.6240718364715576, + "type": "mlp.up_proj" + }, + "model.layers.19.mlp.up_proj": { + "snr": 1.5169662237167358, + "type": "mlp.up_proj" + }, + "model.layers.20.mlp.up_proj": { + "snr": 1.4018198251724243, + "type": "mlp.up_proj" + }, + "model.layers.21.mlp.up_proj": { + "snr": 1.4556466341018677, + "type": "mlp.up_proj" + }, + "model.layers.22.mlp.up_proj": { + "snr": 1.4304454326629639, + "type": "mlp.up_proj" + }, + "model.layers.23.mlp.up_proj": { + "snr": 1.2785290479660034, + "type": "mlp.up_proj" + }, + "model.layers.24.mlp.up_proj": { + "snr": 1.023495078086853, + "type": "mlp.up_proj" + }, + "model.layers.25.mlp.up_proj": { + "snr": 0.6992124915122986, + "type": "mlp.up_proj" + }, + "model.layers.26.mlp.up_proj": { + "snr": 0.4549211859703064, + "type": "mlp.up_proj" + }, + "model.layers.27.mlp.up_proj": { + "snr": 0.23889905214309692, + "type": "mlp.up_proj" + }, + "model.embed_tokens": { + "snr": Infinity, + "type": "model.embed_tokens" + }, + "model.norm": { + "snr": Infinity, + "type": "model.norm" + }, + "model.layers.0.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.1.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.2.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.3.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.4.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.5.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.6.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.7.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.8.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.9.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.10.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.11.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.12.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.13.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.14.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.15.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.16.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.17.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.18.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.19.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.20.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.21.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.22.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.23.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.24.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.25.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.26.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.27.post_attention_layernorm": { + "snr": Infinity, + "type": "post_attention_layernorm" + }, + "model.layers.0.self_attn.k_proj": { + "snr": 0.08150045573711395, + "type": "self_attn.k_proj" + }, + "model.layers.1.self_attn.k_proj": { + "snr": 0.1428358554840088, + "type": "self_attn.k_proj" + }, + "model.layers.2.self_attn.k_proj": { + "snr": 0.2096949815750122, + "type": "self_attn.k_proj" + }, + "model.layers.3.self_attn.k_proj": { + "snr": 0.22633400559425354, + "type": "self_attn.k_proj" + }, + "model.layers.4.self_attn.k_proj": { + "snr": 0.2293967455625534, + "type": "self_attn.k_proj" + }, + "model.layers.5.self_attn.k_proj": { + "snr": 0.23336802423000336, + "type": "self_attn.k_proj" + }, + "model.layers.6.self_attn.k_proj": { + "snr": 0.23429904878139496, + "type": "self_attn.k_proj" + }, + "model.layers.7.self_attn.k_proj": { + "snr": 0.19610290229320526, + "type": "self_attn.k_proj" + }, + "model.layers.8.self_attn.k_proj": { + "snr": 0.2163258045911789, + "type": "self_attn.k_proj" + }, + "model.layers.9.self_attn.k_proj": { + "snr": 0.21039333939552307, + "type": "self_attn.k_proj" + }, + "model.layers.10.self_attn.k_proj": { + "snr": 0.23533931374549866, + "type": "self_attn.k_proj" + }, + "model.layers.11.self_attn.k_proj": { + "snr": 0.21457058191299438, + "type": "self_attn.k_proj" + }, + "model.layers.12.self_attn.k_proj": { + "snr": 0.21686571836471558, + "type": "self_attn.k_proj" + }, + "model.layers.13.self_attn.k_proj": { + "snr": 0.22398065030574799, + "type": "self_attn.k_proj" + }, + "model.layers.14.self_attn.k_proj": { + "snr": 0.20160657167434692, + "type": "self_attn.k_proj" + }, + "model.layers.15.self_attn.k_proj": { + "snr": 0.23705022037029266, + "type": "self_attn.k_proj" + }, + "model.layers.16.self_attn.k_proj": { + "snr": 0.23254962265491486, + "type": "self_attn.k_proj" + }, + "model.layers.17.self_attn.k_proj": { + "snr": 0.2892642617225647, + "type": "self_attn.k_proj" + }, + "model.layers.18.self_attn.k_proj": { + "snr": 0.27587130665779114, + "type": "self_attn.k_proj" + }, + "model.layers.19.self_attn.k_proj": { + "snr": 0.30891212821006775, + "type": "self_attn.k_proj" + }, + "model.layers.20.self_attn.k_proj": { + "snr": 0.28997519612312317, + "type": "self_attn.k_proj" + }, + "model.layers.21.self_attn.k_proj": { + "snr": 0.27534863352775574, + "type": "self_attn.k_proj" + }, + "model.layers.22.self_attn.k_proj": { + "snr": 0.35139667987823486, + "type": "self_attn.k_proj" + }, + "model.layers.23.self_attn.k_proj": { + "snr": 0.2773109972476959, + "type": "self_attn.k_proj" + }, + "model.layers.24.self_attn.k_proj": { + "snr": 0.2853511571884155, + "type": "self_attn.k_proj" + }, + "model.layers.25.self_attn.k_proj": { + "snr": 0.5030262470245361, + "type": "self_attn.k_proj" + }, + "model.layers.26.self_attn.k_proj": { + "snr": 0.2317112237215042, + "type": "self_attn.k_proj" + }, + "model.layers.27.self_attn.k_proj": { + "snr": 0.24419328570365906, + "type": "self_attn.k_proj" + }, + "model.layers.0.self_attn.o_proj": { + "snr": 0.17767645418643951, + "type": "self_attn.o_proj" + }, + "model.layers.1.self_attn.o_proj": { + "snr": 0.14102177321910858, + "type": "self_attn.o_proj" + }, + "model.layers.2.self_attn.o_proj": { + "snr": 0.1523692011833191, + "type": "self_attn.o_proj" + }, + "model.layers.3.self_attn.o_proj": { + "snr": 0.16522075235843658, + "type": "self_attn.o_proj" + }, + "model.layers.4.self_attn.o_proj": { + "snr": 0.17483487725257874, + "type": "self_attn.o_proj" + }, + "model.layers.5.self_attn.o_proj": { + "snr": 0.227921262383461, + "type": "self_attn.o_proj" + }, + "model.layers.6.self_attn.o_proj": { + "snr": 0.2196175903081894, + "type": "self_attn.o_proj" + }, + "model.layers.7.self_attn.o_proj": { + "snr": 0.24270132184028625, + "type": "self_attn.o_proj" + }, + "model.layers.8.self_attn.o_proj": { + "snr": 0.2118290364742279, + "type": "self_attn.o_proj" + }, + "model.layers.9.self_attn.o_proj": { + "snr": 0.20525991916656494, + "type": "self_attn.o_proj" + }, + "model.layers.10.self_attn.o_proj": { + "snr": 0.22847208380699158, + "type": "self_attn.o_proj" + }, + "model.layers.11.self_attn.o_proj": { + "snr": 0.19665324687957764, + "type": "self_attn.o_proj" + }, + "model.layers.12.self_attn.o_proj": { + "snr": 0.23233532905578613, + "type": "self_attn.o_proj" + }, + "model.layers.13.self_attn.o_proj": { + "snr": 0.2624332308769226, + "type": "self_attn.o_proj" + }, + "model.layers.14.self_attn.o_proj": { + "snr": 0.1868327558040619, + "type": "self_attn.o_proj" + }, + "model.layers.15.self_attn.o_proj": { + "snr": 0.17706255614757538, + "type": "self_attn.o_proj" + }, + "model.layers.16.self_attn.o_proj": { + "snr": 0.19422705471515656, + "type": "self_attn.o_proj" + }, + "model.layers.17.self_attn.o_proj": { + "snr": 0.2000615894794464, + "type": "self_attn.o_proj" + }, + "model.layers.18.self_attn.o_proj": { + "snr": 0.1874573826789856, + "type": "self_attn.o_proj" + }, + "model.layers.19.self_attn.o_proj": { + "snr": 0.21297843754291534, + "type": "self_attn.o_proj" + }, + "model.layers.20.self_attn.o_proj": { + "snr": 0.2100859135389328, + "type": "self_attn.o_proj" + }, + "model.layers.21.self_attn.o_proj": { + "snr": 0.22561520338058472, + "type": "self_attn.o_proj" + }, + "model.layers.22.self_attn.o_proj": { + "snr": 0.20994484424591064, + "type": "self_attn.o_proj" + }, + "model.layers.23.self_attn.o_proj": { + "snr": 0.18978221714496613, + "type": "self_attn.o_proj" + }, + "model.layers.24.self_attn.o_proj": { + "snr": 0.1571759581565857, + "type": "self_attn.o_proj" + }, + "model.layers.25.self_attn.o_proj": { + "snr": 0.1349896937608719, + "type": "self_attn.o_proj" + }, + "model.layers.26.self_attn.o_proj": { + "snr": 0.1368866115808487, + "type": "self_attn.o_proj" + }, + "model.layers.27.self_attn.o_proj": { + "snr": 0.1571887582540512, + "type": "self_attn.o_proj" + }, + "model.layers.0.self_attn.q_proj": { + "snr": 0.05295897275209427, + "type": "self_attn.q_proj" + }, + "model.layers.1.self_attn.q_proj": { + "snr": 0.06835605204105377, + "type": "self_attn.q_proj" + }, + "model.layers.2.self_attn.q_proj": { + "snr": 0.0746372863650322, + "type": "self_attn.q_proj" + }, + "model.layers.3.self_attn.q_proj": { + "snr": 0.06615085154771805, + "type": "self_attn.q_proj" + }, + "model.layers.4.self_attn.q_proj": { + "snr": 0.06788161396980286, + "type": "self_attn.q_proj" + }, + "model.layers.5.self_attn.q_proj": { + "snr": 0.07514483481645584, + "type": "self_attn.q_proj" + }, + "model.layers.6.self_attn.q_proj": { + "snr": 0.07777862250804901, + "type": "self_attn.q_proj" + }, + "model.layers.7.self_attn.q_proj": { + "snr": 0.07534090429544449, + "type": "self_attn.q_proj" + }, + "model.layers.8.self_attn.q_proj": { + "snr": 0.09494179487228394, + "type": "self_attn.q_proj" + }, + "model.layers.9.self_attn.q_proj": { + "snr": 0.09699037671089172, + "type": "self_attn.q_proj" + }, + "model.layers.10.self_attn.q_proj": { + "snr": 0.09426294267177582, + "type": "self_attn.q_proj" + }, + "model.layers.11.self_attn.q_proj": { + "snr": 0.08260341733694077, + "type": "self_attn.q_proj" + }, + "model.layers.12.self_attn.q_proj": { + "snr": 0.10650420933961868, + "type": "self_attn.q_proj" + }, + "model.layers.13.self_attn.q_proj": { + "snr": 0.10250870138406754, + "type": "self_attn.q_proj" + }, + "model.layers.14.self_attn.q_proj": { + "snr": 0.08775162696838379, + "type": "self_attn.q_proj" + }, + "model.layers.15.self_attn.q_proj": { + "snr": 0.08071447163820267, + "type": "self_attn.q_proj" + }, + "model.layers.16.self_attn.q_proj": { + "snr": 0.07530857622623444, + "type": "self_attn.q_proj" + }, + "model.layers.17.self_attn.q_proj": { + "snr": 0.06964966654777527, + "type": "self_attn.q_proj" + }, + "model.layers.18.self_attn.q_proj": { + "snr": 0.07150755077600479, + "type": "self_attn.q_proj" + }, + "model.layers.19.self_attn.q_proj": { + "snr": 0.0676807165145874, + "type": "self_attn.q_proj" + }, + "model.layers.20.self_attn.q_proj": { + "snr": 0.06511317938566208, + "type": "self_attn.q_proj" + }, + "model.layers.21.self_attn.q_proj": { + "snr": 0.06773187220096588, + "type": "self_attn.q_proj" + }, + "model.layers.22.self_attn.q_proj": { + "snr": 0.06400436162948608, + "type": "self_attn.q_proj" + }, + "model.layers.23.self_attn.q_proj": { + "snr": 0.0726117342710495, + "type": "self_attn.q_proj" + }, + "model.layers.24.self_attn.q_proj": { + "snr": 0.06882446259260178, + "type": "self_attn.q_proj" + }, + "model.layers.25.self_attn.q_proj": { + "snr": 0.07506493479013443, + "type": "self_attn.q_proj" + }, + "model.layers.26.self_attn.q_proj": { + "snr": 0.07797915488481522, + "type": "self_attn.q_proj" + }, + "model.layers.27.self_attn.q_proj": { + "snr": 0.06680692732334137, + "type": "self_attn.q_proj" + }, + "model.layers.0.self_attn.v_proj": { + "snr": 1.326789379119873, + "type": "self_attn.v_proj" + }, + "model.layers.1.self_attn.v_proj": { + "snr": 3.043806791305542, + "type": "self_attn.v_proj" + }, + "model.layers.2.self_attn.v_proj": { + "snr": 2.295107841491699, + "type": "self_attn.v_proj" + }, + "model.layers.3.self_attn.v_proj": { + "snr": 5.2584614753723145, + "type": "self_attn.v_proj" + }, + "model.layers.4.self_attn.v_proj": { + "snr": 4.038785934448242, + "type": "self_attn.v_proj" + }, + "model.layers.5.self_attn.v_proj": { + "snr": 3.0907773971557617, + "type": "self_attn.v_proj" + }, + "model.layers.6.self_attn.v_proj": { + "snr": 3.114994525909424, + "type": "self_attn.v_proj" + }, + "model.layers.7.self_attn.v_proj": { + "snr": 1.9747973680496216, + "type": "self_attn.v_proj" + }, + "model.layers.8.self_attn.v_proj": { + "snr": 3.0469374656677246, + "type": "self_attn.v_proj" + }, + "model.layers.9.self_attn.v_proj": { + "snr": 1.602966547012329, + "type": "self_attn.v_proj" + }, + "model.layers.10.self_attn.v_proj": { + "snr": 1.489019513130188, + "type": "self_attn.v_proj" + }, + "model.layers.11.self_attn.v_proj": { + "snr": 1.7490826845169067, + "type": "self_attn.v_proj" + }, + "model.layers.12.self_attn.v_proj": { + "snr": 2.451310396194458, + "type": "self_attn.v_proj" + }, + "model.layers.13.self_attn.v_proj": { + "snr": 3.250821590423584, + "type": "self_attn.v_proj" + }, + "model.layers.14.self_attn.v_proj": { + "snr": 7.944663047790527, + "type": "self_attn.v_proj" + }, + "model.layers.15.self_attn.v_proj": { + "snr": 7.013208389282227, + "type": "self_attn.v_proj" + }, + "model.layers.16.self_attn.v_proj": { + "snr": 2.68644118309021, + "type": "self_attn.v_proj" + }, + "model.layers.17.self_attn.v_proj": { + "snr": 3.9063122272491455, + "type": "self_attn.v_proj" + }, + "model.layers.18.self_attn.v_proj": { + "snr": 4.1816816329956055, + "type": "self_attn.v_proj" + }, + "model.layers.19.self_attn.v_proj": { + "snr": 6.794488906860352, + "type": "self_attn.v_proj" + }, + "model.layers.20.self_attn.v_proj": { + "snr": 3.401334285736084, + "type": "self_attn.v_proj" + }, + "model.layers.21.self_attn.v_proj": { + "snr": 2.051994562149048, + "type": "self_attn.v_proj" + }, + "model.layers.22.self_attn.v_proj": { + "snr": 3.614379405975342, + "type": "self_attn.v_proj" + }, + "model.layers.23.self_attn.v_proj": { + "snr": 11.180968284606934, + "type": "self_attn.v_proj" + }, + "model.layers.24.self_attn.v_proj": { + "snr": 2.3629775047302246, + "type": "self_attn.v_proj" + }, + "model.layers.25.self_attn.v_proj": { + "snr": 4.137593746185303, + "type": "self_attn.v_proj" + }, + "model.layers.26.self_attn.v_proj": { + "snr": 2.3465518951416016, + "type": "self_attn.v_proj" + }, + "model.layers.27.self_attn.v_proj": { + "snr": 3.10064697265625, + "type": "self_attn.v_proj" + } +}