tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fsdp embeddings should be float32 per comment

Browse Source
This commit is contained in:
Wing Lian
2025-05-03 01:56:09 -04:00
parent ed922796b7
commit 37c27aedc1
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/utils/models.py
View File

@@ -1309,7 +1309,7 @@ class ModelLoader:
# make sure these are fp32 per Ramesh et al. (2021) # make sure these are fp32 per Ramesh et al. (2021)
embedding_modules = get_linear_embedding_layers(self.cfg.model_config_type) embedding_modules = get_linear_embedding_layers(self.cfg.model_config_type)
if not self.cfg.fsdp: if self.cfg.fsdp:
# FSDP doesn't like mixed Float and BFloat16 # FSDP doesn't like mixed Float and BFloat16
self.convert_embedding_modules_dtype( self.convert_embedding_modules_dtype(
embedding_modules, embedding_modules,