tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

use uint8 dtype for qlora

Browse Source
This commit is contained in:
Wing Lian
2025-05-05 08:27:34 -04:00
parent 72ece3dadf
commit 985ee95f2d
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/utils/models.py
View File

@@ -912,7 +912,7 @@ class ModelLoader:
"bnb_4bit_compute_dtype": self.cfg.torch_dtype, "bnb_4bit_compute_dtype": self.cfg.torch_dtype,
"bnb_4bit_use_double_quant": True, "bnb_4bit_use_double_quant": True,
"bnb_4bit_quant_type": "nf4", "bnb_4bit_quant_type": "nf4",
"bnb_4bit_quant_storage": torch.bfloat16, "bnb_4bit_quant_storage": torch.uint8,
} }
if self.cfg.model_config_type in ["jamba", "qwen2_moe"] and not ( if self.cfg.model_config_type in ["jamba", "qwen2_moe"] and not (
self.cfg.deepspeed or self.cfg.fsdp self.cfg.deepspeed or self.cfg.fsdp