From 985ee95f2d1883ea092138344a1d20cc8270af88 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Mon, 5 May 2025 08:27:34 -0400 Subject: [PATCH] use uint8 dtype for qlora --- src/axolotl/utils/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index 57669936e..9d842a1d0 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -912,7 +912,7 @@ class ModelLoader: "bnb_4bit_compute_dtype": self.cfg.torch_dtype, "bnb_4bit_use_double_quant": True, "bnb_4bit_quant_type": "nf4", - "bnb_4bit_quant_storage": torch.bfloat16, + "bnb_4bit_quant_storage": torch.uint8, } if self.cfg.model_config_type in ["jamba", "qwen2_moe"] and not ( self.cfg.deepspeed or self.cfg.fsdp