tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix keyerror on load_in_8bit/load_in_4bit access in _set_quantization_config (#3023)

Browse Source 
* set load_in_8bit/load_in_4bit in _set_quantization_config to prevent keyerror

* use dict.get instead
This commit is contained in:
Wing Lian
2025-08-06 14:28:52 -04:00
committed by GitHub
parent ba3dba3e4f
commit e442ff22aa
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/axolotl/loaders/model.py
View File

@@ -612,7 +612,9 @@ class ModelLoader:
self.model_kwargs["quantization_config"] = BitsAndBytesConfig(
**self.model_config.quantization_config
)
elif self.cfg.adapter == "qlora" and self.model_kwargs["load_in_4bit"]:
elif self.cfg.adapter == "qlora" and self.model_kwargs.get(
"load_in_4bit", False
):
bnb_config = {
"load_in_4bit": True,
"llm_int8_threshold": 6.0,
@@ -638,7 +640,9 @@ class ModelLoader:
self.model_kwargs["quantization_config"] = BitsAndBytesConfig(
**bnb_config,
)
elif self.cfg.adapter == "lora" and self.model_kwargs["load_in_8bit"]:
elif self.cfg.adapter == "lora" and self.model_kwargs.get(
"load_in_8bit", False
):
bnb_config = {
"load_in_8bit": True,
}