tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Phi2 multipack (#1173)

Browse Source 
* phi2 multipack

* update validation and examples for phi

* more updates to phi examples

* make sure to use the correct collator for phi multipack

* phi needs attention mask now for multipack

* if the special token already exists in the tokenizer, don't require in lora modules to save

* fix qlora yml for phi, fix phi test validation

* test qlora too

* make sure flash attention is enabled for the test

* don't use remote code for phi anymore

* reduce sequence len for sample packing phi
This commit is contained in:
Wing Lian
2024-01-23 12:54:36 -05:00
committed by GitHub
parent b715cd549a
commit 814aee6603
18 changed files with 201 additions and 2269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/test_validation.py
View File

@@ -742,11 +742,11 @@ class ValidationCheckModelConfig(BaseValidation):
check_model_config(cfg, model_config)
def test_phi2_add_tokens_adapter(self):
def test_phi_add_tokens_adapter(self):
cfg = DictDefault(
{"adapter": "qlora", "load_in_4bit": True, "tokens": ["<|imstart|>"]}
)
model_config = DictDefault({"model_type": "phi-msft"})
model_config = DictDefault({"model_type": "phi"})
with pytest.raises(
ValueError,
@@ -759,7 +759,7 @@ class ValidationCheckModelConfig(BaseValidation):
"adapter": "qlora",
"load_in_4bit": True,
"tokens": ["<|imstart|>"],
"lora_modules_to_save": ["embed_tokens", "lm_head"],
"lora_modules_to_save": ["embd.wte", "lm_head.linear"],
}
)
@@ -774,7 +774,7 @@ class ValidationCheckModelConfig(BaseValidation):
"adapter": "qlora",
"load_in_4bit": True,
"tokens": ["<|imstart|>"],
"lora_modules_to_save": ["embd.wte", "lm_head.linear"],
"lora_modules_to_save": ["embed_tokens", "lm_head"],
}
)