tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

various tests fixes for flakey tests (#2110)

Browse Source 
* add mhenrichsen/alpaca_2k_test with revision dataset download fixture for flaky tests

* log slowest tests

* pin pynvml==11.5.3

* fix load local hub path

* optimize for speed w smaller models and val_set_size

* replace pynvml

* make the resume from checkpoint e2e faster

* make tests smaller
This commit is contained in:
Wing Lian
2024-12-02 17:28:58 -05:00
committed by GitHub
parent b620ed94d0
commit ce5bcff750
13 changed files with 78 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/e2e/patched/test_fa_xentropy.py
View File

@@ -51,11 +51,11 @@ class TestFAXentropyLlama:
"flash_attn_cross_entropy": True,
"load_in_8bit": True,
"adapter": "lora",
"lora_r": 32,
"lora_alpha": 64,
"lora_r": 8,
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_target_linear": True,
"val_set_size": 0.2,
"val_set_size": 0.05,
"special_tokens": {
"pad_token": "<|endoftext|>",
},

29
tests/e2e/patched/test_resume.py
View File

@@ -29,23 +29,24 @@ class TestResumeLlama(unittest.TestCase):
"""
@with_temp_dir
def test_resume_qlora_packed(self, temp_dir):
def test_resume_lora_packed(self, temp_dir):
# pylint: disable=duplicate-code
cfg = DictDefault(
{
"base_model": "JackFram/llama-68m",
"tokenizer_type": "LlamaTokenizer",
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 1024,
"sample_packing": True,
"flash_attention": True,
"load_in_4bit": True,
"adapter": "qlora",
"lora_r": 32,
"lora_alpha": 64,
"load_in_8bit": True,
"adapter": "lora",
"lora_r": 8,
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_target_linear": True,
"val_set_size": 0.1,
"special_tokens": {},
"val_set_size": 0.01,
"special_tokens": {
"pad_token": "<|endoftext|>",
},
"datasets": [
{
"path": "vicgalle/alpaca-gpt4",
@@ -57,11 +58,11 @@ class TestResumeLlama(unittest.TestCase):
"gradient_accumulation_steps": 1,
"output_dir": temp_dir,
"learning_rate": 0.00001,
"optimizer": "adamw_torch",
"optimizer": "adamw_8bit",
"lr_scheduler": "cosine",
"save_steps": 10,
"save_steps": 3,
"save_total_limit": 5,
"max_steps": 40,
"max_steps": 15,
"use_tensorboard": True,
}
)
@@ -77,7 +78,7 @@ class TestResumeLlama(unittest.TestCase):
resume_cfg = cfg | DictDefault(
{
"resume_from_checkpoint": f"{temp_dir}/checkpoint-30/",
"resume_from_checkpoint": f"{temp_dir}/checkpoint-9/",
}
)
normalize_config(resume_cfg)
@@ -93,4 +94,4 @@ class TestResumeLlama(unittest.TestCase):
)
pattern = r"first_step\s+(\d+)"
first_steps = int(re.findall(pattern, res.stdout)[0])
assert first_steps == 31
assert first_steps == 10

6
tests/e2e/patched/test_unsloth_qlora.py
View File

@@ -42,7 +42,7 @@ class TestUnslothQLoRA:
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_target_linear": True,
"val_set_size": 0.2,
"val_set_size": 0.05,
"special_tokens": {
"pad_token": "<|endoftext|>",
},
@@ -92,7 +92,7 @@ class TestUnslothQLoRA:
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_target_linear": True,
"val_set_size": 0.2,
"val_set_size": 0.05,
"special_tokens": {
"pad_token": "<|endoftext|>",
},
@@ -146,7 +146,7 @@ class TestUnslothQLoRA:
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_target_linear": True,
"val_set_size": 0.2,
"val_set_size": 0.05,
"special_tokens": {
"pad_token": "<|endoftext|>",
},

6
tests/e2e/test_optimizers.py
View File

@@ -94,6 +94,7 @@ class TestCustomOptimizers(unittest.TestCase):
},
],
"num_epochs": 1,
"max_steps": 5,
"micro_batch_size": 8,
"gradient_accumulation_steps": 1,
"output_dir": temp_dir,
@@ -115,7 +116,7 @@ class TestCustomOptimizers(unittest.TestCase):
{
"base_model": "HuggingFaceTB/SmolLM2-135M",
"sequence_len": 1024,
"val_set_size": 0.1,
"val_set_size": 0.01,
"special_tokens": {
"pad_token": "<|endoftext|>",
},
@@ -126,13 +127,14 @@ class TestCustomOptimizers(unittest.TestCase):
},
],
"num_epochs": 1,
"micro_batch_size": 4,
"micro_batch_size": 2,
"gradient_accumulation_steps": 2,
"output_dir": temp_dir,
"learning_rate": 0.00001,
"optimizer": "schedule_free_adamw",
"lr_scheduler": "constant",
"save_safetensors": True,
"max_steps": 10,
}
)
# pylint: disable=duplicate-code

1
tests/e2e/test_relora_llama.py
View File

@@ -52,6 +52,7 @@ class TestReLoraLlama(unittest.TestCase):
],
"warmup_steps": 15,
"num_epochs": 2,
"max_steps": 51, # at least 2x relora_steps
"micro_batch_size": 4,
"gradient_accumulation_steps": 1,
"output_dir": temp_dir,