tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

strip out hacky qlora-fsdp workarounds now that qlora-fsdp fixes are upstreamed (#1428)

Browse Source
This commit is contained in:
Wing Lian
2024-03-21 11:56:13 -04:00
committed by GitHub
parent 7d55607368
commit 2a1589f6f6
8 changed files with 27 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/e2e/test_mixtral.py
View File

@@ -77,7 +77,7 @@ class TestMixtral(unittest.TestCase):
model, _ = train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
assert (
model.base_model.model.model.layers[0].block_sparse_moe.gate.weight.dtype
== torch.uint8
== torch.float32
)
assert (Path(temp_dir) / "adapter_model.bin").exists()
@@ -131,7 +131,7 @@ class TestMixtral(unittest.TestCase):
model, _ = train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
assert (
model.base_model.model.model.layers[0].block_sparse_moe.gate.weight.dtype
== torch.uint8
== torch.float32
)
assert (Path(temp_dir) / "adapter_model.bin").exists()