tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

nit

Browse Source
This commit is contained in:
Sunny Liu
2025-02-21 10:59:03 -05:00
committed by Sung Ching Liu
parent 4caa59a087
commit 3dd5c6f8ec
2 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/config.qmd
View File

@@ -78,8 +78,8 @@ tf32: true # require >=ampere
bfloat16: true # require >=ampere bfloat16: true # require >=ampere
float16: true float16: true
# Tensor parallel # Use Tensor parallel
tp_size: 1 # should be set to the number of cuda devices available tensor_parallel: true # require multi-gGPU
# Limit the memory for all available GPUs to this amount (if an integer, expressed in gigabytes); default: unset # Limit the memory for all available GPUs to this amount (if an integer, expressed in gigabytes); default: unset
gpu_memory_limit: 20GiB gpu_memory_limit: 20GiB

4
src/axolotl/utils/models.py
View File

@@ -762,10 +762,8 @@ class ModelLoader:
return hf_ds_cfg return hf_ds_cfg
skip_move_to_device = False skip_move_to_device = False
if self.cfg.tp_size is not None: if self.cfg.tensor_parallel:
# self.model_kwargs["tp_plan"] = "auto"
del self.model_kwargs["device_map"] del self.model_kwargs["device_map"]
# skip_move_to_device = True
if ( # pylint: disable=condition-evals-to-constant) if ( # pylint: disable=condition-evals-to-constant)
(self.cfg.fsdp and self.cfg.fsdp_config.fsdp_cpu_ram_efficient_loading) (self.cfg.fsdp and self.cfg.fsdp_config.fsdp_cpu_ram_efficient_loading)