tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Set matmul tf32

Browse Source
This commit is contained in:
NanoCode012
2023-06-08 23:41:12 +09:00
parent 73e9ea4069
commit 52765ac588
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/finetune.py
View File

@@ -183,6 +183,9 @@ def train(
cfg.fp16 = True cfg.fp16 = True
cfg.bf16 = False cfg.bf16 = False
if cfg.tf32:
torch.backends.cuda.matmul.allow_tf32 = True
# load the tokenizer first # load the tokenizer first
tokenizer_config = cfg.tokenizer_config or cfg.base_model_config tokenizer_config = cfg.tokenizer_config or cfg.base_model_config
logging.info(f"loading tokenizer... {tokenizer_config}") logging.info(f"loading tokenizer... {tokenizer_config}")