tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

use math.ceil instead of round /cc #498

Browse Source
This commit is contained in:
Aman Karmani
2023-08-29 01:03:41 +00:00
parent 8e197f6fb4
commit fd55bc87e2
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/utils/trainer.py
View File

@@ -588,7 +588,9 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer, total_num_
"padding": True, # True/"longest" is the default "padding": True, # True/"longest" is the default
} }
if cfg.pad_to_sequence_len: if cfg.pad_to_sequence_len:
data_collator_kwargs["pad_to_multiple_of"] = 64 * round(cfg.sequence_len / 64) data_collator_kwargs["pad_to_multiple_of"] = 64 * math.ceil(
cfg.sequence_len / 64
)
else: else:
# A100 is best at 64, while others at 8. Let's use the larger so we don't have to check # A100 is best at 64, while others at 8. Let's use the larger so we don't have to check
# https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html # https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html