tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

make batch smaller

Browse Source
This commit is contained in:
Wing Lian
2024-12-18 16:23:50 -05:00
parent 9e218fbcfd
commit 3d1e2dcef4
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/axolotl/datasets.py
View File

@@ -53,7 +53,11 @@ class TokenizedPromptDataset(Dataset):
map_kwargs["batched"] = True map_kwargs["batched"] = True
map_kwargs["batch_size"] = 100 map_kwargs["batch_size"] = 100
if self.prompt_tokenizer.filter_rows: if self.prompt_tokenizer.filter_rows:
dataset = dataset.filter(self.prompt_tokenizer.filter_rows) dataset = dataset.filter(
self.prompt_tokenizer.filter_rows,
num_proc=num_proc,
desc="Filtering Rows",
)
return dataset.map( return dataset.map(
self.prompt_tokenizer.tokenize_prompt, self.prompt_tokenizer.tokenize_prompt,
num_proc=num_proc, num_proc=num_proc,