tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix drop_long_seq bug due to truncation in prompt tokenization strategies when using chat_template (#1867)

Browse Source
This commit is contained in:
Chiwan Park
2024-08-27 01:56:12 +09:00
committed by GitHub
parent 6819c12cee
commit 2dac1edf72
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/axolotl/prompt_strategies/chat_template.py
View File

@@ -350,7 +350,8 @@ def load(tokenizer, cfg, ds_cfg: Optional[Dict[str, Any]] = None):
), ),
"roles": ds_cfg.get("roles"), "roles": ds_cfg.get("roles"),
"drop_system_message": ds_cfg.get("drop_system_message", False), "drop_system_message": ds_cfg.get("drop_system_message", False),
"max_length": cfg.sequence_len, # we need to add one for detecting sequences with exceeding the `sequence_len` limit.
"max_length": cfg.sequence_len + 1,
} }
strategy_params = { strategy_params = {