tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

better handling when no subjects

Browse Source
This commit is contained in:
Wing Lian
2023-08-21 17:22:06 -04:00
parent 24b0e93235
commit aefd4d74fa
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/axolotl/utils/callbacks.py
View File

@@ -129,9 +129,11 @@ def bench_eval_callback_factory(trainer, tokenizer):
def transform_subject(example): def transform_subject(example):
# Split on ':' and trim whitespace # Split on ':' and trim whitespace
parts = example["subject"].split(":") parts = example["subject"].split(":")
first_part = parts[0].strip().lower() # Lowercase the first part first_part = (
parts[0].strip().lower().replace("-", "_")
) # Lowercase the first part
second_part = ( second_part = (
parts[1].strip().replace("-", "_") parts[1].strip().replace("-", "_") if len(parts) > 1 else "all"
) # Replace hyphens with underscores ) # Replace hyphens with underscores
# Return the transformed values # Return the transformed values