Adding logging enhancement

This commit is contained in:
theobjectivedad
2023-07-14 07:26:19 -05:00
parent ef17e15483
commit 553a86b52c
14 changed files with 119 additions and 73 deletions

View File

@@ -14,6 +14,7 @@ from .prompt_tokenizers import InvalidDataException, PromptTokenizingStrategy
# let's check to ensure we don't truncate an item in the middle, we'll use
# the collators later on to pad the datasets
LOG = logging.getLogger("axolotl")
class TokenizedPromptDataset(IterableDataset):
"""
@@ -115,7 +116,7 @@ class ConstantLengthDataset(IterableDataset):
"attention_mask": attention_mask,
}
else:
logging.warning(
LOG.warning(
f"dropping batch due to tensor size mismatch input_ids: {input_ids.size()}, labels: {labels.size()}, attention_mask: {attention_mask.size()}"
)
buffer = {