don't strip the prompt for check since we don't strip to tokenize anymore (#650)

This commit is contained in:
Wing Lian
2023-09-28 12:21:51 -04:00
committed by GitHub
parent b2edaaeff6
commit 8662e8ffe8

View File

@@ -81,7 +81,7 @@ class PromptTokenizingStrategy(abc.ABC):
self, prompt: str, add_eos_token: bool = True, strip_bos_token: bool = False
) -> BatchEncoding:
result: BatchEncoding
if not prompt.strip():
if not prompt:
LOG.warning("Empty text requested for tokenization.")
result = BatchEncoding(data={"input_ids": [], "attention_mask": []})
else: