diff --git a/src/axolotl/utils/tokenization.py b/src/axolotl/utils/tokenization.py index 7f63a92fe..da0c5bbe2 100644 --- a/src/axolotl/utils/tokenization.py +++ b/src/axolotl/utils/tokenization.py @@ -31,8 +31,8 @@ def check_example_labels(example, tokenizer, text_only=False): ) colored_tokens.append(colored_token) - delimiter = "" if text_only else " " - LOG.info(delimiter.join(colored_tokens)) + output = " ".join(colored_tokens) + LOG.info(output) LOG.info("\n\n\n") - return " ".join(colored_tokens) + return output