From 7ecc3a408c13ab20e1ed4802a77894e2302fe02e Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Sun, 7 Jan 2024 12:45:19 +0900 Subject: [PATCH] Fix(debug): Use space delimiter for debug_text_only also --- src/axolotl/utils/tokenization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/axolotl/utils/tokenization.py b/src/axolotl/utils/tokenization.py index 7f63a92fe..da0c5bbe2 100644 --- a/src/axolotl/utils/tokenization.py +++ b/src/axolotl/utils/tokenization.py @@ -31,8 +31,8 @@ def check_example_labels(example, tokenizer, text_only=False): ) colored_tokens.append(colored_token) - delimiter = "" if text_only else " " - LOG.info(delimiter.join(colored_tokens)) + output = " ".join(colored_tokens) + LOG.info(output) LOG.info("\n\n\n") - return " ".join(colored_tokens) + return output