Fix(debug): Use space delimiter for debug_text_only also

This commit is contained in:
NanoCode012
2024-01-07 12:45:19 +09:00
committed by GitHub
parent 9ca358b671
commit 7ecc3a408c

View File

@@ -31,8 +31,8 @@ def check_example_labels(example, tokenizer, text_only=False):
)
colored_tokens.append(colored_token)
delimiter = "" if text_only else " "
LOG.info(delimiter.join(colored_tokens))
output = " ".join(colored_tokens)
LOG.info(output)
LOG.info("\n\n\n")
return " ".join(colored_tokens)
return output