fix sharegpt handling from hf, don't worry about loading llama if using earlier transformers release

This commit is contained in:
Wing Lian
2023-04-20 09:19:46 -04:00
parent 8e2a5609b3
commit 8d437853c8
4 changed files with 29 additions and 7 deletions

View File

@@ -128,6 +128,10 @@ conv_vicuna_v1_1 = Conversation(
class ShareGPTPrompter:
def build_prompt(self, source, tokenizer):
# ignore the system prompt if provided
if source[0]["from"] == "system":
source.pop(0)
if len(source) < 2:
# If there isn't a back and forth conversation, ignore it
# also happens on the data splitting leaving empty conversations