tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

refactor conversation plucking in sharegpt

Browse Source
This commit is contained in:
Wing Lian
2023-05-28 14:36:33 -04:00
parent 8fe12e3bc1
commit 21c8e2deab
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/axolotl/prompt_tokenizers.py
View File

@@ -268,6 +268,9 @@ class AlpacaReflectionPTStrategy(ReflectionPromptTokenizingStrategy):
class ShareGPTPromptTokenizingStrategy(PromptTokenizingStrategy): class ShareGPTPromptTokenizingStrategy(PromptTokenizingStrategy):
def get_conversation_thread(self, prompt):
return prompt["conversations"]
def tokenize_prompt(self, prompt): def tokenize_prompt(self, prompt):
result = { result = {
"input_ids": [], "input_ids": [],
@@ -279,7 +282,7 @@ class ShareGPTPromptTokenizingStrategy(PromptTokenizingStrategy):
assistant_token = self._get_assistant_token() assistant_token = self._get_assistant_token()
try: try:
for i, part in enumerate( for i, part in enumerate(
self.prompter.build_prompt(prompt["conversations"]) self.prompter.build_prompt(self.get_conversation_thread(prompt))
): ):
if isinstance(part, tuple): if isinstance(part, tuple):
if part[0] == "USER:": if part[0] == "USER:":