pylint for duplicated code for system prompts

This commit is contained in:
Wing Lian
2023-06-18 06:40:28 -04:00
parent 3a38271276
commit 7b57ed7618
2 changed files with 2 additions and 0 deletions

View File

@@ -126,6 +126,7 @@ class ConstantLengthDataset(IterableDataset):
buffer_len = 0
if example:
# FIXME
# just going to drop data points that are too long
if len(example["input_ids"]) <= self.seq_length:
input_ids = example["input_ids"]

View File

@@ -21,6 +21,7 @@ class InstructionWSystemPromptTokenizingStrategy(PromptTokenizingStrategy):
)
def tokenize_prompt(self, prompt):
# pylint: disable=duplicate-code
(
instruction,
input, # pylint: disable=redefined-builtin