pylint for duplicated code for system prompts
This commit is contained in:
@@ -126,6 +126,7 @@ class ConstantLengthDataset(IterableDataset):
|
|||||||
buffer_len = 0
|
buffer_len = 0
|
||||||
|
|
||||||
if example:
|
if example:
|
||||||
|
# FIXME
|
||||||
# just going to drop data points that are too long
|
# just going to drop data points that are too long
|
||||||
if len(example["input_ids"]) <= self.seq_length:
|
if len(example["input_ids"]) <= self.seq_length:
|
||||||
input_ids = example["input_ids"]
|
input_ids = example["input_ids"]
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ class InstructionWSystemPromptTokenizingStrategy(PromptTokenizingStrategy):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def tokenize_prompt(self, prompt):
|
def tokenize_prompt(self, prompt):
|
||||||
|
# pylint: disable=duplicate-code
|
||||||
(
|
(
|
||||||
instruction,
|
instruction,
|
||||||
input, # pylint: disable=redefined-builtin
|
input, # pylint: disable=redefined-builtin
|
||||||
|
|||||||
Reference in New Issue
Block a user