Fix unsupported operand type(s) for |

This commit is contained in:
NanoCode012
2023-05-29 15:33:40 +09:00
parent b832a0ac62
commit be22551435

View File

@@ -4,7 +4,7 @@ import abc
import copy import copy
import functools import functools
import logging import logging
from typing import Dict, List, Tuple from typing import Dict, List, Tuple, Union
from transformers import PreTrainedTokenizer from transformers import PreTrainedTokenizer
@@ -433,7 +433,7 @@ def parse_tokenized_to_result(
current_len: int, current_len: int,
res: Dict[str, List[int]], res: Dict[str, List[int]],
labels: list[int], labels: list[int],
pad_token_id: int | None = None, pad_token_id: Union[int, None] = None,
) -> Tuple[Dict[str, List[int]], int]: ) -> Tuple[Dict[str, List[int]], int]:
""" """
Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result