retry flaky test_packing_stream_dataset test that timesout on read (#2052) [skip ci]

This commit is contained in:
Wing Lian
2024-11-13 13:16:16 -05:00
committed by GitHub
parent 01881c3113
commit 010d0e7ff3
2 changed files with 3 additions and 0 deletions

View File

@@ -1,2 +1,3 @@
pytest
pytest-xdist
pytest-retry

View File

@@ -2,6 +2,7 @@
import functools
import unittest
import pytest
import torch
from datasets import load_dataset
from torch.utils.data import DataLoader
@@ -21,6 +22,7 @@ class TestPretrainingPacking(unittest.TestCase):
self.tokenizer = AutoTokenizer.from_pretrained("huggyllama/llama-7b")
self.tokenizer.pad_token = "</s>"
@pytest.mark.flaky(retries=3, delay=5)
def test_packing_stream_dataset(self):
# pylint: disable=duplicate-code
dataset = load_dataset(