bump HF versions except for trl (#2427)

This commit is contained in:
Wing Lian
2025-03-20 10:22:05 -04:00
committed by GitHub
parent 4d92a68a96
commit 38df5a36ea
10 changed files with 29 additions and 19 deletions

View File

@@ -2,6 +2,7 @@
import functools
import logging
import os
from pathlib import Path
from typing import List, Optional, Tuple, Union
@@ -344,6 +345,7 @@ def load_tokenized_prepared_datasets(
)
ds_from_iter.save_to_disk(str(prepared_ds_path))
else:
os.makedirs(prepared_ds_path, exist_ok=True)
dataset.save_to_disk(str(prepared_ds_path))
if cfg.push_dataset_to_hub:
LOG.info(