let's not cleanup the cached datasets

2023-08-08 21:27:55 -04:00
parent 26983a1974
commit 57d9bf711c
2 changed files with 1 additions and 6 deletions
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -371,12 +371,6 @@ def train(
            model = BetterTransformer.reverse(model)
        model.save_pretrained(cfg.output_dir)

-    trainer.accelerator.wait_for_everyone()
-    if trainer.accelerator.is_main_process:
-        train_dataset.cleanup_cache_files()
-        if eval_dataset:
-            eval_dataset.cleanup_cache_files()
-

 if __name__ == "__main__":
    fire.Fire(train)