feat(merge): save tokenizer on merge (#362)

2023-08-13 00:18:10 +09:00
parent 35c8b90306
commit 289d5c403d
1 changed files with 1 additions and 0 deletions
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -265,6 +265,7 @@ def train(
        if cfg.local_rank == 0:
            LOG.info("saving merged model")
            model.save_pretrained(str(Path(cfg.output_dir) / "merged"))
+            tokenizer.save_pretrained(str(Path(cfg.output_dir) / "merged"))
        return

    if cfg.inference: