From 86a91e260b8515474ac4b7d9c631e35d63c8004c Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Sun, 13 Aug 2023 11:28:58 -0400
Subject: [PATCH] save tokenizer before training starts (#380)

---
 scripts/finetune.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/finetune.py b/scripts/finetune.py
index 5d94490a6..850606356 100644
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -307,6 +307,7 @@ def train(
 
     if not Path(cfg.output_dir).is_dir():
         os.makedirs(cfg.output_dir, exist_ok=True)
+    tokenizer.save_pretrained(cfg.output_dir)
     if cfg.flash_optimum:
         with torch.backends.cuda.sdp_kernel(
             enable_flash=True, enable_math=True, enable_mem_efficient=True