Process reward models (#2241)

* adding model_cfg to set num_labels

* using a num_labels field instead

* linting

* WIP stepwise prompt tokenizer

* this should work?

* trainer working?

* pushing to runpod

* fixing saving

* updating conf

* updating config, adding docs

* adding stepwise supervision docpage

* updating tests

* adding test for dataset

* fixing tests

* linting

* addressing some comments

* adding additional cfg fields support

* updating tests, fixing cfg

* fixing tests

* updating loss

* Update test_process_reward_model_smollm2.py

* updating loss values and seed

* dumb pre-commit

tests/e2e/test_process_reward_model_smollm2.py

@@ -0,0 +1,69 @@
+"""
+E2E tests for process reward model w/ lora llama
+"""
+
+import logging
+import os
+import unittest
+
+from axolotl.cli.args import TrainerCliArgs
+from axolotl.common.datasets import load_datasets
+from axolotl.train import train
+from axolotl.utils.config import normalize_config
+from axolotl.utils.dict import DictDefault
+
+from .utils import check_model_output_exists, check_tensorboard, with_temp_dir
+
+LOG = logging.getLogger("axolotl.tests.e2e")
+os.environ["WANDB_DISABLED"] = "true"
+
+
+class TestProcessRewardSmolLM2(unittest.TestCase):
+    """
+    Test case for Llama process reward models using LoRA
+    """
+
+    @with_temp_dir
+    def test_prm(self, temp_dir):
+        # pylint: disable=duplicate-code
+        cfg = DictDefault(
+            {
+                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "model_type": "AutoModelForTokenClassification",
+                "num_labels": 2,
+                "process_reward_model": True,
+                "sequence_len": 512,
+                "val_set_size": 0.0,
+                "datasets": [
+                    {
+                        "path": "trl-lib/math_shepherd",
+                        "type": "stepwise_supervised",
+                        "step_separator": "\n",
+                        "split": "train[:10%]",
+                    },
+                ],
+                "max_steps": 100,
+                "num_epochs": 1,
+                "micro_batch_size": 4,
+                "gradient_accumulation_steps": 1,
+                "output_dir": temp_dir,
+                "learning_rate": 0.0005,
+                "optimizer": "adamw_torch",
+                "lr_scheduler": "cosine",
+                "gradient_checkpointing": True,
+                "warmup_ratio": 0.1,
+                "use_tensorboard": True,
+                "special_tokens": {"pad_token": "<|endoftext|>"},
+                "seed": 42,
+            }
+        )
+        normalize_config(cfg)
+        cli_args = TrainerCliArgs()
+        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
+
+        train(cfg=cfg, dataset_meta=dataset_meta)
+        check_tensorboard(
+            temp_dir + "/runs", "train/train_loss", 2.5, "Train Loss is too high"
+        )
+
+        check_model_output_exists(temp_dir, cfg)

tests/e2e/test_reward_model_smollm2.py

@@ -12,25 +12,25 @@ from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 
-from .utils import check_model_output_exists, with_temp_dir
+from .utils import check_model_output_exists, check_tensorboard, with_temp_dir
 
 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"
 
 
-class TestRewardModelLoraLlama(unittest.TestCase):
+class TestRewardModelLoraSmolLM2(unittest.TestCase):
     """
     Test case for Llama reward models using LoRA
     """
 
     @with_temp_dir
-    def test_rm_fft(self, temp_dir):
+    def test_rm_lora(self, temp_dir):
         # pylint: disable=duplicate-code
         cfg = DictDefault(
             {
-                "base_model": "JackFram/llama-68m",
+                "base_model": "HuggingFaceTB/SmolLM2-135M",
                 "model_type": "AutoModelForSequenceClassification",
-                "tokenizer_type": "LlamaTokenizer",
+                "num_labels": 1,
                 "chat_template": "alpaca",
                 "reward_model": True,
                 "sequence_len": 1024,
@@ -42,16 +42,16 @@ class TestRewardModelLoraLlama(unittest.TestCase):
                 "lora_target_linear": True,
                 "val_set_size": 0.0,
                 "special_tokens": {
-                    "unk_token": "<unk>",
-                    "bos_token": "<s>",
-                    "eos_token": "</s>",
+                    "pad_token": "<|endoftext|>",
                 },
                 "datasets": [
                     {
                         "path": "argilla/distilabel-intel-orca-dpo-pairs",
                         "type": "bradley_terry.chat_template",
+                        "split": "train[:10%]",
                     },
                 ],
+                "lora_modules_to_save": ["embed_tokens", "lm_head"],
                 "remove_unused_columns": False,
                 "max_steps": 10,
                 "num_epochs": 1,
@@ -59,10 +59,11 @@ class TestRewardModelLoraLlama(unittest.TestCase):
                 "gradient_accumulation_steps": 1,
                 "output_dir": temp_dir,
                 "learning_rate": 0.00001,
-                "optimizer": "adamw_bnb_8bit",
+                "optimizer": "adamw_torch",
                 "lr_scheduler": "cosine",
                 "gradient_checkpointing": True,
                 "warmup_ratio": 0.1,
+                "use_tensorboard": True,
             }
         )
         normalize_config(cfg)
@@ -70,4 +71,7 @@ class TestRewardModelLoraLlama(unittest.TestCase):
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
 
         train(cfg=cfg, dataset_meta=dataset_meta)
+        check_tensorboard(
+            temp_dir + "/runs", "train/train_loss", 2.5, "Train Loss is too high"
+        )
         check_model_output_exists(temp_dir, cfg)