From 8b16ecd448de5872eec3b5bed9c25b6b190014df Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Fri, 25 Aug 2023 21:59:09 -0400
Subject: [PATCH] updated dataset

---
 src/axolotl/utils/callbacks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/axolotl/utils/callbacks.py b/src/axolotl/utils/callbacks.py
index 1aa0168ca..fa3c907df 100644
--- a/src/axolotl/utils/callbacks.py
+++ b/src/axolotl/utils/callbacks.py
@@ -142,7 +142,7 @@ def bench_eval_callback_factory(trainer, tokenizer):
         bench_dataset = load_dataset(
             "pharaouk/dharma-1",
             data_files={
-                "eval": "dharma_eval.json",
+                "eval": "dharma_1_full.json",
             },
         )
         bench_dataset["eval"] = bench_dataset["eval"].map(transform_subject)
@@ -242,7 +242,7 @@ def bench_eval_callback_factory(trainer, tokenizer):
                 loss_bench += loss.item()
             # Extract results by subject.
             results = {"bench_loss": loss_bench / len(data_loader)}
-            bench_name = bench_dataset["name"]
+            bench_name = bench_dataset["subject"]
             bench_names: dict = {s: {"refs": [], "preds": []} for s in set(bench_name)}
             for s, p, r in zip(bench_name, preds, refs):  # pylint: disable=invalid-name
                 bench_names[s]["preds"].append(p)