From ee20600b9a7c3e2416f08ffa256fa5b0fe954b8e Mon Sep 17 00:00:00 2001
From: Wing Lian <wing@axolotl.ai>
Date: Mon, 13 Jan 2025 08:46:35 -0500
Subject: [PATCH] use alternate math-hard repo

---
 examples/cloud/modal.yaml       |  2 +-
 lm_eval-kd.yaml                 | 11 +++++++++++
 src/axolotl/cli/cloud/modal_.py |  1 +
 3 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 lm_eval-kd.yaml

diff --git a/examples/cloud/modal.yaml b/examples/cloud/modal.yaml
index f42a65980..07c784026 100644
--- a/examples/cloud/modal.yaml
+++ b/examples/cloud/modal.yaml
@@ -6,7 +6,7 @@ volumes:
 secrets:
   - HF_TOKEN
   - WANDB_API_KEY
-branch:
+branch: cli-cloud-modal
 gpu: h100
 gpu_count: 1
 memory: 128
diff --git a/lm_eval-kd.yaml b/lm_eval-kd.yaml
new file mode 100644
index 000000000..5e4d8b50e
--- /dev/null
+++ b/lm_eval-kd.yaml
@@ -0,0 +1,11 @@
+lm_eval_model: axolotl-ai-co/numina-8b-ep1-exp1
+lm_eval_tasks:
+  - leaderboard_math_hard
+lm_eval_batch_size: 64
+
+apply_chat_template: false
+wandb_project: numina-kd-experiment
+wandb_entity: axolotl-ai
+bf16: true
+flash_attention: true
+output_dir: ./outputs/model-evals-out
diff --git a/src/axolotl/cli/cloud/modal_.py b/src/axolotl/cli/cloud/modal_.py
index 412d56696..a2513bdb2 100644
--- a/src/axolotl/cli/cloud/modal_.py
+++ b/src/axolotl/cli/cloud/modal_.py
@@ -99,6 +99,7 @@ class ModalCloud(Cloud):
                     # Random id for cache busting of branch commits
                     f"RUN echo '{str(randint(0, 1000000))}'",  # nosec B311
                     f"RUN cd /workspace/axolotl && git fetch && git checkout {self.config.branch}",
+                    "RUN cd /workspace/ && git clone https://github.com/winglian/lm-evaluation-harness.git && cd lm-evaluation-harness && pip install -e .[math]",
                 ]
             )