use alternate math-hard repo

examples/cloud/modal.yaml

@@ -6,7 +6,7 @@ volumes:
 secrets:
   - HF_TOKEN
   - WANDB_API_KEY
-branch:
+branch: cli-cloud-modal
 gpu: h100
 gpu_count: 1
 memory: 128

lm_eval-kd.yaml

@@ -0,0 +1,11 @@
+lm_eval_model: axolotl-ai-co/numina-8b-ep1-exp1
+lm_eval_tasks:
+  - leaderboard_math_hard
+lm_eval_batch_size: 64
+
+apply_chat_template: false
+wandb_project: numina-kd-experiment
+wandb_entity: axolotl-ai
+bf16: true
+flash_attention: true
+output_dir: ./outputs/model-evals-out

src/axolotl/cli/cloud/modal_.py

@@ -99,6 +99,7 @@ class ModalCloud(Cloud):
                     # Random id for cache busting of branch commits
                     f"RUN echo '{str(randint(0, 1000000))}'",  # nosec B311
                     f"RUN cd /workspace/axolotl && git fetch && git checkout {self.config.branch}",
+                    "RUN cd /workspace/ && git clone https://github.com/winglian/lm-evaluation-harness.git && cd lm-evaluation-harness && pip install -e .[math]",
                 ]
             )