tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Correctly reference mount paths (#2347)

Browse Source 
* Correctly reference mount paths

* Also fix mount paths in lm_eval

* chore: lint

---------

Co-authored-by: Wing Lian <wing@axolotl.ai>
This commit is contained in:
Matt Baker
2025-02-24 08:12:57 -08:00
committed by GitHub
parent 2d5826f544
commit 00fc8109e4
1 changed files with 10 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/axolotl/cli/cloud/modal_.py
View File

@@ -258,25 +258,21 @@ class ModalCloud(Cloud):
def _preprocess(config_yaml: str, volumes=None): def _preprocess(config_yaml: str, volumes=None):
Path("/workspace/artifacts/axolotl").mkdir(parents=True, exist_ok=True) Path("/workspace/mounts").mkdir(parents=True, exist_ok=True)
with open( with open("/workspace/mounts/config.yaml", "w", encoding="utf-8") as f_out:
"/workspace/artifacts/axolotl/config.yaml", "w", encoding="utf-8"
) as f_out:
f_out.write(config_yaml) f_out.write(config_yaml)
run_folder = "/workspace/artifacts/axolotl" run_folder = "/workspace/mounts"
run_cmd( run_cmd(
"axolotl preprocess /workspace/artifacts/axolotl/config.yaml --dataset-processes=8", "axolotl preprocess /workspace/mounts/config.yaml --dataset-processes=8",
run_folder, run_folder,
volumes, volumes,
) )
def _train(config_yaml: str, accelerate: bool = True, volumes=None, **kwargs): def _train(config_yaml: str, accelerate: bool = True, volumes=None, **kwargs):
with open( with open("/workspace/mounts/config.yaml", "w", encoding="utf-8") as f_out:
"/workspace/artifacts/axolotl/config.yaml", "w", encoding="utf-8"
) as f_out:
f_out.write(config_yaml) f_out.write(config_yaml)
run_folder = "/workspace/artifacts/axolotl" run_folder = "/workspace/mounts"
if accelerate: if accelerate:
accelerate_args = "--accelerate" accelerate_args = "--accelerate"
else: else:
@@ -285,20 +281,18 @@ def _train(config_yaml: str, accelerate: bool = True, volumes=None, **kwargs):
if num_processes := kwargs.pop("num_processes", None): if num_processes := kwargs.pop("num_processes", None):
num_processes_args = f"--num-processes {num_processes}" num_processes_args = f"--num-processes {num_processes}"
run_cmd( run_cmd(
f"axolotl train {accelerate_args} {num_processes_args} /workspace/artifacts/axolotl/config.yaml", f"axolotl train {accelerate_args} {num_processes_args} /workspace/mounts/config.yaml",
run_folder, run_folder,
volumes, volumes,
) )
def _lm_eval(config_yaml: str, volumes=None): def _lm_eval(config_yaml: str, volumes=None):
with open( with open("/workspace/mounts/config.yaml", "w", encoding="utf-8") as f_out:
"/workspace/artifacts/axolotl/config.yaml", "w", encoding="utf-8"
) as f_out:
f_out.write(config_yaml) f_out.write(config_yaml)
run_folder = "/workspace/artifacts/axolotl" run_folder = "/workspace/mounts"
run_cmd( run_cmd(
"axolotl lm-eval /workspace/artifacts/axolotl/config.yaml", "axolotl lm-eval /workspace/mounts/config.yaml",
run_folder, run_folder,
volumes, volumes,
) )