9de5b76336
* feat: move to uv first * fix: update doc to uv first * fix: merge dev/tests into uv pyproject * fix: update docker docs to match current config * fix: migrate examples to readme * fix: add llmcompressor to conflict * feat: rec uv sync with lockfile for dev/ci * fix: update docker docs to clarify how to use uv images * chore: docs * fix: use system python, no venv * fix: set backend cpu * fix: only set for installing pytorch step * fix: remove unsloth kernel and installs * fix: remove U in tests * fix: set backend in deps too * chore: test * chore: comments * fix: attempt to lock torch * fix: workaround torch cuda and not upgraded * fix: forgot to push * fix: missed source * fix: nightly upstream loralinear config * fix: nightly phi3 long rope not work * fix: forgot commit * fix: test phi3 template change * fix: no more requirements * fix: carry over changes from new requirements to pyproject * chore: remove lockfile per discussion * fix: set match-runtime * fix: remove unneeded hf hub buildtime * fix: duplicate cache delete on nightly * fix: torchvision being overridden * fix: migrate to uv images * fix: leftover from merge * fix: simplify base readme * fix: update assertion message to be clearer * chore: docs * fix: change fallback for cicd script * fix: match against main exactly * fix: peft 0.19.1 change * fix: e2e test * fix: ci * fix: e2e test
86 lines
2.3 KiB
Python
86 lines
2.3 KiB
Python
"""
|
|
modal application to run axolotl gpu tests in Modal
|
|
"""
|
|
|
|
import os
|
|
import pathlib
|
|
import tempfile
|
|
|
|
import jinja2
|
|
import modal
|
|
from jinja2 import select_autoescape
|
|
from modal import App, Image
|
|
|
|
cicd_path = pathlib.Path(__file__).parent.resolve()
|
|
|
|
template_loader = jinja2.FileSystemLoader(searchpath=cicd_path)
|
|
template_env = jinja2.Environment(
|
|
loader=template_loader, autoescape=select_autoescape()
|
|
)
|
|
dockerfile = os.environ.get("E2E_DOCKERFILE", "Dockerfile-uv.jinja")
|
|
df_template = template_env.get_template(dockerfile)
|
|
|
|
df_args = {
|
|
"AXOLOTL_EXTRAS": os.environ.get("AXOLOTL_EXTRAS", ""),
|
|
"AXOLOTL_ARGS": os.environ.get("AXOLOTL_ARGS", ""),
|
|
"PYTORCH_VERSION": os.environ.get("PYTORCH_VERSION", "2.6.0"),
|
|
"BASE_TAG": os.environ.get("BASE_TAG", "main-base-py3.11-cu126-2.6.0"),
|
|
"CUDA": os.environ.get("CUDA", "126"),
|
|
"GITHUB_REF": os.environ.get("GITHUB_REF", "refs/heads/main"),
|
|
"GITHUB_SHA": os.environ.get("GITHUB_SHA", ""),
|
|
"NIGHTLY_BUILD": os.environ.get("NIGHTLY_BUILD", ""),
|
|
"CODECOV_TOKEN": os.environ.get("CODECOV_TOKEN", ""),
|
|
"HF_HOME": "/workspace/data/huggingface-cache/hub",
|
|
"PYTHONUNBUFFERED": os.environ.get("PYTHONUNBUFFERED", "1"),
|
|
"DEEPSPEED_LOG_LEVEL": os.environ.get("DEEPSPEED_LOG_LEVEL", "WARNING"),
|
|
}
|
|
|
|
dockerfile_contents = df_template.render(**df_args)
|
|
|
|
temp_dir = tempfile.mkdtemp()
|
|
with open(pathlib.Path(temp_dir) / "Dockerfile", "w", encoding="utf-8") as f:
|
|
f.write(dockerfile_contents)
|
|
|
|
cicd_image = Image.from_dockerfile(
|
|
pathlib.Path(temp_dir) / "Dockerfile",
|
|
force_build=True,
|
|
gpu="A10G",
|
|
).env(df_args)
|
|
|
|
app = App("Axolotl CI/CD", secrets=[])
|
|
|
|
hf_cache_volume = modal.Volume.from_name(
|
|
"axolotl-ci-hf-hub-cache", create_if_missing=True
|
|
)
|
|
VOLUME_CONFIG = {
|
|
"/workspace/data/huggingface-cache/hub": hf_cache_volume,
|
|
}
|
|
|
|
N_GPUS = int(os.environ.get("N_GPUS", 2))
|
|
GPU_CONFIG = f"H100:{N_GPUS}"
|
|
|
|
|
|
def run_cmd(cmd: str, run_folder: str):
|
|
import subprocess # nosec
|
|
|
|
# Propagate errors from subprocess.
|
|
if exit_code := subprocess.call(cmd.split(), cwd=run_folder): # nosec
|
|
exit(exit_code)
|
|
|
|
|
|
@app.function(
|
|
image=cicd_image,
|
|
gpu=GPU_CONFIG,
|
|
timeout=120 * 60,
|
|
cpu=16.0,
|
|
memory=131072 * N_GPUS,
|
|
volumes=VOLUME_CONFIG,
|
|
)
|
|
def cicd_pytest():
|
|
run_cmd("./cicd/multigpu.sh", "/workspace/axolotl")
|
|
|
|
|
|
@app.local_entrypoint()
|
|
def main():
|
|
cicd_pytest.remote()
|