From c110e3eb48dc9cb4367151e82588d9043539c8a9 Mon Sep 17 00:00:00 2001 From: Dan Saunders Date: Sat, 30 Aug 2025 01:08:53 -0400 Subject: [PATCH] remove setup.py, requirements.txt and refs --- .coveragerc | 1 - .github/workflows/lint.yml | 2 +- .github/workflows/multi-gpu-e2e.yml | 2 - .github/workflows/pypi.yml | 12 +- .github/workflows/tests-nightly.yml | 14 +-- .github/workflows/tests.yml | 8 +- .runpod/Dockerfile | 5 +- MANIFEST.in | 3 +- cicd/Dockerfile.jinja | 10 +- ...setuptools_axolotl_dynamic_dependencies.py | 104 ------------------ 10 files changed, 23 insertions(+), 138 deletions(-) delete mode 100644 src/setuptools_axolotl_dynamic_dependencies.py diff --git a/.coveragerc b/.coveragerc index c5670410c..23732e3a9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,6 @@ source = axolotl omit = */tests/* - setup.py [report] exclude_lines = diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 95a41fbf8..ce5f006e0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,7 +6,7 @@ on: types: [opened, synchronize, reopened, ready_for_review] paths: - '**.py' - - 'requirements.txt' + - 'pyproject.toml' - '.github/workflows/*.yml' - "*.[q]md" - "examples/**/*.y[a]?ml" diff --git a/.github/workflows/multi-gpu-e2e.yml b/.github/workflows/multi-gpu-e2e.yml index 84bc095d6..d4815c75a 100644 --- a/.github/workflows/multi-gpu-e2e.yml +++ b/.github/workflows/multi-gpu-e2e.yml @@ -4,8 +4,6 @@ on: pull_request: paths: - 'tests/e2e/multigpu/**.py' - - 'requirements.txt' - - 'setup.py' - 'pyproject.toml' - '.github/workflows/multi-gpu-e2e.yml' - 'src/axolotl/core/trainers/mixins/sequence_parallel.py' diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 3d16e9240..b89832ea6 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -46,20 +46,16 @@ jobs: - name: Install dependencies run: | uv pip install --system wheel packaging==23.2 - uv pip install --system --no-build-isolation -e . - uv pip install --system -e .[dev] + uv pip install --system --no-build-isolation -e .[dev] - name: Extract tag name id: tag run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3) - - name: Update version in setup.py + - name: Build package run: | - sed -i -E 's/version="([0-9.]+)",/version="${{ steps.tag.outputs.TAG_NAME }}",/g' setup.py - - - name: Build a source dist - run: | - python setup.py sdist + uv pip install --system build + python -m build - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests-nightly.yml b/.github/workflows/tests-nightly.yml index 4490b0fe9..3f90235bb 100644 --- a/.github/workflows/tests-nightly.yml +++ b/.github/workflows/tests-nightly.yml @@ -52,13 +52,13 @@ jobs: run: | uv pip install --system torch==${{ matrix.pytorch_version }} torchvision - - name: Update requirements.txt + - name: Update pyproject.toml for nightly builds run: | - sed -i 's#^transformers.*#transformers @ git+https://github.com/huggingface/transformers.git@main#' requirements.txt - sed -i 's#^peft.*#peft @ git+https://github.com/huggingface/peft.git@main#' requirements.txt - sed -i 's#^accelerate.*#accelerate @ git+https://github.com/huggingface/accelerate.git@main#' requirements.txt - sed -i 's#^trl.*#trl @ git+https://github.com/huggingface/trl.git@main#' requirements.txt - sed -i 's#^datasets.*#datasets @ git+https://github.com/huggingface/datasets.git@main#' requirements.txt + sed -i 's#"transformers==.*"#"transformers @ git+https://github.com/huggingface/transformers.git@main"#' pyproject.toml + sed -i 's#"peft==.*"#"peft @ git+https://github.com/huggingface/peft.git@main"#' pyproject.toml + sed -i 's#"accelerate==.*"#"accelerate @ git+https://github.com/huggingface/accelerate.git@main"#' pyproject.toml + sed -i 's#"trl==.*"#"trl @ git+https://github.com/huggingface/trl.git@main"#' pyproject.toml + sed -i 's#"datasets==.*"#"datasets @ git+https://github.com/huggingface/datasets.git@main"#' pyproject.toml - name: Install dependencies run: | @@ -178,4 +178,4 @@ jobs: echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV - name: Run tests job on Modal run: | - modal run -m cicd.multigpu + modal run cicd.multigpu diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fae436f36..1ec456416 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,6 @@ on: - "main" paths: - '**.py' - - 'requirements.txt' - 'pyproject.toml' - '.github/workflows/*.yml' - 'cicd/cicd.sh' @@ -16,7 +15,6 @@ on: types: [opened, synchronize, reopened, ready_for_review] paths: - '**.py' - - 'requirements.txt' - 'pyproject.toml' - '.github/workflows/*.yml' - 'cicd/cicd.sh' @@ -261,7 +259,7 @@ jobs: echo "E2E_DOCKERFILE=${{ matrix.dockerfile || 'Dockerfile.jinja'}}" >> $GITHUB_ENV - name: Run tests job on Modal run: | - modal run -m cicd.e2e_tests + modal run cicd.e2e_tests docker-e2e-tests: if: > @@ -326,7 +324,7 @@ jobs: echo "E2E_DOCKERFILE=${{ matrix.dockerfile || 'Dockerfile.jinja'}}" >> $GITHUB_ENV - name: Run tests job on Modal run: | - modal run -m cicd.e2e_tests + modal run cicd.e2e_tests docker-e2e-cleanup: runs-on: [self-hosted, modal] @@ -370,4 +368,4 @@ jobs: echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV - name: Run tests job on Modal run: | - modal run -m cicd.cleanup + modal run cicd.cleanup diff --git a/.runpod/Dockerfile b/.runpod/Dockerfile index 107caf5f3..3fcad5df8 100644 --- a/.runpod/Dockerfile +++ b/.runpod/Dockerfile @@ -1,9 +1,8 @@ FROM axolotlai/axolotl-cloud:main-py3.11-cu124-2.6.0 COPY .runpod/requirements.txt /requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - python3 -m pip install --upgrade pip && \ - python3 -m pip install --upgrade -r /requirements.txt +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ + /root/.local/bin/uv pip install --system -r /requirements.txt # Environment settings ARG BASE_VOLUME="/runpod-volume" diff --git a/MANIFEST.in b/MANIFEST.in index 3fbb0edca..149c78ad2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ -include requirements.txt +include pyproject.toml include README.md include LICENSE -include src/setuptools_axolotl_dynamic_dependencies.py include src/axolotl/utils/chat_templates/templates/*.jinja recursive-include axolotl *.py diff --git a/cicd/Dockerfile.jinja b/cicd/Dockerfile.jinja index f05be6857..c69b1d42e 100644 --- a/cicd/Dockerfile.jinja +++ b/cicd/Dockerfile.jinja @@ -24,11 +24,11 @@ RUN git fetch origin +$GITHUB_REF && \ # If AXOLOTL_EXTRAS is set, append it in brackets RUN if [ "$NIGHTLY_BUILD" = "true" ] ; then \ - sed -i 's#^transformers.*#transformers @ git+https://github.com/huggingface/transformers.git@main#' requirements.txt; \ - sed -i 's#^peft.*#peft @ git+https://github.com/huggingface/peft.git@main#' requirements.txt; \ - sed -i 's#^accelerate.*#accelerate @ git+https://github.com/huggingface/accelerate.git@main#' requirements.txt; \ - sed -i 's#^trl.*#trl @ git+https://github.com/huggingface/trl.git@main#' requirements.txt; \ - sed -i 's#^datasets.*#datasets @ git+https://github.com/huggingface/datasets.git@main#' requirements.txt; \ + sed -i 's#"transformers==.*"#"transformers @ git+https://github.com/huggingface/transformers.git@main"#' pyproject.toml; \ + sed -i 's#"peft==.*"#"peft @ git+https://github.com/huggingface/peft.git@main"#' pyproject.toml; \ + sed -i 's#"accelerate==.*"#"accelerate @ git+https://github.com/huggingface/accelerate.git@main"#' pyproject.toml; \ + sed -i 's#"trl==.*"#"trl @ git+https://github.com/huggingface/trl.git@main"#' pyproject.toml; \ + sed -i 's#"datasets==.*"#"datasets @ git+https://github.com/huggingface/datasets.git@main"#' pyproject.toml; \ fi RUN uv pip install packaging==23.2 setuptools==75.8.0 diff --git a/src/setuptools_axolotl_dynamic_dependencies.py b/src/setuptools_axolotl_dynamic_dependencies.py deleted file mode 100644 index ccd7c72d7..000000000 --- a/src/setuptools_axolotl_dynamic_dependencies.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -dynamic requirements for axolotl -""" - -import platform -import re -from importlib.metadata import PackageNotFoundError, version - -from setuptools.command.build_py import build_py as _build_py - - -def parse_requirements(): - _install_requires = [] - _dependency_links = [] - with open("./requirements.txt", encoding="utf-8") as requirements_file: - lines = [r.strip() for r in requirements_file.readlines()] - for line in lines: - is_extras = ( - "flash-attn" in line - or "flash-attention" in line - or "deepspeed" in line - or "mamba-ssm" in line - or "lion-pytorch" in line - ) - if line.startswith("--extra-index-url"): - # Handle custom index URLs - _, url = line.split() - _dependency_links.append(url) - elif not is_extras and line and line[0] != "#": - # Handle standard packages - _install_requires.append(line) - - try: - xformers_version = [req for req in _install_requires if "xformers" in req][0] - torchao_version = [req for req in _install_requires if "torchao" in req][0] - autoawq_version = [req for req in _install_requires if "autoawq" in req][0] - - if "Darwin" in platform.system(): - # don't install xformers on MacOS - _install_requires.pop(_install_requires.index(xformers_version)) - else: - # detect the version of torch already installed - # and set it so dependencies don't clobber the torch version - try: - torch_version = version("torch") - except PackageNotFoundError: - torch_version = "2.5.1" - _install_requires.append(f"torch=={torch_version}") - - version_match = re.match(r"^(\d+)\.(\d+)(?:\.(\d+))?", torch_version) - if version_match: - major, minor, patch = version_match.groups() - major, minor = int(major), int(minor) - patch = ( - int(patch) if patch is not None else 0 - ) # Default patch to 0 if not present - else: - raise ValueError("Invalid version format") - - if (major, minor) >= (2, 5): - _install_requires.pop(_install_requires.index(xformers_version)) - if patch == 0: - _install_requires.append("xformers==0.0.28.post2") - else: - _install_requires.append("xformers==0.0.28.post3") - _install_requires.pop(_install_requires.index(autoawq_version)) - elif (major, minor) >= (2, 4): - if patch == 0: - _install_requires.pop(_install_requires.index(xformers_version)) - _install_requires.append("xformers>=0.0.27") - else: - _install_requires.pop(_install_requires.index(xformers_version)) - _install_requires.append("xformers==0.0.28.post1") - elif (major, minor) >= (2, 3): - _install_requires.pop(_install_requires.index(torchao_version)) - if patch == 0: - _install_requires.pop(_install_requires.index(xformers_version)) - _install_requires.append("xformers>=0.0.26.post1") - else: - _install_requires.pop(_install_requires.index(xformers_version)) - _install_requires.append("xformers>=0.0.27") - elif (major, minor) >= (2, 2): - _install_requires.pop(_install_requires.index(torchao_version)) - _install_requires.pop(_install_requires.index(xformers_version)) - _install_requires.append("xformers>=0.0.25.post1") - else: - _install_requires.pop(_install_requires.index(torchao_version)) - _install_requires.pop(_install_requires.index(xformers_version)) - _install_requires.append("xformers>=0.0.23.post1") - - except PackageNotFoundError: - pass - return _install_requires, _dependency_links - - -class BuildPyCommand(_build_py): - """ - custom build_py command to parse dynamic requirements - """ - - def finalize_options(self): - super().finalize_options() - install_requires, _ = parse_requirements() - self.distribution.install_requires = install_requires