4a5876df7a
* fix: explicit set workflow permission and move secrets to necessary steps only * fix: comment * fix: more permission restrict * chore: add read for pypi
276 lines
10 KiB
YAML
276 lines
10 KiB
YAML
name: ci-cd-base
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- "main"
|
|
paths:
|
|
- 'docker/Dockerfile-base'
|
|
- 'docker/Dockerfile-uv-base'
|
|
- '.github/workflows/base.yml'
|
|
pull_request:
|
|
paths:
|
|
- 'docker/Dockerfile-base'
|
|
- 'docker/Dockerfile-uv-base'
|
|
- '.github/workflows/base.yml'
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
build-base:
|
|
if: ${{ github.repository_owner == 'axolotl-ai-cloud' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }}
|
|
timeout-minutes: 480
|
|
# this job needs to be run on self-hosted GPU runners...
|
|
runs-on: ubuntu-latest-m
|
|
env:
|
|
HAS_DOCKERHUB_CREDS: ${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.8.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.9.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.9.1
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.10.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.12"
|
|
pytorch: 2.10.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
# - cuda: "129"
|
|
# cuda_version: 12.9.1
|
|
# cudnn_version: ""
|
|
# python_version: "3.12"
|
|
# pytorch: 2.9.1
|
|
# torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
# dockerfile: "Dockerfile-base"
|
|
# platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "130"
|
|
cuda_version: 13.0.0
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.9.1
|
|
torch_cuda_arch_list: "9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "130"
|
|
cuda_version: 13.0.0
|
|
cudnn_version: ""
|
|
python_version: "3.12"
|
|
pytorch: 2.9.1
|
|
torch_cuda_arch_list: "9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "130"
|
|
cuda_version: 13.0.0
|
|
cudnn_version: ""
|
|
python_version: "3.12"
|
|
pytorch: 2.10.0
|
|
torch_cuda_arch_list: "9.0+PTX"
|
|
dockerfile: "Dockerfile-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
# - cuda: "128"
|
|
# cuda_version: 12.8.1
|
|
# cudnn_version: ""
|
|
# python_version: "3.11"
|
|
# pytorch: nightly
|
|
# torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
# dockerfile: "Dockerfile-base-nightly"
|
|
# # "next" is for release candidates of pytorch
|
|
# - cuda: "128"
|
|
# cuda_version: 12.8.1
|
|
# cudnn_version: ""
|
|
# python_version: "3.11"
|
|
# pytorch: next
|
|
# torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
# dockerfile: "Dockerfile-base-next"
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
- name: Docker metadata
|
|
id: metadata
|
|
uses: docker/metadata-action@v5
|
|
with:
|
|
images: |
|
|
axolotlai/axolotl-base
|
|
- name: Login to Docker Hub
|
|
uses: docker/login-action@v3
|
|
if: ${{ github.event_name != 'pull_request' && env.HAS_DOCKERHUB_CREDS == 'true' }}
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
- name: Build
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: .
|
|
file: ./docker/${{ matrix.dockerfile }}
|
|
platforms: ${{ matrix.platforms }}
|
|
push: ${{ github.event_name != 'pull_request' }}
|
|
tags: ${{ steps.metadata.outputs.tags }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
|
|
labels: ${{ steps.metadata.outputs.labels }}
|
|
build-args: |
|
|
CUDA_VERSION=${{ matrix.cuda_version }}
|
|
CUDNN_VERSION=${{ matrix.cudnn_version }}
|
|
CUDA=${{ matrix.cuda }}
|
|
PYTHON_VERSION=${{ matrix.python_version }}
|
|
PYTORCH_VERSION=${{ matrix.pytorch }}
|
|
TORCH_CUDA_ARCH_LIST=${{ matrix.torch_cuda_arch_list }}
|
|
build-base-uv:
|
|
if: ${{ github.repository_owner == 'axolotl-ai-cloud' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }}
|
|
timeout-minutes: 480
|
|
runs-on: ubuntu-latest-m
|
|
env:
|
|
HAS_DOCKERHUB_CREDS: ${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.8.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.9.1
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.12"
|
|
pytorch: 2.9.1
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.9.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.10.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "128"
|
|
cuda_version: 12.8.1
|
|
cudnn_version: ""
|
|
python_version: "3.12"
|
|
pytorch: 2.10.0
|
|
torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
# - cuda: "129"
|
|
# cuda_version: 12.9.1
|
|
# cudnn_version: ""
|
|
# python_version: "3.12"
|
|
# pytorch: 2.9.1
|
|
# torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
|
|
# dockerfile: "Dockerfile-uv-base"
|
|
# platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "130"
|
|
cuda_version: 13.0.0
|
|
cudnn_version: ""
|
|
python_version: "3.11"
|
|
pytorch: 2.9.1
|
|
torch_cuda_arch_list: "9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "130"
|
|
cuda_version: 13.0.0
|
|
cudnn_version: ""
|
|
python_version: "3.12"
|
|
pytorch: 2.9.1
|
|
torch_cuda_arch_list: "9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
- cuda: "130"
|
|
cuda_version: 13.0.0
|
|
cudnn_version: ""
|
|
python_version: "3.12"
|
|
pytorch: 2.10.0
|
|
torch_cuda_arch_list: "9.0+PTX"
|
|
dockerfile: "Dockerfile-uv-base"
|
|
platforms: "linux/amd64,linux/arm64"
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
- name: Docker metadata
|
|
id: metadata
|
|
uses: docker/metadata-action@v5
|
|
with:
|
|
images: |
|
|
axolotlai/axolotl-base-uv
|
|
- name: Login to Docker Hub
|
|
uses: docker/login-action@v3
|
|
if: ${{ github.event_name != 'pull_request' && env.HAS_DOCKERHUB_CREDS == 'true' }}
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
- name: Build
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: .
|
|
file: ./docker/${{ matrix.dockerfile }}
|
|
platforms: ${{ matrix.platforms }}
|
|
push: ${{ github.event_name != 'pull_request' }}
|
|
tags: ${{ steps.metadata.outputs.tags }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
|
|
labels: ${{ steps.metadata.outputs.labels }}
|
|
build-args: |
|
|
CUDA_VERSION=${{ matrix.cuda_version }}
|
|
CUDNN_VERSION=${{ matrix.cudnn_version }}
|
|
CUDA=${{ matrix.cuda }}
|
|
PYTHON_VERSION=${{ matrix.python_version }}
|
|
PYTORCH_VERSION=${{ matrix.pytorch }}
|
|
TORCH_CUDA_ARCH_LIST=${{ matrix.torch_cuda_arch_list }}
|