tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

auto gptq support

Browse Source
This commit is contained in:
Wing Lian
2023-08-23 21:16:48 -04:00
parent 548787daae
commit 1deb767fe8
4 changed files with 44 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@@ -1,3 +1,5 @@
--extra-index-url https://huggingface.github.io/autogptq-index/whl/cu117/
auto-gptq
packaging packaging
peft @ git+https://github.com/huggingface/peft.git peft @ git+https://github.com/huggingface/peft.git
transformers @ git+https://github.com/huggingface/transformers.git transformers @ git+https://github.com/huggingface/transformers.git

4
src/axolotl/utils/config.py
View File

@@ -97,9 +97,7 @@ def validate_config(cfg):
"To calculate the equivalent gradient_accumulation_steps, divide batch_size / micro_batch_size / number of gpus.", "To calculate the equivalent gradient_accumulation_steps, divide batch_size / micro_batch_size / number of gpus.",
) )
if cfg.load_4bit: if cfg.load_4bit:
raise ValueError( raise ValueError("cfg.load_4bit parameter has been deprecated")
"cfg.load_4bit parameter has been deprecated and replaced by cfg.gptq"
)
if cfg.adapter == "qlora": if cfg.adapter == "qlora":
if cfg.merge_lora: if cfg.merge_lora:

127
src/axolotl/utils/models.py
View File

@@ -4,19 +4,29 @@
import logging import logging
import math import math
import os import os
<<<<<<< HEAD
from pathlib import Path from pathlib import Path
from typing import Optional, Tuple # noqa: F401 from typing import Optional, Tuple # noqa: F401
=======
from typing import TYPE_CHECKING, Optional, Tuple # noqa: F401
>>>>>>> 10d25df (auto gptq support)
import bitsandbytes as bnb import bitsandbytes as bnb
import torch import torch
import transformers import transformers
from optimum.bettertransformer import BetterTransformer from optimum.bettertransformer import BetterTransformer
<<<<<<< HEAD
from peft import PeftConfig from peft import PeftConfig
=======
from peft import prepare_model_for_kbit_training
from peft.tuners.lora import LoraLayer
>>>>>>> 10d25df (auto gptq support)
from transformers import ( # noqa: F401 from transformers import ( # noqa: F401
AutoConfig, AutoConfig,
AutoModelForCausalLM, AutoModelForCausalLM,
AutoTokenizer, AutoTokenizer,
BitsAndBytesConfig, BitsAndBytesConfig,
GPTQConfig,
LlamaConfig, LlamaConfig,
PreTrainedModel, PreTrainedModel,
PreTrainedTokenizerBase, PreTrainedTokenizerBase,
@@ -155,32 +165,15 @@ def load_model(
LOG.info("patching _expand_mask") LOG.info("patching _expand_mask")
hijack_expand_mask() hijack_expand_mask()
try:
if cfg.gptq:
from alpaca_lora_4bit.monkeypatch.peft_tuners_lora_monkey_patch import (
replace_peft_model_with_int4_lora_model,
)
replace_peft_model_with_int4_lora_model()
except Exception as err:
LOG.exception(err)
raise err
if not cfg.gptq and (
(cfg.adapter == "lora" and load_in_8bit)
or (cfg.adapter == "qlora" and cfg.load_in_4bit)
):
try:
from peft import prepare_model_for_kbit_training
except ImportError:
# For backward compatibility
from peft import (
prepare_model_for_int8_training as prepare_model_for_kbit_training,
)
model_kwargs = {} model_kwargs = {}
if cfg.model_revision: if cfg.model_revision:
model_kwargs["revision"] = cfg.model_revision model_kwargs["revision"] = cfg.model_revision
if cfg.gptq:
# TODO we should figure out how read the models config.json first
model_kwargs["quantization_config"] = GPTQConfig(
bits=cfg.gptq_bits,
disable_exllama=True,
)
if cfg.adapter == "qlora" and cfg.load_in_4bit: if cfg.adapter == "qlora" and cfg.load_in_4bit:
model_kwargs["quantization_config"] = BitsAndBytesConfig( model_kwargs["quantization_config"] = BitsAndBytesConfig(
load_in_4bit=True, load_in_4bit=True,
@@ -191,45 +184,7 @@ def load_model(
bnb_4bit_quant_type="nf4", bnb_4bit_quant_type="nf4",
) )
try: try:
if cfg.gptq and cfg.is_llama_derived_model: if cfg.is_llama_derived_model and not cfg.trust_remote_code and not cfg.gptq:
from alpaca_lora_4bit.autograd_4bit import load_llama_model_4bit_low_ram
from huggingface_hub import snapshot_download
try:
snapshot_download_kwargs = {}
if cfg.base_model_ignore_patterns:
snapshot_download_kwargs[
"ignore_patterns"
] = cfg.base_model_ignore_patterns
cache_model_path = Path(
snapshot_download(base_model, **snapshot_download_kwargs)
)
files = (
list(cache_model_path.glob("*.pt"))
+ list(cache_model_path.glob("*.safetensors"))
+ list(cache_model_path.glob("*.bin"))
)
if len(files) > 0:
model_path = str(files[0])
else:
LOG.warning(
"unable to find a cached model file, this will likely fail..."
)
model_path = str(cache_model_path)
except Exception: # pylint: disable=broad-exception-caught
model_path = cfg.base_model
model, _ = load_llama_model_4bit_low_ram(
base_model_config if base_model_config else base_model,
model_path,
device_map=cfg.device_map,
half=cfg.fp16,
groupsize=cfg.gptq_groupsize if cfg.gptq_groupsize else -1,
is_v1_model=cfg.gptq_model_v1
if cfg.gptq_model_v1 is not None
else True,
)
load_in_8bit = False
elif cfg.is_llama_derived_model and not cfg.trust_remote_code:
from transformers import LlamaForCausalLM from transformers import LlamaForCausalLM
config_kwargs = {} config_kwargs = {}
@@ -275,15 +230,24 @@ def load_model(
# ) # )
# model.train() # sets to train instead of eval mode # model.train() # sets to train instead of eval mode
elif model_type and not cfg.trust_remote_code: elif model_type and not cfg.trust_remote_code:
model = getattr(transformers, model_type).from_pretrained( if cfg.gptq:
base_model, model = AutoModelForCausalLM.from_pretrained(
device_map=cfg.device_map, base_model,
load_in_8bit=cfg.load_in_8bit and cfg.adapter is not None, device_map=cfg.device_map,
load_in_4bit=cfg.load_in_4bit and cfg.adapter is not None, torch_dtype=cfg.torch_dtype,
torch_dtype=cfg.torch_dtype, trust_remote_code=cfg.trust_remote_code or False,
trust_remote_code=cfg.trust_remote_code or False, **model_kwargs,
**model_kwargs, )
) else:
model = getattr(transformers, model_type).from_pretrained(
base_model,
device_map=cfg.device_map,
load_in_8bit=cfg.load_in_8bit and cfg.adapter is not None,
load_in_4bit=cfg.load_in_4bit and cfg.adapter is not None,
torch_dtype=cfg.torch_dtype,
trust_remote_code=cfg.trust_remote_code or False,
**model_kwargs,
)
else: else:
config = AutoConfig.from_pretrained( config = AutoConfig.from_pretrained(
base_model, base_model,
@@ -359,11 +323,12 @@ def load_model(
module.to(torch.float32) module.to(torch.float32)
needs_fa2_dtype = cfg.adapter or cfg.fsdp needs_fa2_dtype = cfg.adapter or cfg.fsdp
if not cfg.gptq and ( if (cfg.adapter == "lora" and load_in_8bit) or (
(cfg.adapter == "lora" and load_in_8bit) cfg.adapter == "qlora" and cfg.load_in_4bit
or (cfg.adapter == "qlora" and cfg.load_in_4bit)
): ):
LOG.info("converting PEFT model w/ prepare_model_for_kbit_training") LOG.info("converting PEFT model w/ prepare_model_for_kbit_training")
if cfg.gradient_checkpointing:
model.gradient_checkpointing_enable()
model = prepare_model_for_kbit_training( model = prepare_model_for_kbit_training(
model, use_gradient_checkpointing=cfg.gradient_checkpointing model, use_gradient_checkpointing=cfg.gradient_checkpointing
) )
@@ -385,22 +350,10 @@ def load_model(
if cfg.ddp and not load_in_8bit: if cfg.ddp and not load_in_8bit:
model.to(f"cuda:{cfg.local_rank}") model.to(f"cuda:{cfg.local_rank}")
if cfg.gptq:
# Scales to half
LOG.info("Fitting 4bit scales and zeros to half")
for _, module in model.named_modules():
if "Autograd4bitQuantLinear" in str(type(module)) or "Linear4bitLt" in str(
type(module)
):
if hasattr(module, "is_v1_model") and module.is_v1_model:
module.zeros = module.zeros.half()
module.scales = module.scales.half()
module.bias = module.bias.half()
if ( if (
torch.cuda.device_count() > 1 torch.cuda.device_count() > 1
and int(os.getenv("WORLD_SIZE", "1")) > 1 and int(os.getenv("WORLD_SIZE", "1")) > 1
and (cfg.gptq or cfg.load_in_4bit) and (cfg.load_in_4bit)
): ):
# llama is PROBABLY model parallelizable, but the default isn't that it is # llama is PROBABLY model parallelizable, but the default isn't that it is
# so let's only set it for the 4bit, see # so let's only set it for the 4bit, see

18
src/axolotl/utils/trainer.py
View File

@@ -447,23 +447,7 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer, total_num_
training_arguments_kwargs["seed"] = cfg.seed training_arguments_kwargs["seed"] = cfg.seed
if cfg.gradient_checkpointing: if cfg.gradient_checkpointing:
if cfg.gptq: training_arguments_kwargs["gradient_checkpointing"] = cfg.gradient_checkpointing
from alpaca_lora_4bit.gradient_checkpointing import (
apply_gradient_checkpointing,
)
gradient_checkpointing_ratio = (
cfg.gradient_checkpointing_ratio
if cfg.gradient_checkpointing_ratio
else 1.0
)
apply_gradient_checkpointing(
model, checkpoint_ratio=gradient_checkpointing_ratio
)
else:
training_arguments_kwargs[
"gradient_checkpointing"
] = cfg.gradient_checkpointing
if cfg.fsdp: if cfg.fsdp:
training_arguments_kwargs["fsdp"] = cfg.fsdp training_arguments_kwargs["fsdp"] = cfg.fsdp
if cfg.fsdp_config: if cfg.fsdp_config: