From f18925fb4bb7af8f4a6e690242e24cef6249e2c0 Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Wed, 14 Aug 2024 22:46:46 +0900 Subject: [PATCH 1/4] fix: parse eager_attention (#1824) --- src/axolotl/utils/config/models/input/v0_4_1/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py index b765263ba..7bf77965c 100644 --- a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py +++ b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py @@ -614,6 +614,8 @@ class AxolotlInputConfig( flash_attn_fuse_mlp: Optional[bool] = None flash_optimum: Optional[bool] = None + eager_attention: Optional[bool] = None + unsloth_cross_entropy_loss: Optional[bool] = None unsloth_lora_mlp: Optional[bool] = None unsloth_lora_qkv: Optional[bool] = None From 68a3c7678a3cbf5098239f84a548c9492ebe4387 Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Fri, 16 Aug 2024 20:51:19 +0900 Subject: [PATCH 2/4] fix: parse model_kwargs (#1825) --- src/axolotl/utils/config/models/input/v0_4_1/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py index 7bf77965c..647d6b88c 100644 --- a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py +++ b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py @@ -321,6 +321,8 @@ class ModelInputConfig(BaseModel): ) trust_remote_code: Optional[bool] = None + model_kwargs: Optional[Dict[str, Any]] = None + @field_validator("trust_remote_code") @classmethod def hint_trust_remote_code(cls, trust_remote_code): From 803fed3e904a527ce302fc818aef3500e07722c9 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 16 Aug 2024 10:41:51 -0400 Subject: [PATCH 3/4] update sklearn versrion, torch compile env vars, don't worry about failure on preprocess load model (#1821) * update sklearn versrion, torch compile env vars, don't worry about failure on preprocess load model * There is already a condition check within the function. This outer one is not necessary Co-authored-by: NanoCode012 --------- Co-authored-by: NanoCode012 --- requirements.txt | 2 +- src/axolotl/cli/preprocess.py | 9 ++++++++- src/axolotl/utils/trainer.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f32af373b..dc74b916f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ numpy>=1.24.4 # qlora things evaluate==0.4.1 scipy -scikit-learn==1.2.2 +scikit-learn==1.4.2 pynvml art fschat @ git+https://github.com/lm-sys/FastChat.git@27a05b04a35510afb1d767ae7e5990cbd278f8fe diff --git a/src/axolotl/cli/preprocess.py b/src/axolotl/cli/preprocess.py index e0dd7c2dc..e12462c00 100644 --- a/src/axolotl/cli/preprocess.py +++ b/src/axolotl/cli/preprocess.py @@ -82,7 +82,14 @@ def do_cli(config: Union[Path, str] = Path("examples/"), **kwargs): # "copying from a non-meta parameter in the checkpoint to a meta parameter in the current model" warnings.simplefilter("ignore") with init_empty_weights(include_buffers=True): - AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True) + # fmt: off + try: + AutoModelForCausalLM.from_pretrained( + model_name, trust_remote_code=True + ) + except Exception as exc: # pylint: disable=broad-exception-caught,unused-variable # nosec B110 # noqa F841 + pass + # fmt: on LOG.info( Fore.GREEN diff --git a/src/axolotl/utils/trainer.py b/src/axolotl/utils/trainer.py index 02234d8b7..26796f2e5 100644 --- a/src/axolotl/utils/trainer.py +++ b/src/axolotl/utils/trainer.py @@ -390,6 +390,14 @@ def calculate_total_num_steps(cfg, train_dataset, update=True): return total_num_steps +def setup_torch_compile_env(cfg): + if cfg.torch_compile: + if not cfg.torch_compile_backend: + os.environ["ACCELERATE_DYNAMO_BACKEND"] = "INDUCTOR" + else: + os.environ["ACCELERATE_DYNAMO_BACKEND"] = cfg.torch_compile_backend.upper() + + def setup_deepspeed_env(cfg, stage=None): os.environ["ACCELERATE_USE_DEEPSPEED"] = "true" os.environ["ACCELERATE_DEEPSPEED_CONFIG_FILE"] = cfg.deepspeed @@ -434,6 +442,8 @@ def prepare_optim_env(cfg): stage = deepspeed_config.get("zero_optimization", {}).get("stage", None) setup_deepspeed_env(cfg, stage=stage) + setup_torch_compile_env(cfg) + if (cfg.bf16 == "auto" and is_torch_bf16_gpu_available()) or cfg.bf16 is True: os.environ["ACCELERATE_MIXED_PRECISION"] = "bf16" elif cfg.fp16: From b1d29212222f5dd566fab5453e3fd342089c8191 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 16 Aug 2024 21:32:00 -0400 Subject: [PATCH 4/4] add validation to prevent 8bit lora finetuning on H100s (#1827) --- .../utils/config/models/input/v0_4_1/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py index 647d6b88c..5e690bb88 100644 --- a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py +++ b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py @@ -1267,6 +1267,19 @@ class AxolotlConfigWCapabilities(AxolotlInputConfig): return data + @model_validator(mode="before") + @classmethod + def check_hopper_8bit_lora(cls, data): + is_sm_90: bool = ( + data["capabilities"] + and data["capabilities"].get("compute_capability") == "sm_90" + ) + if data.get("adapter") and data.get("load_in_8bit") and is_sm_90: + # see https://github.com/bitsandbytes-foundation/bitsandbytes/issues/538#issuecomment-2262945464 + raise ValueError("8-bit LoRA is not supported on Hopper GPUs") + + return data + @model_validator(mode="before") @classmethod def check_fsdp_deepspeed(cls, data):