From 0f82bd2d1894b7317f5d94e715df4c1094d9e2d8 Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Wed, 5 Feb 2025 19:33:15 +0700 Subject: [PATCH] chore: improve instruction and made linearize optional --- src/axolotl/integrations/lolcats/README.md | 6 +++++- src/axolotl/integrations/lolcats/args.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/axolotl/integrations/lolcats/README.md b/src/axolotl/integrations/lolcats/README.md index b665905b0..54c130e79 100644 --- a/src/axolotl/integrations/lolcats/README.md +++ b/src/axolotl/integrations/lolcats/README.md @@ -15,7 +15,9 @@ plugins: linearize: true ``` -Step 2: Remove the config above and finetune with lora with below possible targets. +Run axolotl: `python -m axolotl.cli.convert_linear_attention config.yaml` TODO: change path CLI + +Step 2: Remove the config `linearize: true` and finetune with lora with below possible targets. ```yaml lora_target_modules: ["q_proj", "k_proj", "v_proj", "o_proj"] @@ -24,3 +26,5 @@ lora_target_modules: ["q_proj", "k_proj", "v_proj", "o_proj"] # to allow this config to work with lora # unfrozen_parameters: ['.*feature_map_q.mlp.layer.*', '.*feature_map_k.mlp.layer.*', '.*window_factors.*'] ``` + +`axolotl train config.yaml --base-model={output_dir}/distilled --trust-remote-code` diff --git a/src/axolotl/integrations/lolcats/args.py b/src/axolotl/integrations/lolcats/args.py index 97d6613e9..c2fb58f9c 100644 --- a/src/axolotl/integrations/lolcats/args.py +++ b/src/axolotl/integrations/lolcats/args.py @@ -44,4 +44,4 @@ class LinearAttentionArgs(BaseModel): attention_config: AttentionConfig - linearize: bool + linearize: Optional[bool] = False