chore: improve instruction and made linearize optional

This commit is contained in:
NanoCode012
2025-02-05 19:33:15 +07:00
parent 49746b184f
commit 0f82bd2d18
2 changed files with 6 additions and 2 deletions

View File

@@ -15,7 +15,9 @@ plugins:
linearize: true
```
Step 2: Remove the config above and finetune with lora with below possible targets.
Run axolotl: `python -m axolotl.cli.convert_linear_attention config.yaml` TODO: change path CLI
Step 2: Remove the config `linearize: true` and finetune with lora with below possible targets.
```yaml
lora_target_modules: ["q_proj", "k_proj", "v_proj", "o_proj"]
@@ -24,3 +26,5 @@ lora_target_modules: ["q_proj", "k_proj", "v_proj", "o_proj"]
# to allow this config to work with lora
# unfrozen_parameters: ['.*feature_map_q.mlp.layer.*', '.*feature_map_k.mlp.layer.*', '.*window_factors.*']
```
`axolotl train config.yaml --base-model={output_dir}/distilled --trust-remote-code`

View File

@@ -44,4 +44,4 @@ class LinearAttentionArgs(BaseModel):
attention_config: AttentionConfig
linearize: bool
linearize: Optional[bool] = False