chore: improve instruction and made linearize optional
This commit is contained in:
@@ -15,7 +15,9 @@ plugins:
|
||||
linearize: true
|
||||
```
|
||||
|
||||
Step 2: Remove the config above and finetune with lora with below possible targets.
|
||||
Run axolotl: `python -m axolotl.cli.convert_linear_attention config.yaml` TODO: change path CLI
|
||||
|
||||
Step 2: Remove the config `linearize: true` and finetune with lora with below possible targets.
|
||||
|
||||
```yaml
|
||||
lora_target_modules: ["q_proj", "k_proj", "v_proj", "o_proj"]
|
||||
@@ -24,3 +26,5 @@ lora_target_modules: ["q_proj", "k_proj", "v_proj", "o_proj"]
|
||||
# to allow this config to work with lora
|
||||
# unfrozen_parameters: ['.*feature_map_q.mlp.layer.*', '.*feature_map_k.mlp.layer.*', '.*window_factors.*']
|
||||
```
|
||||
|
||||
`axolotl train config.yaml --base-model={output_dir}/distilled --trust-remote-code`
|
||||
|
||||
@@ -44,4 +44,4 @@ class LinearAttentionArgs(BaseModel):
|
||||
|
||||
attention_config: AttentionConfig
|
||||
|
||||
linearize: bool
|
||||
linearize: Optional[bool] = False
|
||||
|
||||
Reference in New Issue
Block a user