tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

pretrain fixes for mm

Browse Source
This commit is contained in:
Wing Lian
2023-10-30 11:03:55 -04:00
parent 53f93f67bb
commit b52e61a574
3 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/multimodal/pretrain-llava-llama.yml
View File

@@ -7,9 +7,11 @@ is_llama_derived_model: true
multimodal: true
mm_vision_tower: openai/clip-vit-large-patch14
tune_mm_mlp_adapter: true
mm_freeze_backbone: true
mm_vision_select_layer: -2
mm_projector_type: mlp2x_gelu
mm_image_folder: ./llava/
mm_use_im_patch_token: false
load_in_8bit: false
load_in_4bit: false
@@ -54,7 +56,7 @@ flash_attention: true
warmup_steps: 10
eval_steps:
save_steps:
save_steps: 0.1
debug:
deepspeed:
weight_decay: 0.0

2
examples/multimodal/pretrain-llava-mistral.yml
View File

@@ -7,9 +7,11 @@ is_mistral_derived_model: true
multimodal: true
mm_vision_tower: openai/clip-vit-large-patch14
tune_mm_mlp_adapter: true
mm_freeze_backbone: true
mm_vision_select_layer: -2
mm_projector_type: mlp2x_gelu
mm_image_folder: ./llava/
mm_use_im_patch_token: false
load_in_8bit: false
load_in_4bit: false