tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

pretrain fixes for mm

Browse Source
This commit is contained in:
Wing Lian
2023-10-30 11:03:55 -04:00
parent 53f93f67bb
commit b52e61a574
3 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/multimodal/pretrain-llava-llama.yml
View File

@@ -7,9 +7,11 @@ is_llama_derived_model: true
multimodal: true multimodal: true
mm_vision_tower: openai/clip-vit-large-patch14 mm_vision_tower: openai/clip-vit-large-patch14
tune_mm_mlp_adapter: true tune_mm_mlp_adapter: true
mm_freeze_backbone: true
mm_vision_select_layer: -2 mm_vision_select_layer: -2
mm_projector_type: mlp2x_gelu mm_projector_type: mlp2x_gelu
mm_image_folder: ./llava/ mm_image_folder: ./llava/
mm_use_im_patch_token: false
load_in_8bit: false load_in_8bit: false
load_in_4bit: false load_in_4bit: false
@@ -54,7 +56,7 @@ flash_attention: true
warmup_steps: 10 warmup_steps: 10
eval_steps: eval_steps:
save_steps: save_steps: 0.1
debug: debug:
deepspeed: deepspeed:
weight_decay: 0.0 weight_decay: 0.0

2
examples/multimodal/pretrain-llava-mistral.yml
View File

@@ -7,9 +7,11 @@ is_mistral_derived_model: true
multimodal: true multimodal: true
mm_vision_tower: openai/clip-vit-large-patch14 mm_vision_tower: openai/clip-vit-large-patch14
tune_mm_mlp_adapter: true tune_mm_mlp_adapter: true
mm_freeze_backbone: true
mm_vision_select_layer: -2 mm_vision_select_layer: -2
mm_projector_type: mlp2x_gelu mm_projector_type: mlp2x_gelu
mm_image_folder: ./llava/ mm_image_folder: ./llava/
mm_use_im_patch_token: false
load_in_8bit: false load_in_8bit: false
load_in_4bit: false load_in_4bit: false

14
src/axolotl/utils/models.py
View File

@@ -286,8 +286,8 @@ def load_model(
def make_inputs_require_grad( def make_inputs_require_grad(
module, module,
input, input,
output, # pylint: disable=redefined-builtin,unused-argument output,
): ): # pylint: disable=redefined-builtin,unused-argument
output.requires_grad_(True) output.requires_grad_(True)
model.get_input_embeddings().register_forward_hook( model.get_input_embeddings().register_forward_hook(
@@ -304,7 +304,7 @@ def load_model(
pretrain_mm_mlp_adapter=cfg.pretrain_mm_mlp_adapter, pretrain_mm_mlp_adapter=cfg.pretrain_mm_mlp_adapter,
mm_projector_type=cfg.mm_projector_type or "linear", mm_projector_type=cfg.mm_projector_type or "linear",
mm_use_im_start_end=cfg.mm_use_im_start_end or False, mm_use_im_start_end=cfg.mm_use_im_start_end or False,
mm_use_im_patch_token=cfg.mm_use_im_patch_token or True, mm_use_im_patch_token=cfg.mm_use_im_patch_token,
mm_vision_select_feature=cfg.mm_vision_select_feature or "patch", mm_vision_select_feature=cfg.mm_vision_select_feature or "patch",
) )
@@ -330,8 +330,8 @@ def load_model(
data_args.image_processor = vision_tower.image_processor data_args.image_processor = vision_tower.image_processor
model.config.image_aspect_ratio = data_args.image_aspect_ratio model.config.image_aspect_ratio = data_args.image_aspect_ratio
model.config.image_grid_pinpoints = data_args.image_grid_pinpoints model.config.image_grid_pinpoints = data_args.image_grid_pinpoints
model.config.tune_mm_mlp_adapter = model_args.tune_mm_mlp_adapter model.config.tune_mm_mlp_adapter = cfg.tune_mm_mlp_adapter
if model_args.tune_mm_mlp_adapter: if cfg.tune_mm_mlp_adapter:
model.requires_grad_(False) model.requires_grad_(False)
for ( for (
p # pylint: disable=invalid-name p # pylint: disable=invalid-name
@@ -347,8 +347,8 @@ def load_model(
model.config.mm_use_im_start_end = ( model.config.mm_use_im_start_end = (
data_args.mm_use_im_start_end data_args.mm_use_im_start_end
) = model_args.mm_use_im_start_end ) = cfg.mm_use_im_start_end
model.config.mm_use_im_patch_token = model_args.mm_use_im_patch_token model.config.mm_use_im_patch_token = cfg.mm_use_im_patch_token
model.initialize_vision_tokenizer(model_args, tokenizer=tokenizer) model.initialize_vision_tokenizer(model_args, tokenizer=tokenizer)
elif cfg.is_llama_derived_model and not cfg.trust_remote_code and not cfg.gptq: elif cfg.is_llama_derived_model and not cfg.trust_remote_code and not cfg.gptq:
from transformers import LlamaForCausalLM from transformers import LlamaForCausalLM