From dfe591435fc78fed72ccf0a308d11cde366eb60d Mon Sep 17 00:00:00 2001 From: Aman Karmani Date: Tue, 2 Apr 2024 03:19:54 +0000 Subject: [PATCH] make lisa training example work on one 24gb gpu --- examples/llama-2/lisa.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llama-2/lisa.yml b/examples/llama-2/lisa.yml index e692c7ac1..bfb63888a 100644 --- a/examples/llama-2/lisa.yml +++ b/examples/llama-2/lisa.yml @@ -25,7 +25,7 @@ lora_dropout: lora_target_linear: lora_fan_in_fan_out: -lisa_n_layers: 4 +lisa_n_layers: 2 lisa_step_interval: 20 lisa_layers_attribute: model.layers @@ -35,7 +35,7 @@ wandb_watch: wandb_name: wandb_log_model: -gradient_accumulation_steps: 2 +gradient_accumulation_steps: 1 micro_batch_size: 1 num_epochs: 1 optimizer: adamw_bnb_8bit