fix relative path for fixtures

src/axolotl/utils/models.py

@@ -125,7 +125,7 @@ def load_model(
             load_in_4bit=True,
             llm_int8_threshold=6.0,
             llm_int8_has_fp16_weight=False,
-            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_compute_dtype=torch_dtype,
             bnb_4bit_use_double_quant=True,
             bnb_4bit_quant_type="nf4",
         )
@@ -174,7 +174,7 @@ def load_model(
                 load_in_8bit=cfg.load_in_8bit and cfg.adapter is not None,
                 load_in_4bit=cfg.load_in_4bit and cfg.adapter is not None,
                 torch_dtype=torch_dtype,
-                device_map=cfg.device_map,
+                device_map="auto" if cfg.world_size == 1 else cfg.device_map,
                 **model_kwargs,
             )
         # elif model_type == "GPTNeoXForCausalLM" and cfg.flash_attention:
@@ -273,13 +273,13 @@ def load_model(
     if (
         torch.cuda.device_count() > 1
         and int(os.getenv("WORLD_SIZE", "1")) > 1
-        and cfg.gptq
+        and (cfg.gptq or cfg.load_in_4bit)
     ):
         # llama is PROBABLY model parallelizable, but the default isn't that it is
         # so let's only set it for the 4bit, see
         # https://github.com/johnsmith0031/alpaca_lora_4bit/blob/08b3fca4a4a9e0d3945be1bab4529f100a428636/finetune.py#L130-L133
-        model.is_parallelizable = True
-        model.model_parallel = True
+        setattr(model, 'is_parallelizable', True)
+        setattr(model, 'model_parallel', True)
 
     requires_grad = []
     for name, param in model.named_parameters(recurse=True):

src/axolotl/utils/trainer.py

@@ -113,7 +113,8 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
         output_dir=cfg.output_dir,
         save_total_limit=3,
         load_best_model_at_end=True
-        if cfg.val_set_size > 0
+        if cfg.load_best_model_at_end is not False  # if explicitly set to False, it should be resort to False
+        and cfg.val_set_size > 0
         and save_steps is not None
         and save_steps % eval_steps == 0
         and cfg.load_in_8bit is not True
@@ -218,7 +219,7 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
 
     trainer_cls = (
         OneCycleLRSchedulerTrainer
-        if cfg.lr_scheduler == "one_cycle" and cfg.fsdp
+        if cfg.lr_scheduler == "one_cycle" and (cfg.fsdp or cfg.adapter == "qlora")
         else transformers.Trainer
     )
     trainer = trainer_cls(