From f6060a664e39e40478224049f306ed91f5f93704 Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Wed, 13 Sep 2023 11:45:30 -0400
Subject: [PATCH] Model parallel (#538)

* model-parallel for single process

* fix device/device_map

* fix handling for device
---
 src/axolotl/utils/bench.py  | 2 +-
 src/axolotl/utils/config.py | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/axolotl/utils/bench.py b/src/axolotl/utils/bench.py
index d19e81ecd..30f0985e7 100644
--- a/src/axolotl/utils/bench.py
+++ b/src/axolotl/utils/bench.py
@@ -28,7 +28,7 @@ def gpu_memory_usage_smi(device=0):
 
 
 def log_gpu_memory_usage(log, msg, device):
-    if not torch.cuda.is_available():
+    if not torch.cuda.is_available() or device == "auto":
         return (0, 0, 0)
 
     usage, cache, misc = gpu_memory_usage_all(device)
diff --git a/src/axolotl/utils/config.py b/src/axolotl/utils/config.py
index 9b9f3cdb8..90ed409b9 100644
--- a/src/axolotl/utils/config.py
+++ b/src/axolotl/utils/config.py
@@ -25,7 +25,9 @@ def choose_device(cfg):
             return "cpu"
 
     cfg.device = get_device()
-    if cfg.device_map != "auto":
+    if cfg.world_size == 1:
+        cfg.device_map = "auto"
+    else:
         if cfg.device.startswith("cuda"):
             cfg.device_map = {"": cfg.local_rank}
         else: