diff --git a/deepspeed/zero3_cpu.json b/deepspeed/zero3_cpu.json
index 9c8ac81eb..a2cb1a064 100644
--- a/deepspeed/zero3_cpu.json
+++ b/deepspeed/zero3_cpu.json
@@ -2,7 +2,12 @@
   "zero_optimization": {
     "stage": 3,
     "offload_optimizer": {
-      "device": "cpu"
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "offload_param": {
+      "device": "cpu",
+      "pin_memory": true
     },
     "overlap_comm": true,
     "contiguous_gradients": true,