diff --git a/comfy/model_management.py b/comfy/model_management.py
index 70c2d5e22..412752503 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -823,7 +823,7 @@ def unet_inital_load_device(parameters, dtype):
 
     mem_dev = get_free_memory(torch_dev)
     mem_cpu = get_free_memory(cpu_dev)
-    if mem_dev > mem_cpu and model_size < mem_dev:
+    if mem_dev > mem_cpu and model_size < mem_dev and comfy.memory_management.aimdo_allocator is None:
         return torch_dev
     else:
         return cpu_dev