diff --git a/comfy/ops.py b/comfy/ops.py
index 279f6b1a7..0d88fb17c 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -84,7 +84,8 @@ def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None, of
         if device is None:
             device = input.device
 
-    if offloadable:
+    if offloadable and (device != s.weight.device or
+                        (s.bias is not None and device != s.bias.device)):
         offload_stream = comfy.model_management.get_offload_stream(device)
     else:
         offload_stream = None