From 98f828fad98643d4f8f31e355cc79a1fd42e1bb8 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 18 May 2024 09:36:26 -0400 Subject: [PATCH 1/9] Remove unnecessary code. --- comfy/ldm/modules/attention.py | 11 ++--------- comfy/ldm/modules/diffusionmodules/model.py | 1 - 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 426530867..88ee2f32d 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -318,11 +318,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None): return attention_pytorch(q, k, v, heads, mask) q, k, v = map( - lambda t: t.unsqueeze(3) - .reshape(b, -1, heads, dim_head) - .permute(0, 2, 1, 3) - .reshape(b * heads, -1, dim_head) - .contiguous(), + lambda t: t.reshape(b, -1, heads, dim_head), (q, k, v), ) @@ -335,10 +331,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None): out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=mask) out = ( - out.unsqueeze(0) - .reshape(b, heads, -1, dim_head) - .permute(0, 2, 1, 3) - .reshape(b, -1, heads * dim_head) + out.reshape(b, -1, heads * dim_head) ) return out diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py index fabc5c5e5..04eb83b21 100644 --- a/comfy/ldm/modules/diffusionmodules/model.py +++ b/comfy/ldm/modules/diffusionmodules/model.py @@ -3,7 +3,6 @@ import math import torch import torch.nn as nn import numpy as np -from einops import rearrange from typing import Optional, Any import logging From 0bdc2b15c75426af75a326d5966ad47aab5b76d3 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 18 May 2024 10:11:44 -0400 Subject: [PATCH 2/9] Cleanup. --- comfy/ldm/modules/attention.py | 10 +++------- comfy/ldm/modules/diffusionmodules/openaimodel.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 88ee2f32d..aa74b6323 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -6,7 +6,7 @@ from einops import rearrange, repeat from typing import Optional, Any import logging -from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding +from .diffusionmodules.util import AlphaBlender, timestep_embedding from .sub_quadratic_attention import efficient_dot_product_attention from comfy import model_management @@ -454,15 +454,11 @@ class BasicTransformerBlock(nn.Module): self.norm1 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) self.norm3 = operations.LayerNorm(inner_dim, dtype=dtype, device=device) - self.checkpoint = checkpoint self.n_heads = n_heads self.d_head = d_head self.switch_temporal_ca_to_sa = switch_temporal_ca_to_sa def forward(self, x, context=None, transformer_options={}): - return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint) - - def _forward(self, x, context=None, transformer_options={}): extra_options = {} block = transformer_options.get("block", None) block_index = transformer_options.get("block_index", 0) @@ -629,7 +625,7 @@ class SpatialTransformer(nn.Module): x = self.norm(x) if not self.use_linear: x = self.proj_in(x) - x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + x = x.movedim(1, -1).flatten(1, 2).contiguous() if self.use_linear: x = self.proj_in(x) for i, block in enumerate(self.transformer_blocks): @@ -637,7 +633,7 @@ class SpatialTransformer(nn.Module): x = block(x, context=context[i], transformer_options=transformer_options) if self.use_linear: x = self.proj_out(x) - x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + x = x.reshape(x.shape[0], h, w, x.shape[-1]).movedim(-1, 1).contiguous() if not self.use_linear: x = self.proj_out(x) return x + x_in diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.py index 1f5a4ded2..ba8fc2c4a 100644 --- a/comfy/ldm/modules/diffusionmodules/openaimodel.py +++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py @@ -258,7 +258,7 @@ class ResBlock(TimestepBlock): else: if emb_out is not None: if self.exchange_temb_dims: - emb_out = rearrange(emb_out, "b t c ... -> b c t ...") + emb_out = emb_out.movedim(1, 2) h = h + emb_out h = self.out_layers(h) return self.skip_connection(x) + h From f37a47110bef97186b8392830d5c49148a06ecef Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 19 May 2024 11:45:36 -0400 Subject: [PATCH 3/9] Make --preview-method auto default to the fast latent2rgb previews. --- latent_preview.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/latent_preview.py b/latent_preview.py index 4dbcbf455..12927647a 100644 --- a/latent_preview.py +++ b/latent_preview.py @@ -63,8 +63,6 @@ def get_previewer(device, latent_format): if method == LatentPreviewMethod.Auto: method = LatentPreviewMethod.Latent2RGB - if taesd_decoder_path: - method = LatentPreviewMethod.TAESD if method == LatentPreviewMethod.TAESD: if taesd_decoder_path: From 4ae1515f14d338a347fdf8f647e062f5bc17d196 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 19 May 2024 17:42:35 -0400 Subject: [PATCH 4/9] Slightly faster latent2rgb previews. --- latent_preview.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/latent_preview.py b/latent_preview.py index 12927647a..05d750e2c 100644 --- a/latent_preview.py +++ b/latent_preview.py @@ -37,12 +37,13 @@ class Latent2RGBPreviewer(LatentPreviewer): self.latent_rgb_factors = torch.tensor(latent_rgb_factors, device="cpu") def decode_latent_to_preview(self, x0): - latent_image = x0[0].permute(1, 2, 0).cpu() @ self.latent_rgb_factors + self.latent_rgb_factors = self.latent_rgb_factors.to(dtype=x0.dtype, device=x0.device) + latent_image = x0[0].permute(1, 2, 0) @ self.latent_rgb_factors latents_ubyte = (((latent_image + 1) / 2) .clamp(0, 1) # change scale from -1..1 to 0..1 .mul(0xFF) # to 0..255 - .byte()).cpu() + ).to(device="cpu", dtype=torch.uint8, non_blocking=True) return Image.fromarray(latents_ubyte.numpy()) From 11a2ad5110a96ac5895e889d455c227d1b80dd30 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 19 May 2024 17:58:03 -0400 Subject: [PATCH 5/9] Fix controlnet not upcasting on models that have it enabled. --- comfy/cldm/cldm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py index 5eee5a51c..28076dd92 100644 --- a/comfy/cldm/cldm.py +++ b/comfy/cldm/cldm.py @@ -52,6 +52,7 @@ class ControlNet(nn.Module): adm_in_channels=None, transformer_depth_middle=None, transformer_depth_output=None, + attn_precision=None, device=None, operations=comfy.ops.disable_weight_init, **kwargs, @@ -202,7 +203,7 @@ class ControlNet(nn.Module): SpatialTransformer( ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim, disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations + use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations ) ) self.input_blocks.append(TimestepEmbedSequential(*layers)) @@ -262,7 +263,7 @@ class ControlNet(nn.Module): mid_block += [SpatialTransformer( # always uses a self-attn ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim, disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations + use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations ), ResBlock( ch, From 09e069ae6ca700cabb885662a83a05227b4f79a5 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 20 May 2024 06:22:29 -0400 Subject: [PATCH 6/9] Log the pytorch version. --- comfy/model_management.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/comfy/model_management.py b/comfy/model_management.py index 21ae8d291..b21928628 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -120,6 +120,11 @@ total_vram = get_total_memory(get_torch_device()) / (1024 * 1024) total_ram = psutil.virtual_memory().total / (1024 * 1024) logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram)) +try: + logging.info("pytorch version: {}".format(torch.version.__version__)) +except: + pass + try: OOM_EXCEPTION = torch.cuda.OutOfMemoryError except: From 4bc1884478a14dea2d64a34f9beb1c52a2d16b09 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" <128333288+ltdrdata@users.noreply.github.com> Date: Mon, 20 May 2024 19:58:46 +0900 Subject: [PATCH 7/9] Provide a better error message when attempting to execute the workflow with a missing node. (#3517) --- execution.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/execution.py b/execution.py index 47d58b9d7..86bde1c46 100644 --- a/execution.py +++ b/execution.py @@ -622,8 +622,17 @@ def full_type_name(klass): def validate_prompt(prompt): outputs = set() for x in prompt: + if 'class_type' not in prompt[x]: + error = { + "type": "invalid_prompt", + "message": f"Cannot execute due to a missing node", + "details": f"Node ID '#{x}'", + "extra_info": {} + } + return (False, error, [], []) + class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']] - if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE == True: + if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE is True: outputs.add(x) if len(outputs) == 0: From 276f8fce9f5a80b500947fb5745a4dde9e84622d Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 20 May 2024 07:03:06 -0400 Subject: [PATCH 8/9] Print error when node is missing. --- execution.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/execution.py b/execution.py index 86bde1c46..76225a962 100644 --- a/execution.py +++ b/execution.py @@ -625,13 +625,23 @@ def validate_prompt(prompt): if 'class_type' not in prompt[x]: error = { "type": "invalid_prompt", - "message": f"Cannot execute due to a missing node", + "message": f"Cannot execute because a node is missing the class_type property.", + "details": f"Node ID '#{x}'", + "extra_info": {} + } + return (False, error, [], []) + + class_type = prompt[x]['class_type'] + class_ = nodes.NODE_CLASS_MAPPINGS.get(class_type, None) + if class_ is None: + error = { + "type": "invalid_prompt", + "message": f"Cannot execute because node {class_type} does not exist.", "details": f"Node ID '#{x}'", "extra_info": {} } return (False, error, [], []) - class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']] if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE is True: outputs.add(x) From 1900e5119f70d6db0677fe91194050be3c4476c4 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 20 May 2024 08:19:54 -0400 Subject: [PATCH 9/9] Fix potential issue. --- comfy/ldm/modules/attention.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index aa74b6323..74a2fd99a 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -625,7 +625,7 @@ class SpatialTransformer(nn.Module): x = self.norm(x) if not self.use_linear: x = self.proj_in(x) - x = x.movedim(1, -1).flatten(1, 2).contiguous() + x = x.movedim(1, 3).flatten(1, 2).contiguous() if self.use_linear: x = self.proj_in(x) for i, block in enumerate(self.transformer_blocks): @@ -633,7 +633,7 @@ class SpatialTransformer(nn.Module): x = block(x, context=context[i], transformer_options=transformer_options) if self.use_linear: x = self.proj_out(x) - x = x.reshape(x.shape[0], h, w, x.shape[-1]).movedim(-1, 1).contiguous() + x = x.reshape(x.shape[0], h, w, x.shape[-1]).movedim(3, 1).contiguous() if not self.use_linear: x = self.proj_out(x) return x + x_in