diff --git a/README.md b/README.md index 1de9d4c3b..84c10bfe2 100644 --- a/README.md +++ b/README.md @@ -87,13 +87,13 @@ Put your SD checkpoints (the huge ckpt/safetensors files) in: models/checkpoints Put your VAE in: models/vae -At the time of writing this pytorch has issues with python versions higher than 3.10 so make sure your python/pip versions are 3.10. - ### AMD GPUs (Linux only) AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version: ```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.4.2``` +This is the command to install the nightly with ROCm 5.5 that supports the 7000 series and might have some performance improvements: +```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm5.5 -r requirements.txt``` ### NVIDIA @@ -178,16 +178,6 @@ To use a textual inversion concepts/embeddings in a text prompt put them in the ```embedding:embedding_filename.pt``` -### Fedora - -To get python 3.10 on fedora: -```dnf install python3.10``` - -Then you can: - -```python3.10 -m ensurepip``` - -This will let you use: pip3.10 to install all the dependencies. ## How to increase generation speed? diff --git a/comfy/cli_args.py b/comfy/cli_args.py index b56497de0..f1306ef7f 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -59,12 +59,14 @@ attn_group.add_argument("--use-pytorch-cross-attention", action="store_true", he parser.add_argument("--disable-xformers", action="store_true", help="Disable xformers.") vram_group = parser.add_mutually_exclusive_group() +vram_group.add_argument("--gpu-only", action="store_true", help="Store and run everything (text encoders/CLIP models, etc... on the GPU).") vram_group.add_argument("--highvram", action="store_true", help="By default models will be unloaded to CPU memory after being used. This option keeps them in GPU memory.") vram_group.add_argument("--normalvram", action="store_true", help="Used to force normal vram use if lowvram gets automatically enabled.") vram_group.add_argument("--lowvram", action="store_true", help="Split the unet in parts to use less vram.") vram_group.add_argument("--novram", action="store_true", help="When lowvram isn't enough.") vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") + parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.") parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.") parser.add_argument("--windows-standalone-build", action="store_true", help="Windows standalone build: Enable convenient things that most people using the standalone windows build will probably enjoy (like auto opening the page on startup).") diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index 5fb4fa2af..62707dfd2 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -51,9 +51,9 @@ def init_(tensor): # feedforward class GEGLU(nn.Module): - def __init__(self, dim_in, dim_out): + def __init__(self, dim_in, dim_out, dtype=None): super().__init__() - self.proj = comfy.ops.Linear(dim_in, dim_out * 2) + self.proj = comfy.ops.Linear(dim_in, dim_out * 2, dtype=dtype) def forward(self, x): x, gate = self.proj(x).chunk(2, dim=-1) @@ -61,19 +61,19 @@ class GEGLU(nn.Module): class FeedForward(nn.Module): - def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0., dtype=None): super().__init__() inner_dim = int(dim * mult) dim_out = default(dim_out, dim) project_in = nn.Sequential( - comfy.ops.Linear(dim, inner_dim), + comfy.ops.Linear(dim, inner_dim, dtype=dtype), nn.GELU() - ) if not glu else GEGLU(dim, inner_dim) + ) if not glu else GEGLU(dim, inner_dim, dtype=dtype) self.net = nn.Sequential( project_in, nn.Dropout(dropout), - comfy.ops.Linear(inner_dim, dim_out) + comfy.ops.Linear(inner_dim, dim_out, dtype=dtype) ) def forward(self, x): @@ -89,8 +89,8 @@ def zero_module(module): return module -def Normalize(in_channels): - return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) +def Normalize(in_channels, dtype=None): + return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype) class SpatialSelfAttention(nn.Module): @@ -147,7 +147,7 @@ class SpatialSelfAttention(nn.Module): class CrossAttentionBirchSan(nn.Module): - def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.): + def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None): super().__init__() inner_dim = dim_head * heads context_dim = default(context_dim, query_dim) @@ -155,12 +155,12 @@ class CrossAttentionBirchSan(nn.Module): self.scale = dim_head ** -0.5 self.heads = heads - self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False) - self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False) - self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False) + self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype) + self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) + self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) self.to_out = nn.Sequential( - comfy.ops.Linear(inner_dim, query_dim), + comfy.ops.Linear(inner_dim, query_dim, dtype=dtype), nn.Dropout(dropout) ) @@ -244,7 +244,7 @@ class CrossAttentionBirchSan(nn.Module): class CrossAttentionDoggettx(nn.Module): - def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.): + def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None): super().__init__() inner_dim = dim_head * heads context_dim = default(context_dim, query_dim) @@ -252,12 +252,12 @@ class CrossAttentionDoggettx(nn.Module): self.scale = dim_head ** -0.5 self.heads = heads - self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False) - self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False) - self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False) + self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype) + self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) + self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) self.to_out = nn.Sequential( - comfy.ops.Linear(inner_dim, query_dim), + comfy.ops.Linear(inner_dim, query_dim, dtype=dtype), nn.Dropout(dropout) ) @@ -342,7 +342,7 @@ class CrossAttentionDoggettx(nn.Module): return self.to_out(r2) class CrossAttention(nn.Module): - def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.): + def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None): super().__init__() inner_dim = dim_head * heads context_dim = default(context_dim, query_dim) @@ -350,12 +350,12 @@ class CrossAttention(nn.Module): self.scale = dim_head ** -0.5 self.heads = heads - self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False) - self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False) - self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False) + self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype) + self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) + self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) self.to_out = nn.Sequential( - comfy.ops.Linear(inner_dim, query_dim), + comfy.ops.Linear(inner_dim, query_dim, dtype=dtype), nn.Dropout(dropout) ) @@ -398,7 +398,7 @@ class CrossAttention(nn.Module): class MemoryEfficientCrossAttention(nn.Module): # https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223 - def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): + def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0, dtype=None): super().__init__() print(f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, context_dim is {context_dim} and using " f"{heads} heads.") @@ -408,11 +408,11 @@ class MemoryEfficientCrossAttention(nn.Module): self.heads = heads self.dim_head = dim_head - self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False) - self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False) - self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False) + self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype) + self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) + self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) - self.to_out = nn.Sequential(comfy.ops.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + self.to_out = nn.Sequential(comfy.ops.Linear(inner_dim, query_dim, dtype=dtype), nn.Dropout(dropout)) self.attention_op: Optional[Any] = None def forward(self, x, context=None, value=None, mask=None): @@ -449,7 +449,7 @@ class MemoryEfficientCrossAttention(nn.Module): return self.to_out(out) class CrossAttentionPytorch(nn.Module): - def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.): + def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None): super().__init__() inner_dim = dim_head * heads context_dim = default(context_dim, query_dim) @@ -457,11 +457,11 @@ class CrossAttentionPytorch(nn.Module): self.heads = heads self.dim_head = dim_head - self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False) - self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False) - self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False) + self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype) + self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) + self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype) - self.to_out = nn.Sequential(comfy.ops.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + self.to_out = nn.Sequential(comfy.ops.Linear(inner_dim, query_dim, dtype=dtype), nn.Dropout(dropout)) self.attention_op: Optional[Any] = None def forward(self, x, context=None, value=None, mask=None): @@ -507,17 +507,17 @@ else: class BasicTransformerBlock(nn.Module): def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True, - disable_self_attn=False): + disable_self_attn=False, dtype=None): super().__init__() self.disable_self_attn = disable_self_attn self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout, - context_dim=context_dim if self.disable_self_attn else None) # is a self-attention if not self.disable_self_attn - self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) + context_dim=context_dim if self.disable_self_attn else None, dtype=dtype) # is a self-attention if not self.disable_self_attn + self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff, dtype=dtype) self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim, - heads=n_heads, dim_head=d_head, dropout=dropout) # is self-attn if context is none - self.norm1 = nn.LayerNorm(dim) - self.norm2 = nn.LayerNorm(dim) - self.norm3 = nn.LayerNorm(dim) + heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype) # is self-attn if context is none + self.norm1 = nn.LayerNorm(dim, dtype=dtype) + self.norm2 = nn.LayerNorm(dim, dtype=dtype) + self.norm3 = nn.LayerNorm(dim, dtype=dtype) self.checkpoint = checkpoint def forward(self, x, context=None, transformer_options={}): @@ -588,34 +588,34 @@ class SpatialTransformer(nn.Module): def __init__(self, in_channels, n_heads, d_head, depth=1, dropout=0., context_dim=None, disable_self_attn=False, use_linear=False, - use_checkpoint=True): + use_checkpoint=True, dtype=None): super().__init__() if exists(context_dim) and not isinstance(context_dim, list): context_dim = [context_dim] self.in_channels = in_channels inner_dim = n_heads * d_head - self.norm = Normalize(in_channels) + self.norm = Normalize(in_channels, dtype=dtype) if not use_linear: self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, - padding=0) + padding=0, dtype=dtype) else: - self.proj_in = comfy.ops.Linear(in_channels, inner_dim) + self.proj_in = comfy.ops.Linear(in_channels, inner_dim, dtype=dtype) self.transformer_blocks = nn.ModuleList( [BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim[d], - disable_self_attn=disable_self_attn, checkpoint=use_checkpoint) + disable_self_attn=disable_self_attn, checkpoint=use_checkpoint, dtype=dtype) for d in range(depth)] ) if not use_linear: self.proj_out = nn.Conv2d(inner_dim,in_channels, kernel_size=1, stride=1, - padding=0) + padding=0, dtype=dtype) else: - self.proj_out = comfy.ops.Linear(in_channels, inner_dim) + self.proj_out = comfy.ops.Linear(in_channels, inner_dim, dtype=dtype) self.use_linear = use_linear def forward(self, x, context=None, transformer_options={}): diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.py index 5aef23f33..e170f6779 100644 --- a/comfy/ldm/modules/diffusionmodules/openaimodel.py +++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py @@ -111,14 +111,14 @@ class Upsample(nn.Module): upsampling occurs in the inner-two dimensions. """ - def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1): + def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None): super().__init__() self.channels = channels self.out_channels = out_channels or channels self.use_conv = use_conv self.dims = dims if use_conv: - self.conv = conv_nd(dims, self.channels, self.out_channels, 3, padding=padding) + self.conv = conv_nd(dims, self.channels, self.out_channels, 3, padding=padding, dtype=dtype) def forward(self, x, output_shape=None): assert x.shape[1] == self.channels @@ -160,7 +160,7 @@ class Downsample(nn.Module): downsampling occurs in the inner-two dimensions. """ - def __init__(self, channels, use_conv, dims=2, out_channels=None,padding=1): + def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None): super().__init__() self.channels = channels self.out_channels = out_channels or channels @@ -169,7 +169,7 @@ class Downsample(nn.Module): stride = 2 if dims != 3 else (1, 2, 2) if use_conv: self.op = conv_nd( - dims, self.channels, self.out_channels, 3, stride=stride, padding=padding + dims, self.channels, self.out_channels, 3, stride=stride, padding=padding, dtype=dtype ) else: assert self.channels == self.out_channels @@ -208,6 +208,7 @@ class ResBlock(TimestepBlock): use_checkpoint=False, up=False, down=False, + dtype=None ): super().__init__() self.channels = channels @@ -219,19 +220,19 @@ class ResBlock(TimestepBlock): self.use_scale_shift_norm = use_scale_shift_norm self.in_layers = nn.Sequential( - normalization(channels), + normalization(channels, dtype=dtype), nn.SiLU(), - conv_nd(dims, channels, self.out_channels, 3, padding=1), + conv_nd(dims, channels, self.out_channels, 3, padding=1, dtype=dtype), ) self.updown = up or down if up: - self.h_upd = Upsample(channels, False, dims) - self.x_upd = Upsample(channels, False, dims) + self.h_upd = Upsample(channels, False, dims, dtype=dtype) + self.x_upd = Upsample(channels, False, dims, dtype=dtype) elif down: - self.h_upd = Downsample(channels, False, dims) - self.x_upd = Downsample(channels, False, dims) + self.h_upd = Downsample(channels, False, dims, dtype=dtype) + self.x_upd = Downsample(channels, False, dims, dtype=dtype) else: self.h_upd = self.x_upd = nn.Identity() @@ -239,15 +240,15 @@ class ResBlock(TimestepBlock): nn.SiLU(), linear( emb_channels, - 2 * self.out_channels if use_scale_shift_norm else self.out_channels, + 2 * self.out_channels if use_scale_shift_norm else self.out_channels, dtype=dtype ), ) self.out_layers = nn.Sequential( - normalization(self.out_channels), + normalization(self.out_channels, dtype=dtype), nn.SiLU(), nn.Dropout(p=dropout), zero_module( - conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1) + conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, dtype=dtype) ), ) @@ -255,10 +256,10 @@ class ResBlock(TimestepBlock): self.skip_connection = nn.Identity() elif use_conv: self.skip_connection = conv_nd( - dims, channels, self.out_channels, 3, padding=1 + dims, channels, self.out_channels, 3, padding=1, dtype=dtype ) else: - self.skip_connection = conv_nd(dims, channels, self.out_channels, 1) + self.skip_connection = conv_nd(dims, channels, self.out_channels, 1, dtype=dtype) def forward(self, x, emb): """ @@ -558,9 +559,9 @@ class UNetModel(nn.Module): time_embed_dim = model_channels * 4 self.time_embed = nn.Sequential( - linear(model_channels, time_embed_dim), + linear(model_channels, time_embed_dim, dtype=self.dtype), nn.SiLU(), - linear(time_embed_dim, time_embed_dim), + linear(time_embed_dim, time_embed_dim, dtype=self.dtype), ) if self.num_classes is not None: @@ -573,9 +574,9 @@ class UNetModel(nn.Module): assert adm_in_channels is not None self.label_emb = nn.Sequential( nn.Sequential( - linear(adm_in_channels, time_embed_dim), + linear(adm_in_channels, time_embed_dim, dtype=self.dtype), nn.SiLU(), - linear(time_embed_dim, time_embed_dim), + linear(time_embed_dim, time_embed_dim, dtype=self.dtype), ) ) else: @@ -584,7 +585,7 @@ class UNetModel(nn.Module): self.input_blocks = nn.ModuleList( [ TimestepEmbedSequential( - conv_nd(dims, in_channels, model_channels, 3, padding=1) + conv_nd(dims, in_channels, model_channels, 3, padding=1, dtype=self.dtype) ) ] ) @@ -603,6 +604,7 @@ class UNetModel(nn.Module): dims=dims, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype ) ] ch = mult * model_channels @@ -631,7 +633,7 @@ class UNetModel(nn.Module): ) if not use_spatial_transformer else SpatialTransformer( ch, num_heads, dim_head, depth=transformer_depth, context_dim=context_dim, disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint + use_checkpoint=use_checkpoint, dtype=self.dtype ) ) self.input_blocks.append(TimestepEmbedSequential(*layers)) @@ -650,10 +652,11 @@ class UNetModel(nn.Module): use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, down=True, + dtype=self.dtype ) if resblock_updown else Downsample( - ch, conv_resample, dims=dims, out_channels=out_ch + ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype ) ) ) @@ -678,6 +681,7 @@ class UNetModel(nn.Module): dims=dims, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype ), AttentionBlock( ch, @@ -688,7 +692,7 @@ class UNetModel(nn.Module): ) if not use_spatial_transformer else SpatialTransformer( # always uses a self-attn ch, num_heads, dim_head, depth=transformer_depth, context_dim=context_dim, disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint + use_checkpoint=use_checkpoint, dtype=self.dtype ), ResBlock( ch, @@ -697,6 +701,7 @@ class UNetModel(nn.Module): dims=dims, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype ), ) self._feature_size += ch @@ -714,6 +719,7 @@ class UNetModel(nn.Module): dims=dims, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype ) ] ch = model_channels * mult @@ -742,7 +748,7 @@ class UNetModel(nn.Module): ) if not use_spatial_transformer else SpatialTransformer( ch, num_heads, dim_head, depth=transformer_depth, context_dim=context_dim, disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint + use_checkpoint=use_checkpoint, dtype=self.dtype ) ) if level and i == self.num_res_blocks[level]: @@ -757,18 +763,19 @@ class UNetModel(nn.Module): use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, up=True, + dtype=self.dtype ) if resblock_updown - else Upsample(ch, conv_resample, dims=dims, out_channels=out_ch) + else Upsample(ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype) ) ds //= 2 self.output_blocks.append(TimestepEmbedSequential(*layers)) self._feature_size += ch self.out = nn.Sequential( - normalization(ch), + normalization(ch, dtype=self.dtype), nn.SiLU(), - zero_module(conv_nd(dims, model_channels, out_channels, 3, padding=1)), + zero_module(conv_nd(dims, model_channels, out_channels, 3, padding=1, dtype=self.dtype)), ) if self.predict_codebook_ids: self.id_predictor = nn.Sequential( diff --git a/comfy/ldm/modules/diffusionmodules/util.py b/comfy/ldm/modules/diffusionmodules/util.py index d6a4778e4..d890c8044 100644 --- a/comfy/ldm/modules/diffusionmodules/util.py +++ b/comfy/ldm/modules/diffusionmodules/util.py @@ -206,13 +206,13 @@ def mean_flat(tensor): return tensor.mean(dim=list(range(1, len(tensor.shape)))) -def normalization(channels): +def normalization(channels, dtype=None): """ Make a standard normalization layer. :param channels: number of input channels. :return: an nn.Module for normalization. """ - return GroupNorm32(32, channels) + return GroupNorm32(32, channels, dtype=dtype) # PyTorch 1.7 has SiLU, but we support PyTorch 1.5. diff --git a/comfy/model_management.py b/comfy/model_management.py index 1a8a1be17..d64dce187 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -151,7 +151,7 @@ if args.lowvram: lowvram_available = True elif args.novram: set_vram_to = VRAMState.NO_VRAM -elif args.highvram: +elif args.highvram or args.gpu_only: vram_state = VRAMState.HIGH_VRAM FORCE_FP32 = False @@ -307,6 +307,12 @@ def unload_if_low_vram(model): return model.cpu() return model +def text_encoder_device(): + if args.gpu_only: + return get_torch_device() + else: + return torch.device("cpu") + def get_autocast_device(dev): if hasattr(dev, 'type'): return dev.type diff --git a/comfy/sd.py b/comfy/sd.py index db04e0426..24806dd01 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -467,7 +467,11 @@ class CLIP: clip = sd1_clip.SD1ClipModel tokenizer = sd1_clip.SD1Tokenizer + self.device = model_management.text_encoder_device() + params["device"] = self.device self.cond_stage_model = clip(**(params)) + self.cond_stage_model = self.cond_stage_model.to(self.device) + self.tokenizer = tokenizer(embedding_directory=embedding_directory) self.patcher = ModelPatcher(self.cond_stage_model) self.layer_idx = None diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index c2d4df092..fa6d22dcb 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -20,7 +20,7 @@ class ClipTokenWeightEncoder: output += [z] if (len(output) == 0): return self.encode(self.empty_tokens) - return torch.cat(output, dim=-2) + return torch.cat(output, dim=-2).cpu() class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder): """Uses the CLIP transformer encoder for text (from huggingface)""" diff --git a/server.py b/server.py index 300221f6c..f385cefb8 100644 --- a/server.py +++ b/server.py @@ -30,6 +30,11 @@ import comfy.model_management class BinaryEventTypes: PREVIEW_IMAGE = 1 +async def send_socket_catch_exception(function, message): + try: + await function(message) + except (aiohttp.ClientError, aiohttp.ClientPayloadError, ConnectionResetError) as err: + print("send error:", err) @web.middleware async def cache_control(request: web.Request, handler): @@ -487,18 +492,18 @@ class PromptServer(): if sid is None: for ws in self.sockets.values(): - await ws.send_bytes(message) + await send_socket_catch_exception(ws.send_bytes, message) elif sid in self.sockets: - await self.sockets[sid].send_bytes(message) + await send_socket_catch_exception(self.sockets[sid].send_bytes, message) async def send_json(self, event, data, sid=None): message = {"type": event, "data": data} if sid is None: for ws in self.sockets.values(): - await ws.send_json(message) + await send_socket_catch_exception(ws.send_json, message) elif sid in self.sockets: - await self.sockets[sid].send_json(message) + await send_socket_catch_exception(self.sockets[sid].send_json, message) def send_sync(self, event, data, sid=None): self.loop.call_soon_threadsafe(