From 4201181b35402e0a992b861f8d2f0e0b267f52fa Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 9 Apr 2024 04:25:45 -0400 Subject: [PATCH 01/25] Add ModelMergeSD1, ModelMergeSD2 and ModelMergeSDXL. --- .../nodes_model_merging_model_specific.py | 60 +++++++++++++++++++ nodes.py | 1 + 2 files changed, 61 insertions(+) create mode 100644 comfy_extras/nodes_model_merging_model_specific.py diff --git a/comfy_extras/nodes_model_merging_model_specific.py b/comfy_extras/nodes_model_merging_model_specific.py new file mode 100644 index 000000000..f2d008d8b --- /dev/null +++ b/comfy_extras/nodes_model_merging_model_specific.py @@ -0,0 +1,60 @@ +import comfy_extras.nodes_model_merging + +class ModelMergeSD1(comfy_extras.nodes_model_merging.ModelMergeBlocks): + CATEGORY = "advanced/model_merging/model_specific" + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["time_embed."] = argument + arg_dict["label_emb."] = argument + + for i in range(12): + arg_dict["input_blocks.{}.".format(i)] = argument + + for i in range(3): + arg_dict["middle_block.{}.".format(i)] = argument + + for i in range(12): + arg_dict["output_blocks.{}.".format(i)] = argument + + arg_dict["out."] = argument + + return {"required": arg_dict} + + +class ModelMergeSDXL(comfy_extras.nodes_model_merging.ModelMergeBlocks): + CATEGORY = "advanced/model_merging/model_specific" + + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["time_embed."] = argument + arg_dict["label_emb."] = argument + + for i in range(9): + arg_dict["input_blocks.{}".format(i)] = argument + + for i in range(3): + arg_dict["middle_block.{}".format(i)] = argument + + for i in range(9): + arg_dict["output_blocks.{}".format(i)] = argument + + arg_dict["out."] = argument + + return {"required": arg_dict} + + +NODE_CLASS_MAPPINGS = { + "ModelMergeSD1": ModelMergeSD1, + "ModelMergeSD2": ModelMergeSD1, #SD1 and SD2 have the same blocks + "ModelMergeSDXL": ModelMergeSDXL, +} diff --git a/nodes.py b/nodes.py index a1baa98a6..78e0cf116 100644 --- a/nodes.py +++ b/nodes.py @@ -1941,6 +1941,7 @@ def init_custom_nodes(): "nodes_stable_cascade.py", "nodes_differential_diffusion.py", "nodes_ip2p.py", + "nodes_model_merging_model_specific.py", ] import_failed = [] From 831511a1eecbe271e302f2f2053f285f00614180 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 9 Apr 2024 23:20:43 -0400 Subject: [PATCH 02/25] Fix issue with sampling_settings persisting across models. --- comfy/supported_models_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.py index 6196daabf..cf7cdff34 100644 --- a/comfy/supported_models_base.py +++ b/comfy/supported_models_base.py @@ -47,7 +47,8 @@ class BASE: return self.unet_config["in_channels"] > 4 def __init__(self, unet_config): - self.unet_config = unet_config + self.unet_config = unet_config.copy() + self.sampling_settings = self.sampling_settings.copy() self.latent_format = self.latent_format() for x in self.unet_extra_config: self.unet_config[x] = self.unet_extra_config[x] From fd7c63668080f8c2eb56edffeef028aa3932e0fe Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 10 Apr 2024 20:29:35 -0400 Subject: [PATCH 03/25] Add an AddNoise node to add noise depending on the sigma. --- comfy_extras/nodes_custom_sampler.py | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py index 1971f2c57..06238f892 100644 --- a/comfy_extras/nodes_custom_sampler.py +++ b/comfy_extras/nodes_custom_sampler.py @@ -538,6 +538,52 @@ class SamplerCustomAdvanced: out_denoised = out return (out, out_denoised) +class AddNoise: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model": ("MODEL",), + "noise": ("NOISE", ), + "sigmas": ("SIGMAS", ), + "latent_image": ("LATENT", ), + } + } + + RETURN_TYPES = ("LATENT",) + + FUNCTION = "add_noise" + + CATEGORY = "_for_testing/custom_sampling/noise" + + def add_noise(self, model, noise, sigmas, latent_image): + if len(sigmas) == 0: + return latent_image + + latent = latent_image + latent_image = latent["samples"] + + noisy = noise.generate_noise(latent) + + model_sampling = model.get_model_object("model_sampling") + process_latent_out = model.get_model_object("process_latent_out") + process_latent_in = model.get_model_object("process_latent_in") + + if len(sigmas) > 1: + scale = torch.abs(sigmas[0] - sigmas[-1]) + else: + scale = sigmas[0] + + if torch.count_nonzero(latent_image) > 0: #Don't shift the empty latent image. + latent_image = process_latent_in(latent_image) + noisy = model_sampling.noise_scaling(scale, noisy, latent_image) + noisy = process_latent_out(noisy) + noisy = torch.nan_to_num(noisy, nan=0.0, posinf=0.0, neginf=0.0) + + out = latent.copy() + out["samples"] = noisy + return (out,) + + NODE_CLASS_MAPPINGS = { "SamplerCustom": SamplerCustom, "BasicScheduler": BasicScheduler, @@ -561,5 +607,6 @@ NODE_CLASS_MAPPINGS = { "BasicGuider": BasicGuider, "RandomNoise": RandomNoise, "DisableNoise": DisableNoise, + "AddNoise": AddNoise, "SamplerCustomAdvanced": SamplerCustomAdvanced, } From 4bd7d55b9028d79829a645edfe8259f7b7a049c0 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 11 Apr 2024 22:43:05 -0400 Subject: [PATCH 04/25] Add some colors to SamplerCustom links. If you don't like them I am open to a PR. --- web/extensions/core/colorPalette.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/extensions/core/colorPalette.js b/web/extensions/core/colorPalette.js index b8d83613d..02546782f 100644 --- a/web/extensions/core/colorPalette.js +++ b/web/extensions/core/colorPalette.js @@ -20,6 +20,10 @@ const colorPalettes = { "MODEL": "#B39DDB", // light lavender-purple "STYLE_MODEL": "#C2FFAE", // light green-yellow "VAE": "#FF6E6E", // bright red + "NOISE": "#B0B0B0", // gray + "GUIDER": "#66FFFF", // cyan + "SAMPLER": "#ECB4B4", // very soft red + "SIGMAS": "#CDFFCD", // soft lime green "TAESD": "#DCC274", // cheesecake }, "litegraph_base": { From 2bef134ebfe38dd1ce4d25eefe933c6748f0f35a Mon Sep 17 00:00:00 2001 From: NyaamZ <43065065+NyaamZ@users.noreply.github.com> Date: Sat, 13 Apr 2024 06:02:17 +0900 Subject: [PATCH 05/25] change Convert.. input (#3246) --- web/extensions/core/widgetInputs.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index 23f51d812..e6db9f71a 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -292,12 +292,23 @@ app.registerExtension({ } } } + + //Convert.. main menu if (toInput.length) { - options.push(...toInput, null); + options.push({ + content: `Convert input to 🔘..`, + submenu: { + options: toInput, + }, + }); } - if (toWidget.length) { - options.push(...toWidget, null); + options.push({ + content: `Convert 🔘 to widget..`, + submenu: { + options: toWidget, + }, + }); } } From 0256e7f7699271b114593198278c5590a2e690d4 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 12 Apr 2024 20:01:34 -0400 Subject: [PATCH 06/25] Fix tests. --- tests-ui/utils/ezgraph.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests-ui/utils/ezgraph.js b/tests-ui/utils/ezgraph.js index 8a55246ee..8bf8c5d8c 100644 --- a/tests-ui/utils/ezgraph.js +++ b/tests-ui/utils/ezgraph.js @@ -204,13 +204,17 @@ export class EzWidget { convertToWidget() { if (!this.isConvertedToInput) throw new Error(`Widget ${this.widget.name} cannot be converted as it is already a widget.`); - this.node.menu[`Convert ${this.widget.name} to widget`].call(); + var menu = this.node.menu["Convert 🔘 to widget.."].item.submenu.options; + var index = menu.findIndex(a => a.content == `Convert ${this.widget.name} to widget`); + menu[index].callback.call(); } convertToInput() { if (this.isConvertedToInput) throw new Error(`Widget ${this.widget.name} cannot be converted as it is already an input.`); - this.node.menu[`Convert ${this.widget.name} to input`].call(); + var menu = this.node.menu["Convert input to 🔘.."].item.submenu.options; + var index = menu.findIndex(a => a.content == `Convert ${this.widget.name} to input`); + menu[index].callback.call(); } } From 58812ab8ca601cc2dd9dbe64c1f3ffd4929fd0ca Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 12 Apr 2024 22:12:35 -0400 Subject: [PATCH 07/25] Support SDXS 512 model. --- comfy/model_detection.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/comfy/model_detection.py b/comfy/model_detection.py index 795af828a..23358a2c0 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -357,7 +357,14 @@ def unet_config_from_diffusers_unet(state_dict, dtype=None): 'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1], 'use_temporal_attention': False, 'use_temporal_resblock': False, 'disable_self_attentions': [True, False, False]} - supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega, KOALA_700M, KOALA_1B, SD09_XS, SDXL_diffusers_ip2p] + SD_XS = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False, + 'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': [1, 1, 1], + 'transformer_depth': [0, 1, 1], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': -2, 'use_linear_in_transformer': False, + 'context_dim': 768, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 1, 1, 1, 1], + 'use_temporal_attention': False, 'use_temporal_resblock': False} + + + supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega, KOALA_700M, KOALA_1B, SD09_XS, SD_XS, SDXL_diffusers_ip2p] for unet_config in supported_models: matches = True From 744ac944db24795881cf41fc4ea015e5ea07a444 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 13 Apr 2024 16:12:09 -0400 Subject: [PATCH 08/25] Don't make dynamicPrompts the default on multiline string inputs. This should be less confusing to those who want to use multiline input without them. --- comfy_extras/nodes_clip_sdxl.py | 6 +++--- comfy_extras/nodes_cond.py | 2 +- comfy_extras/nodes_photomaker.py | 2 +- nodes.py | 4 ++-- web/extensions/core/dynamicPrompts.js | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/comfy_extras/nodes_clip_sdxl.py b/comfy_extras/nodes_clip_sdxl.py index dcf8859fa..3087b917b 100644 --- a/comfy_extras/nodes_clip_sdxl.py +++ b/comfy_extras/nodes_clip_sdxl.py @@ -8,7 +8,7 @@ class CLIPTextEncodeSDXLRefiner: "ascore": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 1000.0, "step": 0.01}), "width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), "height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), - "text": ("STRING", {"multiline": True}), "clip": ("CLIP", ), + "text": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ), }} RETURN_TYPES = ("CONDITIONING",) FUNCTION = "encode" @@ -30,8 +30,8 @@ class CLIPTextEncodeSDXL: "crop_h": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION}), "target_width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), "target_height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), - "text_g": ("STRING", {"multiline": True, "default": "CLIP_G"}), "clip": ("CLIP", ), - "text_l": ("STRING", {"multiline": True, "default": "CLIP_L"}), "clip": ("CLIP", ), + "text_g": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ), + "text_l": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ), }} RETURN_TYPES = ("CONDITIONING",) FUNCTION = "encode" diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.py index 646fefa17..4c3a1d5bf 100644 --- a/comfy_extras/nodes_cond.py +++ b/comfy_extras/nodes_cond.py @@ -3,7 +3,7 @@ class CLIPTextEncodeControlnet: @classmethod def INPUT_TYPES(s): - return {"required": {"clip": ("CLIP", ), "conditioning": ("CONDITIONING", ), "text": ("STRING", {"multiline": True})}} + return {"required": {"clip": ("CLIP", ), "conditioning": ("CONDITIONING", ), "text": ("STRING", {"multiline": True, "dynamicPrompts": True})}} RETURN_TYPES = ("CONDITIONING",) FUNCTION = "encode" diff --git a/comfy_extras/nodes_photomaker.py b/comfy_extras/nodes_photomaker.py index 90130142b..29d127d74 100644 --- a/comfy_extras/nodes_photomaker.py +++ b/comfy_extras/nodes_photomaker.py @@ -141,7 +141,7 @@ class PhotoMakerEncode: return {"required": { "photomaker": ("PHOTOMAKER",), "image": ("IMAGE",), "clip": ("CLIP", ), - "text": ("STRING", {"multiline": True, "default": "photograph of photomaker"}), + "text": ("STRING", {"multiline": True, "dynamicPrompts": True, "default": "photograph of photomaker"}), }} RETURN_TYPES = ("CONDITIONING",) diff --git a/nodes.py b/nodes.py index 78e0cf116..fda073cfc 100644 --- a/nodes.py +++ b/nodes.py @@ -47,7 +47,7 @@ MAX_RESOLUTION=16384 class CLIPTextEncode: @classmethod def INPUT_TYPES(s): - return {"required": {"text": ("STRING", {"multiline": True}), "clip": ("CLIP", )}} + return {"required": {"text": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", )}} RETURN_TYPES = ("CONDITIONING",) FUNCTION = "encode" @@ -966,7 +966,7 @@ class GLIGENTextBoxApply: return {"required": {"conditioning_to": ("CONDITIONING", ), "clip": ("CLIP", ), "gligen_textbox_model": ("GLIGEN", ), - "text": ("STRING", {"multiline": True}), + "text": ("STRING", {"multiline": True, "dynamicPrompts": True}), "width": ("INT", {"default": 64, "min": 8, "max": MAX_RESOLUTION, "step": 8}), "height": ("INT", {"default": 64, "min": 8, "max": MAX_RESOLUTION, "step": 8}), "x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}), diff --git a/web/extensions/core/dynamicPrompts.js b/web/extensions/core/dynamicPrompts.js index 599a9e685..7417361ba 100644 --- a/web/extensions/core/dynamicPrompts.js +++ b/web/extensions/core/dynamicPrompts.js @@ -17,7 +17,7 @@ app.registerExtension({ // Locate dynamic prompt text widgets // Include any widgets with dynamicPrompts set to true, and customtext const widgets = node.widgets.filter( - (n) => (n.type === "customtext" && n.dynamicPrompts !== false) || n.dynamicPrompts + (n) => n.dynamicPrompts ); for (const widget of widgets) { // Override the serialization of the value to resolve dynamic prompts for all widgets supporting it in this node From 258dbc06c34ebf723ae17fa23207407e2e05fad1 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 14 Apr 2024 12:08:58 -0400 Subject: [PATCH 09/25] Fix some memory related issues. --- comfy/model_management.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 310ec2537..537df41ed 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -617,7 +617,8 @@ def supports_dtype(device, dtype): #TODO def device_supports_non_blocking(device): if is_device_mps(device): return False #pytorch bug? mps doesn't support non blocking - return True + return False + # return True #TODO: figure out why this causes issues def cast_to_device(tensor, device, dtype, copy=False): device_supports_cast = False From 719fb2c81d716ce8edd7f1bdc7804ae160a71d3a Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 14 Apr 2024 23:34:25 -0400 Subject: [PATCH 10/25] Add basic PAG node. --- comfy/model_patcher.py | 31 +++++++++++++++------- comfy_extras/nodes_pag.py | 56 +++++++++++++++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 comfy_extras/nodes_pag.py diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 657ddf843..cf51c4ad8 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -18,6 +18,26 @@ def apply_weight_decompose(dora_scale, weight): return weight * (dora_scale / weight_norm) +def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None): + to = model_options["transformer_options"].copy() + + if "patches_replace" not in to: + to["patches_replace"] = {} + else: + to["patches_replace"] = to["patches_replace"].copy() + + if name not in to["patches_replace"]: + to["patches_replace"][name] = {} + else: + to["patches_replace"][name] = to["patches_replace"][name].copy() + + if transformer_index is not None: + block = (block_name, number, transformer_index) + else: + block = (block_name, number) + to["patches_replace"][name][block] = patch + model_options["transformer_options"] = to + return model_options class ModelPatcher: def __init__(self, model, load_device, offload_device, size=0, current_device=None, weight_inplace_update=False): @@ -109,16 +129,7 @@ class ModelPatcher: to["patches"][name] = to["patches"].get(name, []) + [patch] def set_model_patch_replace(self, patch, name, block_name, number, transformer_index=None): - to = self.model_options["transformer_options"] - if "patches_replace" not in to: - to["patches_replace"] = {} - if name not in to["patches_replace"]: - to["patches_replace"][name] = {} - if transformer_index is not None: - block = (block_name, number, transformer_index) - else: - block = (block_name, number) - to["patches_replace"][name][block] = patch + self.model_options = set_model_options_patch_replace(self.model_options, patch, name, block_name, number, transformer_index=transformer_index) def set_model_attn1_patch(self, patch): self.set_model_patch(patch, "attn1_patch") diff --git a/comfy_extras/nodes_pag.py b/comfy_extras/nodes_pag.py new file mode 100644 index 000000000..c48a3958d --- /dev/null +++ b/comfy_extras/nodes_pag.py @@ -0,0 +1,56 @@ +#Modified/simplified version of the node from: https://github.com/pamparamm/sd-perturbed-attention +#If you want the one with more options see the above repo. + +#My modified one here is more basic but has less chances of breaking with ComfyUI updates. + +import comfy.model_patcher +import comfy.samplers + +class PerturbedAttentionGuidance: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL",), + "scale": ("FLOAT", {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + } + } + + RETURN_TYPES = ("MODEL",) + FUNCTION = "patch" + + CATEGORY = "_for_testing" + + def patch(self, model, scale): + unet_block = "middle" + unet_block_id = 0 + m = model.clone() + + def perturbed_attention(q, k, v, extra_options, mask=None): + return v + + def post_cfg_function(args): + model = args["model"] + cond_pred = args["cond_denoised"] + cond = args["cond"] + cfg_result = args["denoised"] + sigma = args["sigma"] + model_options = args["model_options"].copy() + x = args["input"] + + if scale == 0: + return cfg_result + + # Replace Self-attention with PAG + model_options = comfy.model_patcher.set_model_options_patch_replace(model_options, perturbed_attention, "attn1", unet_block, unet_block_id) + (pag,) = comfy.samplers.calc_cond_batch(model, [cond], x, sigma, model_options) + + return cfg_result + (cond_pred - pag) * scale + + m.set_model_sampler_post_cfg_function(post_cfg_function, disable_cfg1_optimization=True) + + return (m,) + +NODE_CLASS_MAPPINGS = { + "PerturbedAttentionGuidance": PerturbedAttentionGuidance, +} diff --git a/nodes.py b/nodes.py index fda073cfc..ea1e32030 100644 --- a/nodes.py +++ b/nodes.py @@ -1942,6 +1942,7 @@ def init_custom_nodes(): "nodes_differential_diffusion.py", "nodes_ip2p.py", "nodes_model_merging_model_specific.py", + "nodes_pag.py", ] import_failed = [] From 8903dce862616ddd937400ffe6a971b66409262f Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 15 Apr 2024 12:14:00 -0400 Subject: [PATCH 11/25] This can be removed since PAG doesn't use the uncond. --- comfy_extras/nodes_pag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_extras/nodes_pag.py b/comfy_extras/nodes_pag.py index c48a3958d..63f43fd62 100644 --- a/comfy_extras/nodes_pag.py +++ b/comfy_extras/nodes_pag.py @@ -47,7 +47,7 @@ class PerturbedAttentionGuidance: return cfg_result + (cond_pred - pag) * scale - m.set_model_sampler_post_cfg_function(post_cfg_function, disable_cfg1_optimization=True) + m.set_model_sampler_post_cfg_function(post_cfg_function) return (m,) From 45ec1cbe963055798765645c4f727122a7d3e35e Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 16 Apr 2024 02:57:01 -0400 Subject: [PATCH 12/25] Implement PerpNeg as a guider. --- comfy_extras/nodes_perpneg.py | 62 ++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/comfy_extras/nodes_perpneg.py b/comfy_extras/nodes_perpneg.py index 992e8ab62..306cf9cd0 100644 --- a/comfy_extras/nodes_perpneg.py +++ b/comfy_extras/nodes_perpneg.py @@ -3,9 +3,18 @@ import comfy.model_management import comfy.sampler_helpers import comfy.samplers import comfy.utils +import node_helpers +def perp_neg(x, noise_pred_pos, noise_pred_neg, noise_pred_nocond, neg_scale, cond_scale): + pos = noise_pred_pos - noise_pred_nocond + neg = noise_pred_neg - noise_pred_nocond -#TODO: This node should be removed and replaced with one that uses the new Guider/SamplerCustomAdvanced. + perp = neg - ((torch.mul(neg, pos).sum())/(torch.norm(pos)**2)) * pos + perp_neg = perp * neg_scale + cfg_result = noise_pred_nocond + cond_scale*(pos - perp_neg) + return cfg_result + +#TODO: This node should be removed, it has been replaced with PerpNegGuider class PerpNeg: @classmethod def INPUT_TYPES(s): @@ -34,12 +43,7 @@ class PerpNeg: (noise_pred_nocond,) = comfy.samplers.calc_cond_batch(model, [nocond_processed], x, sigma, model_options) - pos = noise_pred_pos - noise_pred_nocond - neg = noise_pred_neg - noise_pred_nocond - perp = neg - ((torch.mul(neg, pos).sum())/(torch.norm(pos)**2)) * pos - perp_neg = perp * neg_scale - cfg_result = noise_pred_nocond + cond_scale*(pos - perp_neg) - cfg_result = x - cfg_result + cfg_result = x - perp_neg(x, noise_pred_pos, noise_pred_neg, noise_pred_nocond, neg_scale, cond_scale) return cfg_result m.set_model_sampler_cfg_function(cfg_function) @@ -47,10 +51,52 @@ class PerpNeg: return (m, ) +class Guider_PerpNeg(comfy.samplers.CFGGuider): + def set_conds(self, positive, negative, empty_negative_prompt): + empty_negative_prompt = node_helpers.conditioning_set_values(empty_negative_prompt, {"prompt_type": "negative"}) + self.inner_set_conds({"positive": positive, "empty_negative_prompt": empty_negative_prompt, "negative": negative}) + + def set_cfg(self, cfg, neg_scale): + self.cfg = cfg + self.neg_scale = neg_scale + + def predict_noise(self, x, timestep, model_options={}, seed=None): + positive_cond = self.conds.get("positive", None) + negative_cond = self.conds.get("negative", None) + empty_cond = self.conds.get("empty_negative_prompt", None) + + out = comfy.samplers.calc_cond_batch(self.inner_model, [negative_cond, positive_cond, empty_cond], x, timestep, model_options) + return perp_neg(x, out[1], out[0], out[2], self.neg_scale, self.cfg) + +class PerpNegGuider: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model": ("MODEL",), + "positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "empty_conditioning": ("CONDITIONING", ), + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}), + "neg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.01}), + } + } + + RETURN_TYPES = ("GUIDER",) + + FUNCTION = "get_guider" + CATEGORY = "_for_testing" + + def get_guider(self, model, positive, negative, empty_conditioning, cfg, neg_scale): + guider = Guider_PerpNeg(model) + guider.set_conds(positive, negative, empty_conditioning) + guider.set_cfg(cfg, neg_scale) + return (guider,) + NODE_CLASS_MAPPINGS = { "PerpNeg": PerpNeg, + "PerpNegGuider": PerpNegGuider, } NODE_DISPLAY_NAME_MAPPINGS = { - "PerpNeg": "Perp-Neg", + "PerpNeg": "Perp-Neg (DEPRECATED by PerpNegGuider)", } From abc69cab4560d0c50bc757de2695cf19fe1bfc3f Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 17 Apr 2024 12:28:05 -0400 Subject: [PATCH 13/25] Add a helpful warning for links that don't point anywhere. --- comfy_extras/nodes_canny.py | 1 - folder_paths.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/comfy_extras/nodes_canny.py b/comfy_extras/nodes_canny.py index 8138b5f73..fab2ab7ac 100644 --- a/comfy_extras/nodes_canny.py +++ b/comfy_extras/nodes_canny.py @@ -1,4 +1,3 @@ -#From https://github.com/kornia/kornia import math import torch diff --git a/folder_paths.py b/folder_paths.py index a84524d9f..3f9a32b4d 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -181,6 +181,8 @@ def get_full_path(folder_name, filename): full_path = os.path.join(x, filename) if os.path.isfile(full_path): return full_path + elif os.path.islink(full_path): + logging.warning("WARNING path {} exists but doesn't link anywhere, skipping.".format(full_path)) return None From 072e3bd2b542eaf7f757d898f2fac67de4dea099 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" <128333288+ltdrdata@users.noreply.github.com> Date: Thu, 18 Apr 2024 05:36:49 +0900 Subject: [PATCH 14/25] Fixed an issue where the main menu disappears intermittently as the coordinates become negative. (#3269) --- web/scripts/ui.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/scripts/ui.js b/web/scripts/ui.js index 5ca6214eb..d0fa46efb 100644 --- a/web/scripts/ui.js +++ b/web/scripts/ui.js @@ -90,12 +90,15 @@ function dragElement(dragEl, settings) { }).observe(dragEl); function ensureInBounds() { - if (dragEl.classList.contains("comfy-menu-manual-pos")) { + try { newPosX = Math.min(document.body.clientWidth - dragEl.clientWidth, Math.max(0, dragEl.offsetLeft)); newPosY = Math.min(document.body.clientHeight - dragEl.clientHeight, Math.max(0, dragEl.offsetTop)); positionElement(); } + catch(exception){ + // robust + } } function positionElement() { From d64e2174276c0f5a1db605af6f0331eb0c75b42d Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 17 Apr 2024 17:34:02 -0400 Subject: [PATCH 15/25] Fix annoying float issue causing the value to be rounded to above the max. --- web/scripts/widgets.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index 678b1b8ec..00c91914d 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -307,7 +307,9 @@ export const ComfyWidgets = { return { widget: node.addWidget(widgetType, inputName, val, function (v) { if (config.round) { - this.value = Math.round(v/config.round)*config.round; + this.value = Math.round((v + Number.EPSILON)/config.round)*config.round; + if (this.value > config.max) this.value = config.max; + if (this.value < config.min) this.value = config.min; } else { this.value = v; } From a88b0ebc2d2f933c94e42aa689c42e836eedaf3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B6rn=20L=C3=B6nnemark?= Date: Thu, 18 Apr 2024 22:41:23 +0200 Subject: [PATCH 16/25] Improve node input/widget conversion sub-menus (#3281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make input/widget conversion sub-menus optional * Improve input/widget conversion sub-menu text - Fix incorrect text for conversion from widget to input, previously it effectively said "convert input to input" - Use "input" instead of "🔘". The former is clearer and consistent with the rest of the application. - Use title case (consistent with the rest of the menu entries). - Strip the trailing periods. There is already a visual indicator for sub-menus, and no other sub-menus use trailing periods. --- tests-ui/utils/ezgraph.js | 4 +-- web/extensions/core/widgetInputs.js | 42 ++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/tests-ui/utils/ezgraph.js b/tests-ui/utils/ezgraph.js index 8bf8c5d8c..97be7aa72 100644 --- a/tests-ui/utils/ezgraph.js +++ b/tests-ui/utils/ezgraph.js @@ -204,7 +204,7 @@ export class EzWidget { convertToWidget() { if (!this.isConvertedToInput) throw new Error(`Widget ${this.widget.name} cannot be converted as it is already a widget.`); - var menu = this.node.menu["Convert 🔘 to widget.."].item.submenu.options; + var menu = this.node.menu["Convert Input to Widget"].item.submenu.options; var index = menu.findIndex(a => a.content == `Convert ${this.widget.name} to widget`); menu[index].callback.call(); } @@ -212,7 +212,7 @@ export class EzWidget { convertToInput() { if (this.isConvertedToInput) throw new Error(`Widget ${this.widget.name} cannot be converted as it is already an input.`); - var menu = this.node.menu["Convert input to 🔘.."].item.submenu.options; + var menu = this.node.menu["Convert Widget to Input"].item.submenu.options; var index = menu.findIndex(a => a.content == `Convert ${this.widget.name} to input`); menu[index].callback.call(); } diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index e6db9f71a..f1a1d22cd 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -256,8 +256,18 @@ export function mergeIfValid(output, config2, forceUpdate, recreateWidget, confi return { customConfig }; } +let useConversionSubmenusSetting; app.registerExtension({ name: "Comfy.WidgetInputs", + init() { + useConversionSubmenusSetting = app.ui.settings.addSetting({ + id: "Comfy.NodeInputConversionSubmenus", + name: "Node widget/input conversion sub-menus", + tooltip: "In the node context menu, place the entries that convert between input/widget in sub-menus.", + type: "boolean", + defaultValue: true, + }); + }, async beforeRegisterNodeDef(nodeType, nodeData, app) { // Add menu options to conver to/from widgets const origGetExtraMenuOptions = nodeType.prototype.getExtraMenuOptions; @@ -295,20 +305,28 @@ app.registerExtension({ //Convert.. main menu if (toInput.length) { - options.push({ - content: `Convert input to 🔘..`, - submenu: { - options: toInput, - }, - }); + if (useConversionSubmenusSetting.value) { + options.push({ + content: "Convert Widget to Input", + submenu: { + options: toInput, + }, + }); + } else { + options.push(...toInput, null); + } } if (toWidget.length) { - options.push({ - content: `Convert 🔘 to widget..`, - submenu: { - options: toWidget, - }, - }); + if (useConversionSubmenusSetting.value) { + options.push({ + content: "Convert Input to Widget", + submenu: { + options: toWidget, + }, + }); + } else { + options.push(...toWidget, null); + } } } From c59fe9f254d9e423aa3bb52ca4b6d8b4679bd11c Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 18 Apr 2024 21:05:33 -0400 Subject: [PATCH 17/25] Support VAE without quant_conv. --- comfy/sd.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 3919f4bfa..57dba0b44 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -214,12 +214,18 @@ class VAE: #default SD1.x/SD2.x VAE parameters ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0} - if 'encoder.down.2.downsample.conv.weight' not in sd: #Stable diffusion x4 upscaler VAE + if 'encoder.down.2.downsample.conv.weight' not in sd and 'decoder.up.3.upsample.conv.weight' not in sd: #Stable diffusion x4 upscaler VAE ddconfig['ch_mult'] = [1, 2, 4] self.downscale_ratio = 4 self.upscale_ratio = 4 - self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4) + self.latent_channels = ddconfig['z_channels'] = sd["decoder.conv_in.weight"].shape[1] + if 'quant_conv.weight' in sd: + self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4) + else: + self.first_stage_model = AutoencodingEngine(regularizer_config={'target': "comfy.ldm.models.autoencoder.DiagonalGaussianRegularizer"}, + encoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Encoder", 'params': ddconfig}, + decoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Decoder", 'params': ddconfig}) else: self.first_stage_model = AutoencoderKL(**(config['params'])) self.first_stage_model = self.first_stage_model.eval() From 5d08802f783a59d4fbdcfd5462f7edd9c6e0204d Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 19 Apr 2024 03:43:09 -0400 Subject: [PATCH 18/25] Sync some minor changes from the other repo. --- execution.py | 2 +- folder_paths.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/execution.py b/execution.py index 35bdb77a6..47d58b9d7 100644 --- a/execution.py +++ b/execution.py @@ -176,7 +176,7 @@ def recursive_execute(server, prompt, outputs, current_item, extra_data, execute for node_id, node_outputs in outputs.items(): output_data_formatted[node_id] = [[format_value(x) for x in l] for l in node_outputs] - logging.error("!!! Exception during processing !!!") + logging.error(f"!!! Exception during processing!!! {ex}") logging.error(traceback.format_exc()) error_details = { diff --git a/folder_paths.py b/folder_paths.py index 3f9a32b4d..489795002 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -2,7 +2,7 @@ import os import time import logging -supported_pt_extensions = set(['.ckpt', '.pt', '.bin', '.pth', '.safetensors']) +supported_pt_extensions = set(['.ckpt', '.pt', '.bin', '.pth', '.safetensors', '.pkl']) folder_names_and_paths = {} From 133dc3351b3277f6ce41da7839ace9055329c64c Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 19 Apr 2024 03:52:02 -0400 Subject: [PATCH 19/25] Faster blur. --- comfy_extras/nodes_post_processing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py index 0110b472f..f0e83fc69 100644 --- a/comfy_extras/nodes_post_processing.py +++ b/comfy_extras/nodes_post_processing.py @@ -5,6 +5,7 @@ from PIL import Image import math import comfy.utils +import comfy.model_management class Blend: @@ -102,6 +103,7 @@ class Blur: if blur_radius == 0: return (image,) + image = image.to(comfy.model_management.get_torch_device()) batch_size, height, width, channels = image.shape kernel_size = blur_radius * 2 + 1 @@ -112,7 +114,7 @@ class Blur: blurred = F.conv2d(padded_image, kernel, padding=kernel_size // 2, groups=channels)[:,:,blur_radius:-blur_radius, blur_radius:-blur_radius] blurred = blurred.permute(0, 2, 3, 1) - return (blurred,) + return (blurred.to(comfy.model_management.intermediate_device()),) class Quantize: def __init__(self): From 644a3ae58d426ffbbc02ef4104034c98e8fc6513 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 20 Apr 2024 04:31:49 -0400 Subject: [PATCH 20/25] Implement Align Your Steps as a AlignYourStepsScheduler node. --- comfy_extras/nodes_align_your_steps.py | 45 ++++++++++++++++++++++++++ nodes.py | 1 + 2 files changed, 46 insertions(+) create mode 100644 comfy_extras/nodes_align_your_steps.py diff --git a/comfy_extras/nodes_align_your_steps.py b/comfy_extras/nodes_align_your_steps.py new file mode 100644 index 000000000..b59f6945b --- /dev/null +++ b/comfy_extras/nodes_align_your_steps.py @@ -0,0 +1,45 @@ +#from: https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/howto.html +import numpy as np +import torch + +def loglinear_interp(t_steps, num_steps): + """ + Performs log-linear interpolation of a given array of decreasing numbers. + """ + xs = np.linspace(0, 1, len(t_steps)) + ys = np.log(t_steps[::-1]) + + new_xs = np.linspace(0, 1, num_steps) + new_ys = np.interp(new_xs, xs, ys) + + interped_ys = np.exp(new_ys)[::-1].copy() + return interped_ys + +NOISE_LEVELS = {"SD1": [14.6146412293, 6.4745760956, 3.8636745985, 2.6946151520, 1.8841921177, 1.3943805092, 0.9642583904, 0.6523686016, 0.3977456272, 0.1515232662, 0.0291671582], + "SDXL":[14.6146412293, 6.3184485287, 3.7681790315, 2.1811480769, 1.3405244945, 0.8620721141, 0.5550693289, 0.3798540708, 0.2332364134, 0.1114188177, 0.0291671582], + "SVD": [700.00, 54.5, 15.886, 7.977, 4.248, 1.789, 0.981, 0.403, 0.173, 0.034, 0.002]} + +class AlignYourStepsScheduler: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model_type": (["SD1", "SDXL", "SVD"], ), + "steps": ("INT", {"default": 10, "min": 10, "max": 10000}), + } + } + RETURN_TYPES = ("SIGMAS",) + CATEGORY = "sampling/custom_sampling/schedulers" + + FUNCTION = "get_sigmas" + + def get_sigmas(self, model_type, steps): + sigmas = NOISE_LEVELS[model_type][:] + if (steps + 1) != len(sigmas): + sigmas = loglinear_interp(sigmas, steps + 1) + + sigmas[-1] = 0 + return (torch.FloatTensor(sigmas), ) + +NODE_CLASS_MAPPINGS = { + "AlignYourStepsScheduler": AlignYourStepsScheduler, +} diff --git a/nodes.py b/nodes.py index ea1e32030..28359e939 100644 --- a/nodes.py +++ b/nodes.py @@ -1943,6 +1943,7 @@ def init_custom_nodes(): "nodes_ip2p.py", "nodes_model_merging_model_specific.py", "nodes_pag.py", + "nodes_align_your_steps.py", ] import_failed = [] From 4ee9aad6caf4c23e69e9e166573c505e757062e5 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 21 Apr 2024 09:02:06 -0400 Subject: [PATCH 21/25] Speed up Sharpen node. --- comfy_extras/nodes_post_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py index f0e83fc69..68f6ef51e 100644 --- a/comfy_extras/nodes_post_processing.py +++ b/comfy_extras/nodes_post_processing.py @@ -227,6 +227,7 @@ class Sharpen: return (image,) batch_size, height, width, channels = image.shape + image = image.to(comfy.model_management.get_torch_device()) kernel_size = sharpen_radius * 2 + 1 kernel = gaussian_kernel(kernel_size, sigma, device=image.device) * -(alpha*10) @@ -241,7 +242,7 @@ class Sharpen: result = torch.clamp(sharpened, 0, 1) - return (result,) + return (result.to(comfy.model_management.intermediate_device()),) class ImageScaleToTotalPixels: upscale_methods = ["nearest-exact", "bilinear", "area", "bicubic", "lanczos"] From d09b5ef4ef150adab31195761725eaba409f6343 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 22 Apr 2024 18:42:41 -0400 Subject: [PATCH 22/25] Free some memory before loading upscale models. --- comfy_extras/nodes_upscale_model.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/comfy_extras/nodes_upscale_model.py b/comfy_extras/nodes_upscale_model.py index 2b5e49a55..4de5d7e37 100644 --- a/comfy_extras/nodes_upscale_model.py +++ b/comfy_extras/nodes_upscale_model.py @@ -37,9 +37,14 @@ class ImageUpscaleWithModel: def upscale(self, upscale_model, image): device = model_management.get_torch_device() + + memory_required = model_management.module_size(upscale_model) + memory_required += (512 * 512 * 3) * image.element_size() * max(upscale_model.scale, 1.0) * 256.0 #The 256.0 is an estimate of how much some of these models take, TODO: make it more accurate + memory_required += image.nelement() * image.element_size() + model_management.free_memory(memory_required, device) + upscale_model.to(device) in_img = image.movedim(-1,-3).to(device) - free_memory = model_management.get_free_memory(device) tile = 512 overlap = 32 From b8218522f112be2e69fd49bbefbe68b57868baa0 Mon Sep 17 00:00:00 2001 From: Pam <42671363+pamparamm@users.noreply.github.com> Date: Tue, 23 Apr 2024 18:40:10 +0500 Subject: [PATCH 23/25] Increase sigma_min/sigma_max range for custom schedulers (#3317) --- comfy_extras/nodes_custom_sampler.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py index 06238f892..7afdbf4bf 100644 --- a/comfy_extras/nodes_custom_sampler.py +++ b/comfy_extras/nodes_custom_sampler.py @@ -39,8 +39,8 @@ class KarrasScheduler: def INPUT_TYPES(s): return {"required": {"steps": ("INT", {"default": 20, "min": 1, "max": 10000}), - "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), - "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), + "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), + "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), "rho": ("FLOAT", {"default": 7.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}), } } @@ -58,8 +58,8 @@ class ExponentialScheduler: def INPUT_TYPES(s): return {"required": {"steps": ("INT", {"default": 20, "min": 1, "max": 10000}), - "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), - "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), + "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), + "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), } } RETURN_TYPES = ("SIGMAS",) @@ -76,8 +76,8 @@ class PolyexponentialScheduler: def INPUT_TYPES(s): return {"required": {"steps": ("INT", {"default": 20, "min": 1, "max": 10000}), - "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), - "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), + "sigma_max": ("FLOAT", {"default": 14.614642, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), + "sigma_min": ("FLOAT", {"default": 0.0291675, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), "rho": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}), } } @@ -117,8 +117,8 @@ class VPScheduler: def INPUT_TYPES(s): return {"required": {"steps": ("INT", {"default": 20, "min": 1, "max": 10000}), - "beta_d": ("FLOAT", {"default": 19.9, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), #TODO: fix default values - "beta_min": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 1000.0, "step":0.01, "round": False}), + "beta_d": ("FLOAT", {"default": 19.9, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), #TODO: fix default values + "beta_min": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 5000.0, "step":0.01, "round": False}), "eps_s": ("FLOAT", {"default": 0.001, "min": 0.0, "max": 1.0, "step":0.0001, "round": False}), } } From 27d5808fc491c7174abc6f407e7dc11c6a7a1ec0 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 23 Apr 2024 13:07:39 -0400 Subject: [PATCH 24/25] Increase max lora strength to 100.0 --- nodes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nodes.py b/nodes.py index 28359e939..a1cfd6365 100644 --- a/nodes.py +++ b/nodes.py @@ -583,8 +583,8 @@ class LoraLoader: return {"required": { "model": ("MODEL",), "clip": ("CLIP", ), "lora_name": (folder_paths.get_filename_list("loras"), ), - "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), - "strength_clip": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "strength_clip": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), }} RETURN_TYPES = ("MODEL", "CLIP") FUNCTION = "load_lora" @@ -617,7 +617,7 @@ class LoraLoaderModelOnly(LoraLoader): def INPUT_TYPES(s): return {"required": { "model": ("MODEL",), "lora_name": (folder_paths.get_filename_list("loras"), ), - "strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), }} RETURN_TYPES = ("MODEL",) FUNCTION = "load_lora_model_only" From 8dc19e40d129c8ee049be7be2657458509717ba5 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 24 Apr 2024 09:20:31 -0400 Subject: [PATCH 25/25] Don't init a VAE model when there are no VAE weights. --- comfy/sd.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/comfy/sd.py b/comfy/sd.py index 57dba0b44..16dc0b732 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -210,7 +210,7 @@ class VAE: self.first_stage_model = StageC_coder() self.downscale_ratio = 32 self.latent_channels = 16 - else: + elif "decoder.conv_in.weight" in sd: #default SD1.x/SD2.x VAE parameters ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0} @@ -226,6 +226,10 @@ class VAE: self.first_stage_model = AutoencodingEngine(regularizer_config={'target': "comfy.ldm.models.autoencoder.DiagonalGaussianRegularizer"}, encoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Encoder", 'params': ddconfig}, decoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Decoder", 'params': ddconfig}) + else: + logging.warning("WARNING: No VAE weights detected, VAE not initalized.") + self.first_stage_model = None + return else: self.first_stage_model = AutoencoderKL(**(config['params'])) self.first_stage_model = self.first_stage_model.eval()