From 1f0f4cc0bdb5483d0ef87bc4edad4b99a3b97d77 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 7 Aug 2023 02:22:26 -0400 Subject: [PATCH 01/12] Add argument to disable auto launching the browser. --- comfy/cli_args.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 83d8cd287..81bbc4796 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -39,6 +39,7 @@ parser.add_argument("--enable-cors-header", type=str, default=None, metavar="ORI parser.add_argument("--extra-model-paths-config", type=str, default=None, metavar="PATH", nargs='+', action='append', help="Load one or more extra_model_paths.yaml files.") parser.add_argument("--output-directory", type=str, default=None, help="Set the ComfyUI output directory.") parser.add_argument("--auto-launch", action="store_true", help="Automatically launch ComfyUI in the default browser.") +parser.add_argument("--disable-auto-launch", action="store_true", help="Disable auto launching the browser.") parser.add_argument("--cuda-device", type=int, default=None, metavar="DEVICE_ID", help="Set the id of the cuda device this instance will use.") cm_group = parser.add_mutually_exclusive_group() cm_group.add_argument("--cuda-malloc", action="store_true", help="Enable cudaMallocAsync (enabled by default for torch 2.0 and up).") @@ -90,3 +91,6 @@ args = parser.parse_args() if args.windows_standalone_build: args.auto_launch = True + +if args.disable_auto_launch: + args.auto_launch = False From 285ea7b79053fc7e3dad3c5f4a38b75a1cca6699 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 7 Aug 2023 08:29:50 -0400 Subject: [PATCH 02/12] Add "display" to custom node example. --- custom_nodes/example_node.py.example | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/custom_nodes/example_node.py.example b/custom_nodes/example_node.py.example index 175202aeb..e37808b03 100644 --- a/custom_nodes/example_node.py.example +++ b/custom_nodes/example_node.py.example @@ -51,9 +51,10 @@ class Example: "default": 0, "min": 0, #Minimum value "max": 4096, #Maximum value - "step": 64 #Slider's step + "step": 64, #Slider's step + "display": "number" # Cosmetic only: display as "number" or "slider" }), - "float_field": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "float_field": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, "display": "number"}), "print_to_screen": (["enable", "disable"],), "string_field": ("STRING", { "multiline": False, #True if you want the field to look like the one on the ClipTextEncode node From 5e2b4893da62dec1c9ee4d0167f6b62e3b11fbef Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 7 Aug 2023 19:29:36 -0400 Subject: [PATCH 03/12] Fix path issue. --- web/extensions/core/linkRenderMode.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/extensions/core/linkRenderMode.js b/web/extensions/core/linkRenderMode.js index 8b8d4e01f..1e9091ec1 100644 --- a/web/extensions/core/linkRenderMode.js +++ b/web/extensions/core/linkRenderMode.js @@ -1,4 +1,4 @@ -import { app } from "/scripts/app.js"; +import { app } from "../../scripts/app.js"; const id = "Comfy.LinkRenderMode"; const ext = { From a5599ed42cef7a8aa77fa9409ab2ce95492bf914 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 8 Aug 2023 10:45:35 -0400 Subject: [PATCH 04/12] Add missing direct dep that gets pulled in by another. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8ee7b83d1..14524485a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ pyyaml Pillow scipy tqdm +psutil From 5ac96897e9782805cd5e8fe85bd98ad03eae2b6f Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 9 Aug 2023 11:31:27 -0400 Subject: [PATCH 05/12] Images can now be uploaded by dragging from another window in chromium. --- web/scripts/widgets.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index d4a15ba84..adf5f26fa 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -433,7 +433,7 @@ export const ComfyWidgets = { // Add handler to check if an image is being dragged over our node node.onDragOver = function (e) { if (e.dataTransfer && e.dataTransfer.items) { - const image = [...e.dataTransfer.items].find((f) => f.kind === "file" && f.type.startsWith("image/")); + const image = [...e.dataTransfer.items].find((f) => f.kind === "file"); return !!image; } From cf10c5592c2ef91d5f0218a6f0e7a536c02c5d96 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 9 Aug 2023 20:32:30 -0400 Subject: [PATCH 06/12] Disable calculating uncond when CFG is 1.0 --- comfy/samplers.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/comfy/samplers.py b/comfy/samplers.py index 044d518a5..de4f36da2 100644 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -189,12 +189,13 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con continue to_run += [(p, COND)] - for x in uncond: - p = get_area_and_mult(x, x_in, cond_concat_in, timestep) - if p is None: - continue + if uncond is not None: + for x in uncond: + p = get_area_and_mult(x, x_in, cond_concat_in, timestep) + if p is None: + continue - to_run += [(p, UNCOND)] + to_run += [(p, UNCOND)] while len(to_run) > 0: first = to_run[0] @@ -282,6 +283,9 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con max_total_area = model_management.maximum_batch_area() + if math.isclose(cond_scale, 1.0): + uncond = None + cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, cond_concat, model_options) if "sampler_cfg_function" in model_options: args = {"cond": cond, "uncond": uncond, "cond_scale": cond_scale, "timestep": timestep} From f7e6a5ed077a6e32a32c6fe65fe86519cc2f0252 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 10 Aug 2023 12:29:56 -0400 Subject: [PATCH 07/12] Fix litegraph button being black on light theme. --- web/lib/litegraph.core.js | 1 + 1 file changed, 1 insertion(+) diff --git a/web/lib/litegraph.core.js b/web/lib/litegraph.core.js index 2682ff309..356c71ac2 100644 --- a/web/lib/litegraph.core.js +++ b/web/lib/litegraph.core.js @@ -9766,6 +9766,7 @@ LGraphNode.prototype.executeAction = function(action) switch (w.type) { case "button": + ctx.fillStyle = background_color; if (w.clicked) { ctx.fillStyle = "#AAA"; w.clicked = false; From c20583286fca8a55324ee7d60cebd3b511b16c91 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 10 Aug 2023 19:44:46 -0400 Subject: [PATCH 08/12] Support diffuser text encoder loras. --- comfy/sd.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/comfy/sd.py b/comfy/sd.py index 2996a938b..b19130ada 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -72,6 +72,7 @@ def load_lora(lora, to_load): regular_lora = "{}.lora_up.weight".format(x) diffusers_lora = "{}_lora.up.weight".format(x) + transformers_lora = "{}.lora_linear_layer.up.weight".format(x) A_name = None if regular_lora in lora.keys(): @@ -82,6 +83,10 @@ def load_lora(lora, to_load): A_name = diffusers_lora B_name = "{}_lora.down.weight".format(x) mid_name = None + elif transformers_lora in lora.keys(): + A_name = transformers_lora + B_name ="{}.lora_linear_layer.down.weight".format(x) + mid_name = None if A_name is not None: mid = None @@ -181,20 +186,29 @@ def model_lora_keys_clip(model, key_map={}): key_map[lora_key] = k lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) key_map[lora_key] = k + lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora + key_map[lora_key] = k k = "clip_l.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c) if k in sdk: lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base key_map[lora_key] = k clip_l_present = True + lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora + key_map[lora_key] = k k = "clip_g.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c) if k in sdk: if clip_l_present: lora_key = "lora_te2_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base + key_map[lora_key] = k + lora_key = "text_encoder_2.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora + key_map[lora_key] = k else: lora_key = "lora_te_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #TODO: test if this is correct for SDXL-Refiner - key_map[lora_key] = k + key_map[lora_key] = k + lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora + key_map[lora_key] = k return key_map From 00877b036344af84731791d4256ecd75cdb2e911 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 11 Aug 2023 02:33:26 -0400 Subject: [PATCH 09/12] Don't ignore extra paths that don't exist. --- folder_paths.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/folder_paths.py b/folder_paths.py index eb7d39b88..d3b76d0dc 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -111,6 +111,8 @@ def add_model_folder_path(folder_name, full_folder_path): global folder_names_and_paths if folder_name in folder_names_and_paths: folder_names_and_paths[folder_name][0].append(full_folder_path) + else: + folder_names_and_paths[folder_name] = ([full_folder_path], set()) def get_folder_paths(folder_name): return folder_names_and_paths[folder_name][0][:] From 2bc12d3d22efb5c63ae3a7fc342bb2dd16b31735 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 11 Aug 2023 05:00:25 -0400 Subject: [PATCH 10/12] Add --temp-directory argument to set temp directory. --- comfy/cli_args.py | 1 + folder_paths.py | 4 ++++ main.py | 6 +++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 81bbc4796..ec7d34a55 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -38,6 +38,7 @@ parser.add_argument("--port", type=int, default=8188, help="Set the listen port. parser.add_argument("--enable-cors-header", type=str, default=None, metavar="ORIGIN", nargs="?", const="*", help="Enable CORS (Cross-Origin Resource Sharing) with optional origin or allow all with default '*'.") parser.add_argument("--extra-model-paths-config", type=str, default=None, metavar="PATH", nargs='+', action='append', help="Load one or more extra_model_paths.yaml files.") parser.add_argument("--output-directory", type=str, default=None, help="Set the ComfyUI output directory.") +parser.add_argument("--temp-directory", type=str, default=None, help="Set the ComfyUI temp directory (default is in the ComfyUI directory).") parser.add_argument("--auto-launch", action="store_true", help="Automatically launch ComfyUI in the default browser.") parser.add_argument("--disable-auto-launch", action="store_true", help="Disable auto launching the browser.") parser.add_argument("--cuda-device", type=int, default=None, metavar="DEVICE_ID", help="Set the id of the cuda device this instance will use.") diff --git a/folder_paths.py b/folder_paths.py index d3b76d0dc..e321690dd 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -43,6 +43,10 @@ def set_output_directory(output_dir): global output_directory output_directory = output_dir +def set_temp_directory(temp_dir): + global temp_directory + temp_directory = temp_dir + def get_output_directory(): global output_directory return output_directory diff --git a/main.py b/main.py index 07ebbd701..1571376bd 100644 --- a/main.py +++ b/main.py @@ -100,7 +100,7 @@ def hijack_progress(server): def cleanup_temp(): - temp_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "temp") + temp_dir = folder_paths.get_temp_directory() if os.path.exists(temp_dir): shutil.rmtree(temp_dir, ignore_errors=True) @@ -127,6 +127,10 @@ def load_extra_path_config(yaml_path): if __name__ == "__main__": + if args.temp_directory: + temp_dir = os.path.join(os.path.abspath(args.temp_directory), "temp") + print(f"Setting temp directory to: {temp_dir}") + folder_paths.set_temp_directory(temp_dir) cleanup_temp() loop = asyncio.new_event_loop() From c8a23ce9e81ce976b0e8637593d33e0790d3d95a Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 11 Aug 2023 13:04:21 -0400 Subject: [PATCH 11/12] Support for yet another lora type based on diffusers. --- comfy/sd.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index b19130ada..bff9ee141 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -223,13 +223,16 @@ def model_lora_keys_unet(model, key_map={}): diffusers_keys = utils.unet_to_diffusers(model.model_config.unet_config) for k in diffusers_keys: if k.endswith(".weight"): + unet_key = "diffusion_model.{}".format(diffusers_keys[k]) key_lora = k[:-len(".weight")].replace(".", "_") - key_map["lora_unet_{}".format(key_lora)] = "diffusion_model.{}".format(diffusers_keys[k]) + key_map["lora_unet_{}".format(key_lora)] = unet_key - diffusers_lora_key = "unet.{}".format(k[:-len(".weight")].replace(".to_", ".processor.to_")) - if diffusers_lora_key.endswith(".to_out.0"): - diffusers_lora_key = diffusers_lora_key[:-2] - key_map[diffusers_lora_key] = "diffusion_model.{}".format(diffusers_keys[k]) + diffusers_lora_prefix = ["", "unet."] + for p in diffusers_lora_prefix: + diffusers_lora_key = "{}{}".format(p, k[:-len(".weight")].replace(".to_", ".processor.to_")) + if diffusers_lora_key.endswith(".to_out.0"): + diffusers_lora_key = diffusers_lora_key[:-2] + key_map[diffusers_lora_key] = unet_key return key_map def set_attr(obj, attr, value): From 8c730dc4a7225b2b103497ea1c2f67e48db145ab Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 12 Aug 2023 01:01:49 -0400 Subject: [PATCH 12/12] Add an ImageCompositeMasked node. --- comfy_extras/nodes_mask.py | 79 +++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/comfy_extras/nodes_mask.py b/comfy_extras/nodes_mask.py index 15377af14..b80c8b9a2 100644 --- a/comfy_extras/nodes_mask.py +++ b/comfy_extras/nodes_mask.py @@ -2,6 +2,35 @@ import torch from nodes import MAX_RESOLUTION +def composite(destination, source, x, y, mask = None, multiplier = 8): + x = max(-source.shape[3] * multiplier, min(x, destination.shape[3] * multiplier)) + y = max(-source.shape[2] * multiplier, min(y, destination.shape[2] * multiplier)) + + left, top = (x // multiplier, y // multiplier) + right, bottom = (left + source.shape[3], top + source.shape[2],) + + + if mask is None: + mask = torch.ones_like(source) + else: + mask = mask.clone() + mask = torch.nn.functional.interpolate(mask[None, None], size=(source.shape[2], source.shape[3]), mode="bilinear") + mask = mask.repeat((source.shape[0], source.shape[1], 1, 1)) + + # calculate the bounds of the source that will be overlapping the destination + # this prevents the source trying to overwrite latent pixels that are out of bounds + # of the destination + visible_width, visible_height = (destination.shape[3] - left + min(0, x), destination.shape[2] - top + min(0, y),) + + mask = mask[:, :, :visible_height, :visible_width] + inverse_mask = torch.ones_like(mask) - mask + + source_portion = mask * source[:, :, :visible_height, :visible_width] + destination_portion = inverse_mask * destination[:, :, top:bottom, left:right] + + destination[:, :, top:bottom, left:right] = source_portion + destination_portion + return destination + class LatentCompositeMasked: @classmethod def INPUT_TYPES(s): @@ -25,36 +54,31 @@ class LatentCompositeMasked: output = destination.copy() destination = destination["samples"].clone() source = source["samples"] + output["samples"] = composite(destination, source, x, y, mask, 8) + return (output,) - x = max(-source.shape[3] * 8, min(x, destination.shape[3] * 8)) - y = max(-source.shape[2] * 8, min(y, destination.shape[2] * 8)) +class ImageCompositeMasked: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "destination": ("IMAGE",), + "source": ("IMAGE",), + "x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}), + "y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}), + }, + "optional": { + "mask": ("MASK",), + } + } + RETURN_TYPES = ("IMAGE",) + FUNCTION = "composite" - left, top = (x // 8, y // 8) - right, bottom = (left + source.shape[3], top + source.shape[2],) - - - if mask is None: - mask = torch.ones_like(source) - else: - mask = mask.clone() - mask = torch.nn.functional.interpolate(mask[None, None], size=(source.shape[2], source.shape[3]), mode="bilinear") - mask = mask.repeat((source.shape[0], source.shape[1], 1, 1)) - - # calculate the bounds of the source that will be overlapping the destination - # this prevents the source trying to overwrite latent pixels that are out of bounds - # of the destination - visible_width, visible_height = (destination.shape[3] - left + min(0, x), destination.shape[2] - top + min(0, y),) - - mask = mask[:, :, :visible_height, :visible_width] - inverse_mask = torch.ones_like(mask) - mask - - source_portion = mask * source[:, :, :visible_height, :visible_width] - destination_portion = inverse_mask * destination[:, :, top:bottom, left:right] - - destination[:, :, top:bottom, left:right] = source_portion + destination_portion - - output["samples"] = destination + CATEGORY = "image" + def composite(self, destination, source, x, y, mask = None): + destination = destination.clone().movedim(-1, 1) + output = composite(destination, source.movedim(-1, 1), x, y, mask, 1).movedim(1, -1) return (output,) class MaskToImage: @@ -253,6 +277,7 @@ class FeatherMask: NODE_CLASS_MAPPINGS = { "LatentCompositeMasked": LatentCompositeMasked, + "ImageCompositeMasked": ImageCompositeMasked, "MaskToImage": MaskToImage, "ImageToMask": ImageToMask, "SolidMask": SolidMask,