From 0d45efb7d6809fae272a9ba68c3aac5f713347e7 Mon Sep 17 00:00:00 2001 From: shawnington <88048838+shawnington@users.noreply.github.com> Date: Sat, 4 May 2024 02:32:41 -0500 Subject: [PATCH 1/8] Fixed Issue with LoadImage node when loading PNG files with embedded ICC profiles. (#3316) * Fix issue with how PIL loads small PNG files nodes.py Added flag to prevent ValueError: Decompressed Data Too Large when loading PNG images with large meta data such as large embedded color profiles * Update LoadImage node to fix error when loading PNG's in nodes.py Fixed Value Error: Decompressed Data Too Large thrown by PIL when attempting to opening PNG files with large embedded ICC colorspaces by setting the follow flag to true when loading png images: ImageFile.LOAD_TRUNCATED_IMAGES = True * Update node_helpers.py to include open_image helper function open_image includes try except to catch Pillow Value Errors that occur when large ICC profiles are embedded in images. * Update LoadImage node to use open_image helper function inplace of Image.open open_image helper function in node_helpers.py fixes a Pillow error when attempting to open images with large embedded ICC profiles by adding an exception handler to load the image with truncated meta data if regular loading is not possible. --- node_helpers.py | 14 ++++++++++++++ nodes.py | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/node_helpers.py b/node_helpers.py index 8828a4ec9..264bd4d50 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -1,3 +1,4 @@ +from PIL import Image, ImageFile def conditioning_set_values(conditioning, values={}): c = [] @@ -8,3 +9,16 @@ def conditioning_set_values(conditioning, values={}): c.append(n) return c + +def open_image(path): + try : + ImageFile.LOAD_TRUNCATED_IMAGES = False + img = Image.open(path) + + except: + ImageFile.LOAD_TRUNCATED_IMAGES = True + img = Image.open(path) + + finally: + ImageFile.LOAD_TRUNCATED_IMAGES = False + return img diff --git a/nodes.py b/nodes.py index acad256f8..aa6d6fa9f 100644 --- a/nodes.py +++ b/nodes.py @@ -12,12 +12,12 @@ import logging from PIL import Image, ImageOps, ImageSequence from PIL.PngImagePlugin import PngInfo + import numpy as np import safetensors.torch sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy")) - import comfy.diffusers_load import comfy.samplers import comfy.sample @@ -1456,7 +1456,9 @@ class LoadImage: FUNCTION = "load_image" def load_image(self, image): image_path = folder_paths.get_annotated_filepath(image) - img = Image.open(image_path) + + img = node_helpers.open_image(image_path) + output_images = [] output_masks = [] for i in ImageSequence.Iterator(img): From 72508a8d19121e2814ea4dfbce8a5311f37dcd61 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 4 May 2024 03:39:37 -0400 Subject: [PATCH 2/8] Only set LOAD_TRUNCATED_IMAGES when if the Image open fails. Document which PIL issues this works around. --- node_helpers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/node_helpers.py b/node_helpers.py index 264bd4d50..60f8fa415 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -1,4 +1,4 @@ -from PIL import Image, ImageFile +from PIL import Image, ImageFile, UnidentifiedImageError def conditioning_set_values(conditioning, values={}): c = [] @@ -11,14 +11,15 @@ def conditioning_set_values(conditioning, values={}): return c def open_image(path): - try : - ImageFile.LOAD_TRUNCATED_IMAGES = False + prev_value = None + + try: img = Image.open(path) - - except: + except (UnidentifiedImageError, ValueError): #PIL issues #4472 and #2445 + prev_value = ImageFile.LOAD_TRUNCATED_IMAGES ImageFile.LOAD_TRUNCATED_IMAGES = True img = Image.open(path) - finally: - ImageFile.LOAD_TRUNCATED_IMAGES = False + if prev_value is not None: + ImageFile.LOAD_TRUNCATED_IMAGES = prev_value return img From 9a70b70de4b98e02dfd8b6e1747387b52a0d5903 Mon Sep 17 00:00:00 2001 From: vilanele <73059775+vilanele@users.noreply.github.com> Date: Sun, 5 May 2024 11:01:06 +0200 Subject: [PATCH 3/8] add opacity slider in maskeditor (#3404) Co-authored-by: vilanele --- web/extensions/core/maskeditor.js | 46 ++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/web/extensions/core/maskeditor.js b/web/extensions/core/maskeditor.js index 4f69ac760..36f7496e7 100644 --- a/web/extensions/core/maskeditor.js +++ b/web/extensions/core/maskeditor.js @@ -164,6 +164,41 @@ class MaskEditorDialog extends ComfyDialog { return divElement; } + createOpacitySlider(self, name, callback) { + const divElement = document.createElement('div'); + divElement.id = "maskeditor-opacity-slider"; + divElement.style.cssFloat = "left"; + divElement.style.fontFamily = "sans-serif"; + divElement.style.marginRight = "4px"; + divElement.style.color = "var(--input-text)"; + divElement.style.backgroundColor = "var(--comfy-input-bg)"; + divElement.style.borderRadius = "8px"; + divElement.style.borderColor = "var(--border-color)"; + divElement.style.borderStyle = "solid"; + divElement.style.fontSize = "15px"; + divElement.style.height = "21px"; + divElement.style.padding = "1px 6px"; + divElement.style.display = "flex"; + divElement.style.position = "relative"; + divElement.style.top = "2px"; + divElement.style.pointerEvents = "auto"; + self.opacity_slider_input = document.createElement('input'); + self.opacity_slider_input.setAttribute('type', 'range'); + self.opacity_slider_input.setAttribute('min', '0.1'); + self.opacity_slider_input.setAttribute('max', '1.0'); + self.opacity_slider_input.setAttribute('step', '0.01') + self.opacity_slider_input.setAttribute('value', '0.7'); + const labelElement = document.createElement("label"); + labelElement.textContent = name; + + divElement.appendChild(labelElement); + divElement.appendChild(self.opacity_slider_input); + + self.opacity_slider_input.addEventListener("input", callback); + + return divElement; + } + setlayout(imgCanvas, maskCanvas) { const self = this; @@ -203,6 +238,13 @@ class MaskEditorDialog extends ComfyDialog { self.updateBrushPreview(self, null, null); }); + this.brush_opacity_slider = this.createOpacitySlider(self, "Opacity", (event) => { + self.brush_opacity = event.target.value; + if (self.brush_color_mode !== "negative") { + self.maskCanvas.style.opacity = self.brush_opacity; + } + }); + this.colorButton = this.createLeftButton(this.getColorButtonText(), () => { if (self.brush_color_mode === "black") { self.brush_color_mode = "white"; @@ -237,6 +279,7 @@ class MaskEditorDialog extends ComfyDialog { bottom_panel.appendChild(this.saveButton); bottom_panel.appendChild(cancelButton); bottom_panel.appendChild(this.brush_size_slider); + bottom_panel.appendChild(this.brush_opacity_slider); bottom_panel.appendChild(this.colorButton); imgCanvas.style.position = "absolute"; @@ -472,7 +515,7 @@ class MaskEditorDialog extends ComfyDialog { else { return { mixBlendMode: "initial", - opacity: "0.7", + opacity: this.brush_opacity, }; } } @@ -538,6 +581,7 @@ class MaskEditorDialog extends ComfyDialog { this.maskCtx.putImageData(maskData, 0, 0); } + brush_opacity = 0.7; brush_size = 10; brush_color_mode = "black"; drawing_mode = false; From 565eb6d176d2c1c25382585379c4007436aba438 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 5 May 2024 05:24:36 -0400 Subject: [PATCH 4/8] Add a SplitSigmasDenoise node as an alternative to SplitSigmas. --- comfy_extras/nodes_custom_sampler.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py index f3dff000e..47f08bf60 100644 --- a/comfy_extras/nodes_custom_sampler.py +++ b/comfy_extras/nodes_custom_sampler.py @@ -139,6 +139,7 @@ class SplitSigmas: } } RETURN_TYPES = ("SIGMAS","SIGMAS") + RETURN_NAMES = ("high_sigmas", "low_sigmas") CATEGORY = "sampling/custom_sampling/sigmas" FUNCTION = "get_sigmas" @@ -148,6 +149,27 @@ class SplitSigmas: sigmas2 = sigmas[step:] return (sigmas1, sigmas2) +class SplitSigmasDenoise: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"sigmas": ("SIGMAS", ), + "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + } + } + RETURN_TYPES = ("SIGMAS","SIGMAS") + RETURN_NAMES = ("high_sigmas", "low_sigmas") + CATEGORY = "sampling/custom_sampling/sigmas" + + FUNCTION = "get_sigmas" + + def get_sigmas(self, sigmas, denoise): + steps = max(sigmas.shape[-1] - 1, 0) + total_steps = round(steps * denoise) + sigmas1 = sigmas[:-(total_steps)] + sigmas2 = sigmas[-(total_steps + 1):] + return (sigmas1, sigmas2) + class FlipSigmas: @classmethod def INPUT_TYPES(s): @@ -599,6 +621,7 @@ NODE_CLASS_MAPPINGS = { "SamplerDPMPP_SDE": SamplerDPMPP_SDE, "SamplerDPMAdaptative": SamplerDPMAdaptative, "SplitSigmas": SplitSigmas, + "SplitSigmasDenoise": SplitSigmasDenoise, "FlipSigmas": FlipSigmas, "CFGGuider": CFGGuider, From 3787b4f246e05302c4502be116a2bc1a15d03ab1 Mon Sep 17 00:00:00 2001 From: Pam <42671363+pamparamm@users.noreply.github.com> Date: Tue, 7 May 2024 03:39:39 +0500 Subject: [PATCH 5/8] Use get_model_object in Deep Shrink node (#3408) --- comfy_extras/nodes_model_downscale.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/comfy_extras/nodes_model_downscale.py b/comfy_extras/nodes_model_downscale.py index 48bcc6892..58b5073ec 100644 --- a/comfy_extras/nodes_model_downscale.py +++ b/comfy_extras/nodes_model_downscale.py @@ -20,8 +20,9 @@ class PatchModelAddDownscale: CATEGORY = "_for_testing" def patch(self, model, block_number, downscale_factor, start_percent, end_percent, downscale_after_skip, downscale_method, upscale_method): - sigma_start = model.model.model_sampling.percent_to_sigma(start_percent) - sigma_end = model.model.model_sampling.percent_to_sigma(end_percent) + model_sampling = model.get_model_object("model_sampling") + sigma_start = model_sampling.percent_to_sigma(start_percent) + sigma_end = model_sampling.percent_to_sigma(end_percent) def input_block_patch(h, transformer_options): if transformer_options["block"][1] == block_number: From c61eadf69a3ba4033dcf22e2e190fd54f779fc5b Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 6 May 2024 20:04:39 -0400 Subject: [PATCH 6/8] Make the load checkpoint with config function call the regular one. I was going to completely remove this function because it is unmaintainable but I think this is the best compromise. The clip skip and v_prediction parts of the configs should still work but not the fp16 vs fp32. --- comfy/sd.py | 81 ++++++++--------------------------------------------- 1 file changed, 11 insertions(+), 70 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 16dc0b732..ceb080b3d 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -418,6 +418,8 @@ def load_gligen(ckpt_path): return comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device()) def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_clip=True, embedding_directory=None, state_dict=None, config=None): + logging.warning("Warning: The load checkpoint with config function is deprecated and will eventually be removed, please use the other one.") + model, clip, vae, _ = load_checkpoint_guess_config(ckpt_path, output_vae=output_vae, output_clip=output_clip, output_clipvision=False, embedding_directory=embedding_directory, output_model=True) #TODO: this function is a mess and should be removed eventually if config is None: with open(config_path, 'r') as stream: @@ -425,81 +427,20 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_cl model_config_params = config['model']['params'] clip_config = model_config_params['cond_stage_config'] scale_factor = model_config_params['scale_factor'] - vae_config = model_config_params['first_stage_config'] - - fp16 = False - if "unet_config" in model_config_params: - if "params" in model_config_params["unet_config"]: - unet_config = model_config_params["unet_config"]["params"] - if "use_fp16" in unet_config: - fp16 = unet_config.pop("use_fp16") - if fp16: - unet_config["dtype"] = torch.float16 - - noise_aug_config = None - if "noise_aug_config" in model_config_params: - noise_aug_config = model_config_params["noise_aug_config"] - - model_type = model_base.ModelType.EPS if "parameterization" in model_config_params: if model_config_params["parameterization"] == "v": - model_type = model_base.ModelType.V_PREDICTION + m = model.clone() + class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingDiscrete, comfy.model_sampling.V_PREDICTION): + pass + m.add_object_patch("model_sampling", ModelSamplingAdvanced(model.model.model_config)) + model = m - clip = None - vae = None + layer_idx = clip_config.get("params", {}).get("layer_idx", None) + if layer_idx is not None: + clip.clip_layer(layer_idx) - class WeightsLoader(torch.nn.Module): - pass - - if state_dict is None: - state_dict = comfy.utils.load_torch_file(ckpt_path) - - class EmptyClass: - pass - - model_config = comfy.supported_models_base.BASE({}) - - from . import latent_formats - model_config.latent_format = latent_formats.SD15(scale_factor=scale_factor) - model_config.unet_config = model_detection.convert_config(unet_config) - - if config['model']["target"].endswith("ImageEmbeddingConditionedLatentDiffusion"): - model = model_base.SD21UNCLIP(model_config, noise_aug_config["params"], model_type=model_type) - else: - model = model_base.BaseModel(model_config, model_type=model_type) - - if config['model']["target"].endswith("LatentInpaintDiffusion"): - model.set_inpaint() - - if fp16: - model = model.half() - - offload_device = model_management.unet_offload_device() - model = model.to(offload_device) - model.load_model_weights(state_dict, "model.diffusion_model.") - - if output_vae: - vae_sd = comfy.utils.state_dict_prefix_replace(state_dict, {"first_stage_model.": ""}, filter_keys=True) - vae = VAE(sd=vae_sd, config=vae_config) - - if output_clip: - w = WeightsLoader() - clip_target = EmptyClass() - clip_target.params = clip_config.get("params", {}) - if clip_config["target"].endswith("FrozenOpenCLIPEmbedder"): - clip_target.clip = sd2_clip.SD2ClipModel - clip_target.tokenizer = sd2_clip.SD2Tokenizer - clip = CLIP(clip_target, embedding_directory=embedding_directory) - w.cond_stage_model = clip.cond_stage_model.clip_h - elif clip_config["target"].endswith("FrozenCLIPEmbedder"): - clip_target.clip = sd1_clip.SD1ClipModel - clip_target.tokenizer = sd1_clip.SD1Tokenizer - clip = CLIP(clip_target, embedding_directory=embedding_directory) - w.cond_stage_model = clip.cond_stage_model.clip_l - load_clip_weights(w, state_dict) - - return (comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device), clip, vae) + return (model, clip, vae) def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True): sd = comfy.utils.load_torch_file(ckpt_path) From d7fa417bfa24f98fb20495c221faff818ebf5988 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" <128333288+ltdrdata@users.noreply.github.com> Date: Tue, 7 May 2024 17:40:56 +0900 Subject: [PATCH 7/8] feat: shortcuts for zoom in/out (#3410) * feat: shortcuts for zoom in/out * feat: pen support for canvas zoom ctrl + LMB + vertical drag * Ctrl+LMB+Drag -> ctrl+Shift+LMB+Drag --------- Co-authored-by: Lt.Dr.Data --- README.md | 49 ++++++++++++++++++++++++---------------------- web/scripts/app.js | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index eb07540cc..2636ce140 100644 --- a/README.md +++ b/README.md @@ -41,29 +41,32 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git ## Shortcuts -| Keybind | Explanation | -|---------------------------|--------------------------------------------------------------------------------------------------------------------| -| Ctrl + Enter | Queue up current graph for generation | -| Ctrl + Shift + Enter | Queue up current graph as first for generation | -| Ctrl + Z/Ctrl + Y | Undo/Redo | -| Ctrl + S | Save workflow | -| Ctrl + O | Load workflow | -| Ctrl + A | Select all nodes | -| Alt + C | Collapse/uncollapse selected nodes | -| Ctrl + M | Mute/unmute selected nodes | -| Ctrl + B | Bypass selected nodes (acts like the node was removed from the graph and the wires reconnected through) | -| Delete/Backspace | Delete selected nodes | -| Ctrl + Delete/Backspace | Delete the current graph | -| Space | Move the canvas around when held and moving the cursor | -| Ctrl/Shift + Click | Add clicked node to selection | -| Ctrl + C/Ctrl + V | Copy and paste selected nodes (without maintaining connections to outputs of unselected nodes) | -| Ctrl + C/Ctrl + Shift + V | Copy and paste selected nodes (maintaining connections from outputs of unselected nodes to inputs of pasted nodes) | -| Shift + Drag | Move multiple selected nodes at the same time | -| Ctrl + D | Load default graph | -| Q | Toggle visibility of the queue | -| H | Toggle visibility of history | -| R | Refresh graph | -| Double-Click LMB | Open node quick search palette | +| Keybind | Explanation | +|------------------------------------|--------------------------------------------------------------------------------------------------------------------| +| Ctrl + Enter | Queue up current graph for generation | +| Ctrl + Shift + Enter | Queue up current graph as first for generation | +| Ctrl + Z/Ctrl + Y | Undo/Redo | +| Ctrl + S | Save workflow | +| Ctrl + O | Load workflow | +| Ctrl + A | Select all nodes | +| Alt + C | Collapse/uncollapse selected nodes | +| Ctrl + M | Mute/unmute selected nodes | +| Ctrl + B | Bypass selected nodes (acts like the node was removed from the graph and the wires reconnected through) | +| Delete/Backspace | Delete selected nodes | +| Ctrl + Delete/Backspace | Delete the current graph | +| Space | Move the canvas around when held and moving the cursor | +| Ctrl/Shift + Click | Add clicked node to selection | +| Ctrl + C/Ctrl + V | Copy and paste selected nodes (without maintaining connections to outputs of unselected nodes) | +| Ctrl + C/Ctrl + Shift + V | Copy and paste selected nodes (maintaining connections from outputs of unselected nodes to inputs of pasted nodes) | +| Shift + Drag | Move multiple selected nodes at the same time | +| Ctrl + D | Load default graph | +| Alt + `+` | Canvas Zoom in | +| Alt + `-` | Canvas Zoom out | +| Ctrl + Shift + LMB + Vertical drag | Canvas Zoom in/out | +| Q | Toggle visibility of the queue | +| H | Toggle visibility of history | +| R | Refresh graph | +| Double-Click LMB | Open node quick search palette | Ctrl can also be replaced with Cmd instead for macOS users diff --git a/web/scripts/app.js b/web/scripts/app.js index 77f29b8e5..a3105c275 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -953,6 +953,12 @@ export class ComfyApp { const origProcessMouseDown = LGraphCanvas.prototype.processMouseDown; LGraphCanvas.prototype.processMouseDown = function(e) { + // prepare for ctrl+shift drag: zoom start + if(e.ctrlKey && e.shiftKey && e.buttons) { + self.zoom_drag_start = [e.x, e.y, this.ds.scale]; + return; + } + const res = origProcessMouseDown.apply(this, arguments); this.selected_group_moving = false; @@ -973,6 +979,26 @@ export class ComfyApp { const origProcessMouseMove = LGraphCanvas.prototype.processMouseMove; LGraphCanvas.prototype.processMouseMove = function(e) { + // handle ctrl+shift drag + if(e.ctrlKey && e.shiftKey && self.zoom_drag_start) { + // stop canvas zoom action + if(!e.buttons) { + self.zoom_drag_start = null; + return; + } + + // calculate delta + let deltaY = e.y - self.zoom_drag_start[1]; + let startScale = self.zoom_drag_start[2]; + + let scale = startScale - deltaY/100; + + this.ds.changeScale(scale, [this.ds.element.width/2, this.ds.element.height/2]); + this.graph.change(); + + return; + } + const orig_selected_group = this.selected_group; if (this.selected_group && !this.selected_group_resizing && !this.selected_group_moving) { @@ -1059,6 +1085,20 @@ export class ComfyApp { // Trigger onPaste return true; } + + if((e.key === '+') && e.altKey) { + block_default = true; + let scale = this.ds.scale * 1.1; + this.ds.changeScale(scale, [this.ds.element.width/2, this.ds.element.height/2]); + this.graph.change(); + } + + if((e.key === '-') && e.altKey) { + block_default = true; + let scale = this.ds.scale * 1 / 1.1; + this.ds.changeScale(scale, [this.ds.element.width/2, this.ds.element.height/2]); + this.graph.change(); + } } this.graph.change(); From c33412288fbdcd132265c9029a38001fd9696aa5 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 7 May 2024 05:41:06 -0400 Subject: [PATCH 8/8] Fix issue with loading some JPG: #3416 --- nodes.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nodes.py b/nodes.py index aa6d6fa9f..4d3171b8a 100644 --- a/nodes.py +++ b/nodes.py @@ -10,7 +10,7 @@ import time import random import logging -from PIL import Image, ImageOps, ImageSequence +from PIL import Image, ImageOps, ImageSequence, ImageFile from PIL.PngImagePlugin import PngInfo import numpy as np @@ -1462,7 +1462,17 @@ class LoadImage: output_images = [] output_masks = [] for i in ImageSequence.Iterator(img): - i = ImageOps.exif_transpose(i) + prev_value = None + try: + i = ImageOps.exif_transpose(i) + except OSError: + prev_value = ImageFile.LOAD_TRUNCATED_IMAGES + ImageFile.LOAD_TRUNCATED_IMAGES = True + i = ImageOps.exif_transpose(i) + finally: + if prev_value is not None: + ImageFile.LOAD_TRUNCATED_IMAGES = prev_value + if i.mode == 'I': i = i.point(lambda i: i * (1 / 255)) image = i.convert("RGB")