From 8c5d45e7f8b870129062b9027578937bfed673f0 Mon Sep 17 00:00:00 2001 From: Zentropivity Date: Thu, 16 Mar 2023 17:05:26 +0100 Subject: [PATCH 01/22] Correct html head and switch litegraph to pointer mode. --- web/index.html | 3 ++- web/lib/litegraph.core.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/web/index.html b/web/index.html index cd95594f8..b5911fd4d 100644 --- a/web/index.html +++ b/web/index.html @@ -1,10 +1,11 @@ + + - From e9206a52da64dc020657565f1341a417534c27a4 Mon Sep 17 00:00:00 2001 From: Zentropivity Date: Thu, 23 Mar 2023 10:06:06 +0100 Subject: [PATCH 03/22] Fix double click handling by not using faulty is_primary check --- web/lib/litegraph.core.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/lib/litegraph.core.js b/web/lib/litegraph.core.js index e86ecf67b..ba88bd264 100644 --- a/web/lib/litegraph.core.js +++ b/web/lib/litegraph.core.js @@ -5801,7 +5801,7 @@ LGraphNode.prototype.executeAction = function(action) var skip_action = false; var now = LiteGraph.getTime(); var is_primary = (e.isPrimary === undefined || !e.isPrimary); - var is_double_click = (now - this.last_mouseclick < 300) && is_primary; + var is_double_click = (now - this.last_mouseclick < 300); this.mouse[0] = e.clientX; this.mouse[1] = e.clientY; this.graph_mouse[0] = e.canvasX; From 3ebf7452c32398ffed53d838cfdc35bf5257f0b8 Mon Sep 17 00:00:00 2001 From: "Guo Y.K" Date: Thu, 23 Mar 2023 23:33:35 +0800 Subject: [PATCH 04/22] nodes: add ImagePadForOutpaint --- nodes.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/nodes.py b/nodes.py index 0b8be7659..1f495900b 100644 --- a/nodes.py +++ b/nodes.py @@ -908,6 +908,45 @@ class ImageInvert: return (s,) +class ImagePadForOutpaint: + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ("IMAGE",), + "left": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), + "top": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), + "right": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), + "bottom": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), + } + } + + RETURN_TYPES = ("IMAGE", "MASK") + FUNCTION = "expand_image" + + CATEGORY = "image" + + def expand_image(self, image, left, top, right, bottom): + d1, d2, d3, d4 = image.size() + + new_image = torch.zeros( + (d1, d2 + top + bottom, d3 + left + right, d4), + dtype=torch.float32, + ) + new_image[:, top:top + d2, left:left + d3, :] = image + + mask = torch.ones( + (d2 + top + bottom, d3 + left + right), + dtype=torch.float32, + ) + mask[top:top + d2, left:left + d3] = torch.zeros( + (d2, d3), + dtype=torch.float32, + ) + return (new_image, mask) + + NODE_CLASS_MAPPINGS = { "KSampler": KSampler, "CheckpointLoader": CheckpointLoader, @@ -926,6 +965,7 @@ NODE_CLASS_MAPPINGS = { "LoadImageMask": LoadImageMask, "ImageScale": ImageScale, "ImageInvert": ImageInvert, + "ImagePadForOutpaint": ImagePadForOutpaint, "ConditioningCombine": ConditioningCombine, "ConditioningSetArea": ConditioningSetArea, "KSamplerAdvanced": KSamplerAdvanced, From 7a7e3288ee413a600bb945d6b2df235991904b99 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Thu, 23 Mar 2023 21:37:19 +0000 Subject: [PATCH 05/22] Added support for converting widgets to inputs (and back) --- web/extensions/core/widgetInputs.js | 315 ++++++++++++++++++++++++++++ web/jsconfig.json | 9 + web/scripts/app.js | 31 +-- web/scripts/widgets.js | 50 ++++- 4 files changed, 385 insertions(+), 20 deletions(-) create mode 100644 web/extensions/core/widgetInputs.js create mode 100644 web/jsconfig.json diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js new file mode 100644 index 000000000..4ee778026 --- /dev/null +++ b/web/extensions/core/widgetInputs.js @@ -0,0 +1,315 @@ +import { ComfyWidgets, addRandomizeWidget } from "/scripts/widgets.js"; +import { app } from "/scripts/app.js"; + +const CONVERTED_TYPE = "converted-widget"; +const VALID_TYPES = ["STRING", "combo", "number"]; + +function isConvertableWidget(widget, config) { + return VALID_TYPES.includes(widget.type) || VALID_TYPES.includes(config[0]); +} + +function hideWidget(node, widget, suffix = "") { + widget.origType = widget.type; + widget.origComputeSize = widget.computeSize; + widget.origSerializeValue = widget.serializeValue; + widget.computeSize = () => [0, -4]; // -4 is due to the gap litegraph adds between widgets automatically + widget.type = CONVERTED_TYPE + suffix; + widget.serializeValue = () => { + // Prevent serializing the widget if we have no input linked + const { link } = node.inputs.find((i) => i.widget?.name === widget.name); + if (link == null) { + return undefined; + } + return widget.value; + }; + + // Hide any linked widgets, e.g. seed+randomize + if (widget.linkedWidgets) { + for (const w of widget.linkedWidgets) { + hideWidget(node, w, ":" + widget.name); + } + } +} + +function showWidget(widget) { + widget.type = widget.origType; + widget.computeSize = widget.origComputeSize; + widget.serializeValue = widget.origSerializeValue; + + delete widget.origType; + delete widget.origComputeSize; + delete widget.origSerializeValue; + + // Hide any linked widgets, e.g. seed+randomize + if (widget.linkedWidgets) { + for (const w of widget.linkedWidgets) { + showWidget(w); + } + } +} + +function convertToInput(node, widget, config) { + hideWidget(node, widget); + + const { linkType } = getWidgetType(config); + + // Add input and store widget config for creating on primitive node + node.addInput(widget.name, linkType, { + widget: { name: widget.name, config }, + }); +} + +function convertToWidget(node, widget) { + showWidget(widget); + node.removeInput(node.inputs.findIndex((i) => i.widget?.name === widget.name)); +} + +function getWidgetType(config) { + // Special handling for COMBO so we restrict links based on the entries + let type = config[0]; + let linkType = type; + if (type instanceof Array) { + type = "COMBO"; + linkType = linkType.join(","); + } + return { type, linkType }; +} + +app.registerExtension({ + name: "Comfy.WidgetInputs", + async beforeRegisterNodeDef(nodeType, nodeData, app) { + // Add menu options to conver to/from widgets + const origGetExtraMenuOptions = nodeType.prototype.getExtraMenuOptions; + nodeType.prototype.getExtraMenuOptions = function (_, options) { + const r = origGetExtraMenuOptions ? origGetExtraMenuOptions.apply(this, arguments) : undefined; + + if (this.widgets) { + let toInput = []; + let toWidget = []; + for (const w of this.widgets) { + if (w.type === CONVERTED_TYPE) { + toWidget.push({ + content: `Convert ${w.name} to widget`, + callback: () => convertToWidget(this, w), + }); + } else { + const config = nodeData?.input?.required[w.name] || [w.type, w.options || {}]; + if (isConvertableWidget(w, config)) { + toInput.push({ + content: `Convert ${w.name} to input`, + callback: () => convertToInput(this, w, config), + }); + } + } + } + if (toInput.length) { + options.push(...toInput, null); + } + + if (toWidget.length) { + options.push(...toWidget, null); + } + } + + return r; + }; + + // On initial configure of nodes hide all converted widgets + const origOnConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function () { + const r = origOnConfigure ? origOnConfigure.apply(this, arguments) : undefined; + + if (this.inputs) { + for (const input of this.inputs) { + if (input.widget) { + const w = this.widgets.find((w) => w.name === input.widget.name); + hideWidget(this, w); + } + } + } + + return r; + }; + + // Double click a widget input to automatically attach a primitive + const origOnInputDblClick = nodeType.prototype.onInputDblClick; + nodeType.prototype.onInputDblClick = function (slot) { + const r = origOnInputDblClick ? origOnInputDblClick.apply(this, arguments) : undefined; + + if (this.inputs[slot].widget) { + const node = LiteGraph.createNode("PrimitiveNode"); + app.graph.add(node); + node.pos = [this.pos[0] - node.size[0] - 30, this.pos[1]]; + node.connect(0, this, slot); + } + + return r; + }; + }, + registerCustomNodes() { + class PrimitiveNode { + constructor() { + this.addOutput("connect to widget input", "*"); + this.serialize_widgets = true; + this.isVirtualNode = true; + } + + applyToGraph() { + if (!this.outputs[0].links?.length) return; + + // For each output link copy our value over the original widget value + for (const l of this.outputs[0].links) { + const linkInfo = app.graph.links[l]; + const node = this.graph.getNodeById(linkInfo.target_id); + const input = node.inputs[linkInfo.target_slot]; + const widgetName = input.widget.name; + if (widgetName) { + const widget = node.widgets.find((w) => w.name === widgetName); + if (widget) { + widget.value = this.widgets[0].value; + if (widget.callback) { + widget.callback(widget.value, app.canvas, node, app.canvas.graph_mouse, {}); + } + } + } + } + } + + onConnectionsChange(_, index, connected) { + if (connected) { + if (this.outputs[0].links?.length) { + if (!this.widgets?.length) { + this.#onFirstConnection(); + } + if (!this.widgets?.length && this.outputs[0].widget) { + // On first load it often cant recreate the widget as the other node doesnt exist yet + // Manually recreate it from the output info + this.#createWidget(this.outputs[0].widget.config); + } + } + } else if (!this.outputs[0].links?.length) { + this.#onLastDisconnect(); + } + } + + onConnectOutput(slot, type, input, target_node, target_slot) { + // Fires before the link is made allowing us to reject it if it isn't valid + + // No widget, we cant connect + if (!input.widget) return false; + + if (this.outputs[slot].links?.length) { + return this.#isValidConnection(input); + } + } + + #onFirstConnection() { + // First connection can fire before the graph is ready on initial load so random things can be missing + const linkId = this.outputs[0].links[0]; + const link = this.graph.links[linkId]; + if (!link) return; + + const theirNode = this.graph.getNodeById(link.target_id); + if (!theirNode || !theirNode.inputs) return; + + const input = theirNode.inputs[link.target_slot]; + if (!input) return; + + const widget = input.widget; + const { type, linkType } = getWidgetType(widget.config); + + // Update our output to restrict to the widget type + this.outputs[0].type = linkType; + this.outputs[0].name = type; + this.outputs[0].widget = widget; + + this.#createWidget(widget.config, theirNode, widget.name); + } + + #createWidget(inputData, node, widgetName) { + let type = inputData[0]; + + if (type instanceof Array) { + type = "COMBO"; + } + + let widget; + if (type in ComfyWidgets) { + widget = (ComfyWidgets[type](this, "value", inputData, app) || {}).widget; + } else { + widget = this.addWidget(type, "value", null, () => {}, {}); + } + + if (node?.widgets && widget) { + const theirWidget = node.widgets.find((w) => w.name === widgetName); + if (theirWidget) { + widget.value = theirWidget.value; + } + } + + if (widget.type === "number") { + addRandomizeWidget(this, widget, "Random after every gen"); + } + + // Grow our node if required + const sz = this.computeSize(); + if (this.size[0] < sz[0]) { + this.size[0] = sz[0]; + } + if (this.size[1] < sz[1]) { + this.size[1] = sz[1]; + } + + requestAnimationFrame(() => { + if (this.onResize) { + this.onResize(this.size); + } + }); + } + + #isValidConnection(input) { + // Only allow connections where the configs match + const config1 = this.outputs[0].widget.config; + const config2 = input.widget.config; + + if (config1[0] !== config2[0]) return false; + + for (const k in config1[1]) { + if (k !== "default") { + if (config1[1][k] !== config2[1][k]) { + return false; + } + } + } + + return true; + } + + #onLastDisconnect() { + // We cant remove + re-add the output here as if you drag a link over the same link + // it removes, then re-adds, causing it to break + this.outputs[0].type = "*"; + this.outputs[0].name = "connect to widget input"; + delete this.outputs[0].widget; + + if (this.widgets) { + // Allow widgets to cleanup + for (const w of this.widgets) { + if (w.onRemove) { + w.onRemove(); + } + } + this.widgets.length = 0; + } + } + } + + LiteGraph.registerNodeType( + "PrimitiveNode", + Object.assign(PrimitiveNode, { + title: "Primitive", + }) + ); + PrimitiveNode.category = "utils"; + }, +}); diff --git a/web/jsconfig.json b/web/jsconfig.json new file mode 100644 index 000000000..57403d8cf --- /dev/null +++ b/web/jsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "/*": ["./*"] + } + }, + "include": ["."] +} diff --git a/web/scripts/app.js b/web/scripts/app.js index fd410cd30..a33917b39 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -494,7 +494,7 @@ class ComfyApp { // Create and mount the LiteGraph in the DOM const canvasEl = (this.canvasEl = Object.assign(document.createElement("canvas"), { id: "graph-canvas" })); - canvasEl.tabIndex = "1" + canvasEl.tabIndex = "1"; document.body.prepend(canvasEl); this.graph = new LGraph(); @@ -525,7 +525,10 @@ class ComfyApp { this.loadGraphData(workflow); restored = true; } - } catch (err) {} + } catch (err) { + console.error("Error loading previous workflow", err); + debugger; + } // We failed to restore a workflow so load the default if (!restored) { @@ -572,12 +575,8 @@ class ComfyApp { const type = inputData[0]; if (Array.isArray(type)) { - // Enums e.g. latent rotation - let defaultValue = type[0]; - if (inputData[1] && inputData[1].default) { - defaultValue = inputData[1].default; - } - this.addWidget("combo", inputName, defaultValue, () => {}, { values: type }); + // Enums + Object.assign(config, widgets.COMBO(this, inputName, inputData, app) || {}); } else if (`${type}:${inputName}` in widgets) { // Support custom widgets by Type:Name Object.assign(config, widgets[`${type}:${inputName}`](this, inputName, inputData, app) || {}); @@ -667,11 +666,15 @@ class ComfyApp { async graphToPrompt() { const workflow = this.graph.serialize(); const output = {}; - for (const n of workflow.nodes) { - const node = this.graph.getNodeById(n.id); + // Process nodes in order of execution + for (const node of this.graph.computeExecutionOrder(false)) { + const n = workflow.nodes.find((n) => n.id === node.id); if (node.isVirtualNode) { - // Don't serialize frontend only nodes + // Don't serialize frontend only nodes but let them make changes + if (node.applyToGraph) { + node.applyToGraph(workflow); + } continue; } @@ -695,7 +698,11 @@ class ComfyApp { let link = node.getInputLink(i); while (parent && parent.isVirtualNode) { link = parent.getInputLink(link.origin_slot); - parent = parent.getInputNode(link.origin_slot); + if (link) { + parent = parent.getInputNode(link.origin_slot); + } else { + parent = null; + } } if (link) { diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index 30a02e72e..8c78d9551 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -10,9 +10,8 @@ function getNumberDefaults(inputData, defaultStep) { return { val: defaultVal, config: { min, max, step: 10.0 * step } }; } -function seedWidget(node, inputName, inputData) { - const seed = ComfyWidgets.INT(node, inputName, inputData); - const randomize = node.addWidget("toggle", "Random seed after every gen", true, function (v) {}, { +export function addRandomizeWidget(node, targetWidget, name, defaultValue = false) { + const randomize = node.addWidget("toggle", name, defaultValue, function (v) {}, { on: "enabled", off: "disabled", serialize: false, // Don't include this in prompt. @@ -20,14 +19,28 @@ function seedWidget(node, inputName, inputData) { randomize.afterQueued = () => { if (randomize.value) { - seed.widget.value = Math.floor(Math.random() * 1125899906842624); + const min = targetWidget.options?.min; + const max = targetWidget.options?.max; + if (min != null || max != null) { + targetWidget.value = Math.floor(Math.random() * ((max ?? 9999999999) - (min ?? 0) + 1) + (min ?? 0)); + } else { + targetWidget.value = Math.floor(Math.random() * 1125899906842624); + } } }; + return randomize; +} +function seedWidget(node, inputName, inputData) { + const seed = ComfyWidgets.INT(node, inputName, inputData); + const randomize = addRandomizeWidget(node, seed.widget, "Random seed after every gen", true); + + seed.widget.linkedWidgets = [randomize]; return { widget: seed, randomize }; } const MultilineSymbol = Symbol(); +const MultilineResizeSymbol = Symbol(); function addMultilineWidget(node, name, opts, app) { const MIN_SIZE = 50; @@ -95,7 +108,7 @@ function addMultilineWidget(node, name, opts, app) { // Calculate it here instead computeSize(node.size); } - const visible = app.canvas.ds.scale > 0.5; + const visible = app.canvas.ds.scale > 0.5 && this.type === "customtext"; const t = ctx.getTransform(); const margin = 10; Object.assign(this.inputEl.style, { @@ -149,9 +162,22 @@ function addMultilineWidget(node, name, opts, app) { } }; - if (!(MultilineSymbol in node)) { - node[MultilineSymbol] = true; - const onResize = node.onResize; + widget.onRemove = () => { + widget.inputEl?.remove(); + + // Restore original size handler if we are the last + if (!--node[MultilineSymbol]) { + node.onResize = node[MultilineResizeSymbol]; + delete node[MultilineSymbol]; + delete node[MultilineResizeSymbol]; + } + }; + + if (node[MultilineSymbol]) { + node[MultilineSymbol]++; + } else { + node[MultilineSymbol] = 1; + const onResize = (node[MultilineResizeSymbol] = node.onResize); node.onResize = function (size) { computeSize(size); @@ -199,6 +225,14 @@ export const ComfyWidgets = { return { widget: node.addWidget("text", inputName, defaultVal, () => {}, {}) }; } }, + COMBO(node, inputName, inputData) { + const type = inputData[0]; + let defaultValue = type[0]; + if (inputData[1] && inputData[1].default) { + defaultValue = inputData[1].default; + } + return { widget: node.addWidget("combo", inputName, defaultValue, () => {}, { values: type }) }; + }, IMAGEUPLOAD(node, inputName, inputData, app) { const imageWidget = node.widgets.find((w) => w.name === "image"); let uploadWidget; From bb0017673199890cd47f2ba5cd7e8272b6b554b5 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Thu, 23 Mar 2023 21:43:51 +0000 Subject: [PATCH 06/22] Remove debugger --- web/scripts/app.js | 1 - 1 file changed, 1 deletion(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index a33917b39..a9acdce40 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -527,7 +527,6 @@ class ComfyApp { } } catch (err) { console.error("Error loading previous workflow", err); - debugger; } // We failed to restore a workflow so load the default From b13539c3ddcf370977dade661702e5e62801ba0d Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:27:19 +0000 Subject: [PATCH 07/22] Sync widget changes --- web/extensions/core/widgetInputs.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index 4ee778026..e5bfab83e 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -251,6 +251,16 @@ app.registerExtension({ addRandomizeWidget(this, widget, "Random after every gen"); } + // When our value changes, update other widgets to reflect our changes + // e.g. so LoadImage shows correct image + const callback = widget.callback; + const self = this; + widget.callback = function () { + const r = callback ? callback.apply(this, arguments) : undefined; + self.applyToGraph(); + return r; + }; + // Grow our node if required const sz = this.computeSize(); if (this.size[0] < sz[0]) { From 89fd5ed5740fb5ad87842bb0094a531b6dcd2f2d Mon Sep 17 00:00:00 2001 From: Yurii Mazurevich Date: Fri, 24 Mar 2023 14:04:50 +0200 Subject: [PATCH 08/22] Added MPS device support --- comfy/model_management.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 809b19ea2..6288d762c 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -4,6 +4,7 @@ NO_VRAM = 1 LOW_VRAM = 2 NORMAL_VRAM = 3 HIGH_VRAM = 4 +MPS = 4 accelerate_enabled = False vram_state = NORMAL_VRAM @@ -61,7 +62,8 @@ if "--novram" in sys.argv: set_vram_to = NO_VRAM if "--highvram" in sys.argv: vram_state = HIGH_VRAM - +if torch.backends.mps.is_available(): + vram_state = MPS if set_vram_to == LOW_VRAM or set_vram_to == NO_VRAM: try: @@ -79,7 +81,7 @@ if set_vram_to == LOW_VRAM or set_vram_to == NO_VRAM: if "--cpu" in sys.argv: vram_state = CPU -print("Set vram state to:", ["CPU", "NO VRAM", "LOW VRAM", "NORMAL VRAM", "HIGH VRAM"][vram_state]) +print("Set vram state to:", ["CPU", "NO VRAM", "LOW VRAM", "NORMAL VRAM", "HIGH VRAM", "MPS"][vram_state]) current_loaded_model = None @@ -128,6 +130,12 @@ def load_model_gpu(model): current_loaded_model = model if vram_state == CPU: pass + elif vram_state == MPS: + # print(inspect.getmro(real_model.__class__)) + # print(dir(real_model)) + mps_device = torch.device("mps") + real_model.to(mps_device) + pass elif vram_state == NORMAL_VRAM or vram_state == HIGH_VRAM: model_accelerated = False real_model.cuda() @@ -146,6 +154,9 @@ def load_controlnet_gpu(models): global vram_state if vram_state == CPU: return + + if vram_state == MPS: + return if vram_state == LOW_VRAM or vram_state == NO_VRAM: #don't load controlnets like this if low vram because they will be loaded right before running and unloaded right after @@ -173,6 +184,8 @@ def unload_if_low_vram(model): return model def get_torch_device(): + if vram_state == MPS: + return torch.device("mps") if vram_state == CPU: return torch.device("cpu") else: @@ -195,7 +208,7 @@ def get_free_memory(dev=None, torch_free_too=False): if dev is None: dev = get_torch_device() - if hasattr(dev, 'type') and dev.type == 'cpu': + if hasattr(dev, 'type') and (dev.type == 'cpu' or dev.type == 'mps'): mem_free_total = psutil.virtual_memory().available mem_free_torch = mem_free_total else: @@ -224,8 +237,12 @@ def cpu_mode(): global vram_state return vram_state == CPU +def mps_mode(): + global vram_state + return vram_state == MPS + def should_use_fp16(): - if cpu_mode(): + if cpu_mode() or mps_mode(): return False #TODO ? if torch.cuda.is_bf16_supported(): From 4b943d2b60857f5fe88723d3e81e3cd46e4c05f2 Mon Sep 17 00:00:00 2001 From: Yurii Mazurevich Date: Fri, 24 Mar 2023 14:15:30 +0200 Subject: [PATCH 09/22] Removed unnecessary comment --- comfy/model_management.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 6288d762c..db774e49b 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -131,8 +131,6 @@ def load_model_gpu(model): if vram_state == CPU: pass elif vram_state == MPS: - # print(inspect.getmro(real_model.__class__)) - # print(dir(real_model)) mps_device = torch.device("mps") real_model.to(mps_device) pass From 4580f3e64420835aa73b89ec4e8c9bda48c5ee2a Mon Sep 17 00:00:00 2001 From: "Guo Y.K" Date: Fri, 24 Mar 2023 22:39:33 +0800 Subject: [PATCH 10/22] nodes: add feathering to to ImagePadForOutpaint --- nodes.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/nodes.py b/nodes.py index 1f495900b..40e7558fe 100644 --- a/nodes.py +++ b/nodes.py @@ -919,6 +919,7 @@ class ImagePadForOutpaint: "top": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), "right": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), "bottom": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), + "feathering": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 1}), } } @@ -927,7 +928,7 @@ class ImagePadForOutpaint: CATEGORY = "image" - def expand_image(self, image, left, top, right, bottom): + def expand_image(self, image, left, top, right, bottom, feathering): d1, d2, d3, d4 = image.size() new_image = torch.zeros( @@ -940,10 +941,30 @@ class ImagePadForOutpaint: (d2 + top + bottom, d3 + left + right), dtype=torch.float32, ) - mask[top:top + d2, left:left + d3] = torch.zeros( - (d2, d3), - dtype=torch.float32, - ) + + if feathering > 0 and feathering * 2 < d2 and feathering * 2 < d3: + # distances to border + mi, mj = torch.meshgrid( + torch.arange(d2, dtype=torch.float32), + torch.arange(d3, dtype=torch.float32), + indexing='ij', + ) + distances = torch.minimum( + torch.minimum(mi, mj), + torch.minimum(d2 - 1 - mi, d3 - 1 - mj), + ) + # convert distances to square falloff from 1 to 0 + t = (feathering - distances) / feathering + t.clamp_(min=0) + t.square_() + + mask[top:top + d2, left:left + d3] = t + else: + mask[top:top + d2, left:left + d3] = torch.zeros( + (d2, d3), + dtype=torch.float32, + ) + return (new_image, mask) From 7f0fd99b5d2fbaaf23ec707aa357067a6f4e44fb Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 24 Mar 2023 11:39:51 -0400 Subject: [PATCH 11/22] Make ddim work with --cpu --- comfy/samplers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/samplers.py b/comfy/samplers.py index bf4f1796b..17201d9dc 100644 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -450,7 +450,7 @@ class KSampler: noise_mask = None if denoise_mask is not None: noise_mask = 1.0 - denoise_mask - sampler = DDIMSampler(self.model) + sampler = DDIMSampler(self.model, device=self.device) sampler.make_schedule_timesteps(ddim_timesteps=timesteps, verbose=False) z_enc = sampler.stochastic_encode(latent_image, torch.tensor([len(timesteps) - 1] * noise.shape[0]).to(self.device), noise=noise, max_denoise=max_denoise) samples, _ = sampler.sample_custom(ddim_timesteps=timesteps, From 97198f5b749e19bd8ead423911b9d611b99e1613 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 24 Mar 2023 16:36:11 +0000 Subject: [PATCH 12/22] Restore original size after add/remove input --- web/extensions/core/widgetInputs.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index e5bfab83e..b699373ef 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -54,14 +54,22 @@ function convertToInput(node, widget, config) { const { linkType } = getWidgetType(config); // Add input and store widget config for creating on primitive node + const sz = node.size; node.addInput(widget.name, linkType, { widget: { name: widget.name, config }, }); + + // Restore original size but grow if needed + node.setSize([Math.max(sz[0], node.size[0]), Math.max(sz[1], node.size[1])]); } function convertToWidget(node, widget) { showWidget(widget); + const sz = node.size; node.removeInput(node.inputs.findIndex((i) => i.widget?.name === widget.name)); + + // Restore original size but grow if needed + node.setSize([Math.max(sz[0], node.size[0]), Math.max(sz[1], node.size[1])]); } function getWidgetType(config) { From 1fa9ccaa164c6f11c24d6c94c89dd94b45d8582e Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 24 Mar 2023 16:51:17 +0000 Subject: [PATCH 13/22] Set title to widget name --- web/extensions/core/widgetInputs.js | 1 + 1 file changed, 1 insertion(+) diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index b699373ef..ad8bcdf02 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -149,6 +149,7 @@ app.registerExtension({ app.graph.add(node); node.pos = [this.pos[0] - node.size[0] - 30, this.pos[1]]; node.connect(0, this, slot); + node.title = this.inputs[slot].name; } return r; From fc71e7ea08db1f41eaac0409814b25bfbf37da88 Mon Sep 17 00:00:00 2001 From: Yurii Mazurevich Date: Fri, 24 Mar 2023 19:39:55 +0200 Subject: [PATCH 14/22] Fixed typo --- comfy/model_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index db774e49b..dfeef81a5 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -4,7 +4,7 @@ NO_VRAM = 1 LOW_VRAM = 2 NORMAL_VRAM = 3 HIGH_VRAM = 4 -MPS = 4 +MPS = 5 accelerate_enabled = False vram_state = NORMAL_VRAM From 4461ddc9e67875c22d3a222e8c86cd67f8af0f66 Mon Sep 17 00:00:00 2001 From: Zentropivity Date: Fri, 24 Mar 2023 19:18:58 +0100 Subject: [PATCH 15/22] Change close on mouseleave to false --- web/lib/litegraph.core.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/lib/litegraph.core.js b/web/lib/litegraph.core.js index ba88bd264..862d59067 100644 --- a/web/lib/litegraph.core.js +++ b/web/lib/litegraph.core.js @@ -108,7 +108,7 @@ node_box_coloured_when_on: false, // [true!] this make the nodes box (top left circle) coloured when triggered (execute/action), visual feedback node_box_coloured_by_mode: false, // [true!] nodebox based on node mode, visual feedback - dialog_close_on_mouse_leave: true, // [false on mobile] better true if not touch device, TODO add an helper/listener to close if false + dialog_close_on_mouse_leave: false, // [false on mobile] better true if not touch device, TODO add an helper/listener to close if false dialog_close_on_mouse_leave_delay: 500, shift_click_do_break_link_from: false, // [false!] prefer false if results too easy to break links - implement with ALT or TODO custom keys From 4adcea72287981f88758151c73a1892c82724ffe Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 24 Mar 2023 14:30:43 -0400 Subject: [PATCH 16/22] I don't think controlnets were being handled correctly by MPS. --- comfy/model_management.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index dfeef81a5..0d5702b91 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -62,8 +62,7 @@ if "--novram" in sys.argv: set_vram_to = NO_VRAM if "--highvram" in sys.argv: vram_state = HIGH_VRAM -if torch.backends.mps.is_available(): - vram_state = MPS + if set_vram_to == LOW_VRAM or set_vram_to == NO_VRAM: try: @@ -78,6 +77,12 @@ if set_vram_to == LOW_VRAM or set_vram_to == NO_VRAM: total_vram_available_mb = (total_vram - 1024) // 2 total_vram_available_mb = int(max(256, total_vram_available_mb)) +try: + if torch.backends.mps.is_available(): + vram_state = MPS +except: + pass + if "--cpu" in sys.argv: vram_state = CPU @@ -152,9 +157,6 @@ def load_controlnet_gpu(models): global vram_state if vram_state == CPU: return - - if vram_state == MPS: - return if vram_state == LOW_VRAM or vram_state == NO_VRAM: #don't load controlnets like this if low vram because they will be loaded right before running and unloaded right after @@ -164,9 +166,10 @@ def load_controlnet_gpu(models): if m not in models: m.cpu() + device = get_torch_device() current_gpu_controlnets = [] for m in models: - current_gpu_controlnets.append(m.cuda()) + current_gpu_controlnets.append(m.to(device)) def load_if_low_vram(model): From d6830b958c1f37a584ddb5d91beeef54b6540ce4 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 24 Mar 2023 20:20:34 +0000 Subject: [PATCH 17/22] Prevent exactly overlapping nodes Throttle double click --- web/extensions/core/widgetInputs.js | 30 ++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index ad8bcdf02..5e6013471 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -139,17 +139,41 @@ app.registerExtension({ return r; }; + function isNodeAtPos(pos) { + for (const n of app.graph._nodes) { + if (n.pos[0] === pos[0] && n.pos[1] === pos[1]) { + return true; + } + } + return false; + } + // Double click a widget input to automatically attach a primitive const origOnInputDblClick = nodeType.prototype.onInputDblClick; + const ignoreDblClick = Symbol(); nodeType.prototype.onInputDblClick = function (slot) { const r = origOnInputDblClick ? origOnInputDblClick.apply(this, arguments) : undefined; - if (this.inputs[slot].widget) { + const input = this.inputs[slot]; + if (input.widget && !input[ignoreDblClick]) { const node = LiteGraph.createNode("PrimitiveNode"); app.graph.add(node); - node.pos = [this.pos[0] - node.size[0] - 30, this.pos[1]]; + + // Calculate a position that wont directly overlap another node + const pos = [this.pos[0] - node.size[0] - 30, this.pos[1]]; + while (isNodeAtPos(pos)) { + pos[1] += LiteGraph.NODE_TITLE_HEIGHT; + } + + node.pos = pos; node.connect(0, this, slot); - node.title = this.inputs[slot].name; + node.title = input.name; + + // Prevent adding duplicates due to triple clicking + input[ignoreDblClick] = true; + setTimeout(() => { + delete input[ignoreDblClick]; + }, 300); } return r; From 44b6eaad6fa2316d921b6136ab28b0342e240a7a Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 25 Mar 2023 01:04:02 -0400 Subject: [PATCH 18/22] Don't completely break workflow if field name changes. --- web/extensions/core/widgetInputs.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index 5e6013471..ff9227d28 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -131,7 +131,11 @@ app.registerExtension({ for (const input of this.inputs) { if (input.widget) { const w = this.widgets.find((w) => w.name === input.widget.name); - hideWidget(this, w); + if (w) { + hideWidget(this, w); + } else { + convertToWidget(this, input) + } } } } From bfe64dc73cbf49901d46e57c2c0785876d866f6f Mon Sep 17 00:00:00 2001 From: ltdrdata <128333288+ltdrdata@users.noreply.github.com> Date: Sat, 25 Mar 2023 15:23:29 +0900 Subject: [PATCH 19/22] Add support for Google Drive mount feature in Colab Notebook (#193) * Added Google Drive mount feature to Colab Notebook. * Don't set goole drive mount as default and rollback triton installation. * colab notebook robust patch (while switching mounting mode) --------- Co-authored-by: Lt.Dr.Data --- notebooks/comfyui_colab.ipynb | 138 ++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 55 deletions(-) diff --git a/notebooks/comfyui_colab.ipynb b/notebooks/comfyui_colab.ipynb index d9726947d..5108ec830 100644 --- a/notebooks/comfyui_colab.ipynb +++ b/notebooks/comfyui_colab.ipynb @@ -1,29 +1,13 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU", - "gpuClass": "standard" - }, "cells": [ { "cell_type": "markdown", - "source": [ - "Git clone the repo and install the requirements. (ignore the pip errors about protobuf)" - ], "metadata": { "id": "aaaaaaaaaa" - } + }, + "source": [ + "Git clone the repo and install the requirements. (ignore the pip errors about protobuf)" + ] }, { "cell_type": "code", @@ -33,22 +17,55 @@ }, "outputs": [], "source": [ - "!git clone https://github.com/comfyanonymous/ComfyUI\n", - "%cd ComfyUI\n", - "!pip install xformers -r requirements.txt" + "#@title Environment Setup\n", + "\n", + "from pathlib import Path\n", + "\n", + "OPTIONS = {}\n", + "\n", + "USE_GOOGLE_DRIVE = False #@param {type:\"boolean\"}\n", + "UPDATE_COMFY_UI = True #@param {type:\"boolean\"}\n", + "WORKSPACE = 'ComfyUI'\n", + "OPTIONS['USE_GOOGLE_DRIVE'] = USE_GOOGLE_DRIVE\n", + "OPTIONS['UPDATE_COMFY_UI'] = UPDATE_COMFY_UI\n", + "\n", + "if OPTIONS['USE_GOOGLE_DRIVE']:\n", + " !echo \"Mounting Google Drive...\"\n", + " %cd /\n", + " \n", + " from google.colab import drive\n", + " drive.mount('/content/drive')\n", + "\n", + " WORKSPACE = \"/content/drive/MyDrive/ComfyUI\"\n", + " %cd /content/drive/MyDrive\n", + "\n", + "![ ! -d $WORKSPACE ] && echo -= Initial setup ComfyUI =- && git clone https://github.com/comfyanonymous/ComfyUI\n", + "%cd $WORKSPACE\n", + "\n", + "if OPTIONS['UPDATE_COMFY_UI']:\n", + " !echo -= Updating ComfyUI =-\n", + " !git pull\n", + "\n", + "!echo -= Install dependencies =-\n", + "!pip -q install xformers -r requirements.txt" ] }, { "cell_type": "markdown", - "source": [ - "Download some models/checkpoints/vae or custom comfyui nodes (uncomment the commands for the ones you want)" - ], "metadata": { "id": "cccccccccc" - } + }, + "source": [ + "Download some models/checkpoints/vae or custom comfyui nodes (uncomment the commands for the ones you want)" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dddddddddd" + }, + "outputs": [], "source": [ "# Checkpoints\n", "\n", @@ -110,26 +127,26 @@ "#!wget -c https://huggingface.co/sberbank-ai/Real-ESRGAN/resolve/main/RealESRGAN_x4.pth -P ./models/upscale_models/\n", "\n", "\n" - ], - "metadata": { - "id": "dddddddddd" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "kkkkkkkkkkkkkk" + }, "source": [ "### Run ComfyUI with localtunnel (Recommended Way)\n", "\n", "\n" - ], - "metadata": { - "id": "kkkkkkkkkkkkkk" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jjjjjjjjjjjjj" + }, + "outputs": [], "source": [ "!npm install -g localtunnel\n", "\n", @@ -154,15 +171,13 @@ "threading.Thread(target=iframe_thread, daemon=True, args=(8188,)).start()\n", "\n", "!python main.py --dont-print-server" - ], - "metadata": { - "id": "jjjjjjjjjjjjj" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "gggggggggg" + }, "source": [ "### Run ComfyUI with colab iframe (use only in case the previous way with localtunnel doesn't work)\n", "\n", @@ -171,13 +186,15 @@ "If you want to open it in another window use the link.\n", "\n", "Note that some UI features like live image previews won't work because the colab iframe blocks websockets." - ], - "metadata": { - "id": "gggggggggg" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hhhhhhhhhh" + }, + "outputs": [], "source": [ "import threading\n", "import time\n", @@ -198,12 +215,23 @@ "threading.Thread(target=iframe_thread, daemon=True, args=(8188,)).start()\n", "\n", "!python main.py --dont-print-server" - ], - "metadata": { - "id": "hhhhhhhhhh" - }, - "execution_count": null, - "outputs": [] + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 4c01386c19e2dd5be3ba234124575a80283995df Mon Sep 17 00:00:00 2001 From: "Guo Y.K" Date: Sat, 25 Mar 2023 16:27:47 +0800 Subject: [PATCH 20/22] nodes: ImagePadForOutpaint: fix feathering, ignore edges not expanded --- nodes.py | 47 ++++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/nodes.py b/nodes.py index 40e7558fe..a3c033508 100644 --- a/nodes.py +++ b/nodes.py @@ -1,3 +1,5 @@ +import math + import torch import os @@ -942,28 +944,31 @@ class ImagePadForOutpaint: dtype=torch.float32, ) - if feathering > 0 and feathering * 2 < d2 and feathering * 2 < d3: - # distances to border - mi, mj = torch.meshgrid( - torch.arange(d2, dtype=torch.float32), - torch.arange(d3, dtype=torch.float32), - indexing='ij', - ) - distances = torch.minimum( - torch.minimum(mi, mj), - torch.minimum(d2 - 1 - mi, d3 - 1 - mj), - ) - # convert distances to square falloff from 1 to 0 - t = (feathering - distances) / feathering - t.clamp_(min=0) - t.square_() + t = torch.zeros( + (d2, d3), + dtype=torch.float32 + ) - mask[top:top + d2, left:left + d3] = t - else: - mask[top:top + d2, left:left + d3] = torch.zeros( - (d2, d3), - dtype=torch.float32, - ) + if feathering > 0 and feathering * 2 < d2 and feathering * 2 < d3: + + for i in range(d2): + for j in range(d3): + dt = i if top != 0 else d2 + db = d2 - i if bottom != 0 else d2 + + dl = j if left != 0 else d3 + dr = d3 - j if right != 0 else d3 + + d = min(dt, db, dl, dr) + + if d >= feathering: + continue + + v = (feathering - d) / feathering + + t[i, j] = v * v + + mask[top:top + d2, left:left + d3] = t return (new_image, mask) From 3b873029581329fd943a56955550f0b02c548c62 Mon Sep 17 00:00:00 2001 From: "Guo Y.K" Date: Sat, 25 Mar 2023 16:32:48 +0800 Subject: [PATCH 21/22] nodes: ImagePadForOutpaint: clean imports --- nodes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nodes.py b/nodes.py index a3c033508..84f8b01f6 100644 --- a/nodes.py +++ b/nodes.py @@ -1,5 +1,3 @@ -import math - import torch import os From a30526aec1f7c6fb563b5c49503e9d18f6fa73d2 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 25 Mar 2023 04:49:58 -0400 Subject: [PATCH 22/22] Set default value of feathering to 40. --- nodes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nodes.py b/nodes.py index 84f8b01f6..152fa32d1 100644 --- a/nodes.py +++ b/nodes.py @@ -915,11 +915,11 @@ class ImagePadForOutpaint: return { "required": { "image": ("IMAGE",), - "left": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), - "top": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), - "right": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), - "bottom": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 64}), - "feathering": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 1}), + "left": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}), + "top": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}), + "right": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}), + "bottom": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}), + "feathering": ("INT", {"default": 40, "min": 0, "max": MAX_RESOLUTION, "step": 1}), } }