diff --git a/README.md b/README.md index 90931141d..42f922622 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ComfyUI ======= -A powerful and modular stable diffusion GUI. +A powerful and modular stable diffusion GUI and backend. ----------- ![ComfyUI Screenshot](comfyui_screenshot.png) diff --git a/comfy/clip_vision.py b/comfy/clip_vision.py index cb29df432..efb2d5384 100644 --- a/comfy/clip_vision.py +++ b/comfy/clip_vision.py @@ -1,6 +1,7 @@ from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, CLIPImageProcessor from .utils import load_torch_file, transformers_convert import os +import torch class ClipVisionModel(): def __init__(self, json_config): @@ -20,7 +21,8 @@ class ClipVisionModel(): self.model.load_state_dict(sd, strict=False) def encode_image(self, image): - inputs = self.processor(images=[image[0]], return_tensors="pt") + img = torch.clip((255. * image[0]), 0, 255).round().int() + inputs = self.processor(images=[img], return_tensors="pt") outputs = self.model(**inputs) return outputs diff --git a/comfy/model_management.py b/comfy/model_management.py index 2407140fd..8303cb437 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -45,6 +45,8 @@ try: except: OOM_EXCEPTION = Exception +XFORMERS_VERSION = "" +XFORMERS_ENABLED_VAE = True if args.disable_xformers: XFORMERS_IS_AVAILABLE = False else: @@ -52,6 +54,17 @@ else: import xformers import xformers.ops XFORMERS_IS_AVAILABLE = True + try: + XFORMERS_VERSION = xformers.version.__version__ + print("xformers version:", XFORMERS_VERSION) + if XFORMERS_VERSION.startswith("0.0.18"): + print() + print("WARNING: This version of xformers has a major bug where you will get black images when generating high resolution images.") + print("Please downgrade or upgrade xformers to a different version.") + print() + XFORMERS_ENABLED_VAE = False + except: + pass except: XFORMERS_IS_AVAILABLE = False @@ -223,13 +236,8 @@ def xformers_enabled_vae(): enabled = xformers_enabled() if not enabled: return False - try: - #0.0.18 has a bug where Nan is returned when inputs are too big (1152x1920 res images and above) - if xformers.version.__version__ == "0.0.18": - return False - except: - pass - return enabled + + return XFORMERS_ENABLED_VAE def pytorch_attention_enabled(): return ENABLE_PYTORCH_ATTENTION diff --git a/comfyui_screenshot.png b/comfyui_screenshot.png index c357e2439..73272eae6 100644 Binary files a/comfyui_screenshot.png and b/comfyui_screenshot.png differ diff --git a/notebooks/comfyui_colab.ipynb b/notebooks/comfyui_colab.ipynb index 3e59fbde7..071a89969 100644 --- a/notebooks/comfyui_colab.ipynb +++ b/notebooks/comfyui_colab.ipynb @@ -47,7 +47,7 @@ " !git pull\n", "\n", "!echo -= Install dependencies =-\n", - "!pip install xformers -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cu118" + "!pip install xformers!=0.0.18 -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cu118" ] }, { @@ -86,6 +86,11 @@ "#!wget -c https://huggingface.co/waifu-diffusion/wd-1-5-beta2/resolve/main/checkpoints/wd-1-5-beta2-fp16.safetensors -P ./models/checkpoints/\n", "\n", "\n", + "# unCLIP models\n", + "#!wget -c https://huggingface.co/comfyanonymous/illuminatiDiffusionV1_v11_unCLIP/resolve/main/illuminatiDiffusionV1_v11-unclip-h-fp16.safetensors -P ./models/checkpoints/\n", + "#!wget -c https://huggingface.co/comfyanonymous/wd-1.5-beta2_unCLIP/resolve/main/wd-1-5-beta2-aesthetic-unclip-h-fp16.safetensors -P ./models/checkpoints/\n", + "\n", + "\n", "# VAE\n", "!wget -c https://huggingface.co/stabilityai/sd-vae-ft-mse-original/resolve/main/vae-ft-mse-840000-ema-pruned.safetensors -P ./models/vae/\n", "#!wget -c https://huggingface.co/WarriorMama777/OrangeMixs/resolve/main/VAEs/orangemix.vae.pt -P ./models/vae/\n", diff --git a/web/extensions/core/nodeTemplates.js b/web/extensions/core/nodeTemplates.js new file mode 100644 index 000000000..69d09cde8 --- /dev/null +++ b/web/extensions/core/nodeTemplates.js @@ -0,0 +1,184 @@ +import { app } from "/scripts/app.js"; +import { ComfyDialog, $el } from "/scripts/ui.js"; + +// Adds the ability to save and add multiple nodes as a template +// To save: +// Select multiple nodes (ctrl + drag to select a region or ctrl+click individual nodes) +// Right click the canvas +// Save Node Template -> give it a name +// +// To add: +// Right click the canvas +// Node templates -> click the one to add +// +// To delete/rename: +// Right click the canvas +// Node templates -> Manage + +const id = "Comfy.NodeTemplates"; + +class ManageTemplates extends ComfyDialog { + constructor() { + super(); + this.element.classList.add("comfy-manage-templates"); + this.templates = this.load(); + } + + createButtons() { + const btns = super.createButtons(); + btns[0].textContent = "Cancel"; + btns.unshift( + $el("button", { + type: "button", + textContent: "Save", + onclick: () => this.save(), + }) + ); + return btns; + } + + load() { + const templates = localStorage.getItem(id); + if (templates) { + return JSON.parse(templates); + } else { + return []; + } + } + + save() { + // Find all visible inputs and save them as our new list + const inputs = this.element.querySelectorAll("input"); + const updated = []; + + for (let i = 0; i < inputs.length; i++) { + const input = inputs[i]; + if (input.parentElement.style.display !== "none") { + const t = this.templates[i]; + t.name = input.value.trim() || input.getAttribute("data-name"); + updated.push(t); + } + } + + this.templates = updated; + this.store(); + this.close(); + } + + store() { + localStorage.setItem(id, JSON.stringify(this.templates)); + } + + show() { + // Show list of template names + delete button + super.show( + $el( + "div", + { + style: { + display: "grid", + gridTemplateColumns: "1fr auto", + gap: "5px", + }, + }, + this.templates.flatMap((t) => { + let nameInput; + return [ + $el( + "label", + { + textContent: "Name: ", + }, + [ + $el("input", { + value: t.name, + dataset: { name: t.name }, + $: (el) => (nameInput = el), + }), + ] + ), + $el("button", { + textContent: "Delete", + style: { + fontSize: "12px", + color: "red", + fontWeight: "normal", + }, + onclick: (e) => { + nameInput.value = ""; + e.target.style.display = "none"; + e.target.previousElementSibling.style.display = "none"; + }, + }), + ]; + }) + ) + ); + } +} + +app.registerExtension({ + name: id, + setup() { + const manage = new ManageTemplates(); + + const clipboardAction = (cb) => { + // We use the clipboard functions but dont want to overwrite the current user clipboard + // Restore it after we've run our callback + const old = localStorage.getItem("litegrapheditor_clipboard"); + cb(); + localStorage.setItem("litegrapheditor_clipboard", old); + }; + + const orig = LGraphCanvas.prototype.getCanvasMenuOptions; + LGraphCanvas.prototype.getCanvasMenuOptions = function () { + const options = orig.apply(this, arguments); + + options.push(null); + options.push({ + content: `Save Selected as Template`, + disabled: !Object.keys(app.canvas.selected_nodes || {}).length, + callback: () => { + const name = prompt("Enter name"); + if (!name || !name.trim()) return; + + clipboardAction(() => { + app.canvas.copyToClipboard(); + manage.templates.push({ + name, + data: localStorage.getItem("litegrapheditor_clipboard"), + }); + manage.store(); + }); + }, + }); + + // Map each template to a menu item + const subItems = manage.templates.map((t) => ({ + content: t.name, + callback: () => { + clipboardAction(() => { + localStorage.setItem("litegrapheditor_clipboard", t.data); + app.canvas.pasteFromClipboard(); + }); + }, + })); + + if (subItems.length) { + subItems.push(null, { + content: "Manage", + callback: () => manage.show(), + }); + + options.push({ + content: "Node Templates", + submenu: { + options: subItems, + }, + }); + } + + return options; + }; + }, +}); diff --git a/web/scripts/app.js b/web/scripts/app.js index dd9c21f89..5aaafd46d 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -1,5 +1,5 @@ import { ComfyWidgets } from "./widgets.js"; -import { ComfyUI } from "./ui.js"; +import { ComfyUI, $el } from "./ui.js"; import { api } from "./api.js"; import { defaultGraph } from "./defaultGraph.js"; import { getPngMetadata, importA1111 } from "./pnginfo.js"; @@ -864,12 +864,62 @@ class ComfyApp { graphData = structuredClone(defaultGraph); } - // Patch T2IAdapterLoader to ControlNetLoader since they are the same node now + const missingNodeTypes = []; for (let n of graphData.nodes) { + // Patch T2IAdapterLoader to ControlNetLoader since they are the same node now if (n.type == "T2IAdapterLoader") n.type = "ControlNetLoader"; + + // Find missing node types + if (!(n.type in LiteGraph.registered_node_types)) { + missingNodeTypes.push(n.type); + } } - this.graph.configure(graphData); + try { + this.graph.configure(graphData); + } catch (error) { + let errorHint = []; + // Try extracting filename to see if it was caused by an extension script + const filename = error.fileName || (error.stack || "").match(/(\/extensions\/.*\.js)/)?.[1]; + const pos = (filename || "").indexOf("/extensions/"); + if (pos > -1) { + errorHint.push( + $el("span", { textContent: "This may be due to the following script:" }), + $el("br"), + $el("span", { + style: { + fontWeight: "bold", + }, + textContent: filename.substring(pos), + }) + ); + } + + // Show dialog to let the user know something went wrong loading the data + this.ui.dialog.show( + $el("div", [ + $el("p", { textContent: "Loading aborted due to error reloading workflow data" }), + $el("pre", { + style: { padding: "5px", backgroundColor: "rgba(255,0,0,0.2)" }, + textContent: error.toString(), + }), + $el("pre", { + style: { + padding: "5px", + color: "#ccc", + fontSize: "10px", + maxHeight: "50vh", + overflow: "auto", + backgroundColor: "rgba(0,0,0,0.2)", + }, + textContent: error.stack || "No stacktrace available", + }), + ...errorHint, + ]).outerHTML + ); + + return; + } for (const node of this.graph._nodes) { const size = node.computeSize(); @@ -897,6 +947,14 @@ class ComfyApp { this.#invokeExtensions("loadedGraphNode", node); } } + + if (missingNodeTypes.length) { + this.ui.dialog.show( + `When loading the graph, the following node types were not found: Nodes that have failed to load will show as red on the graph.` + ); + } } /** diff --git a/web/scripts/pnginfo.js b/web/scripts/pnginfo.js index 580030d81..31f470739 100644 --- a/web/scripts/pnginfo.js +++ b/web/scripts/pnginfo.js @@ -32,8 +32,9 @@ export function getPngMetadata(file) { } const keyword = String.fromCharCode(...pngData.slice(offset + 8, keyword_end)); // Get the text - const text = String.fromCharCode(...pngData.slice(keyword_end + 1, offset + 8 + length)); - txt_chunks[keyword] = text; + const contentArraySegment = pngData.slice(keyword_end + 1, offset + 8 + length); + const contentJson = Array.from(contentArraySegment).map(s=>String.fromCharCode(s)).join('') + txt_chunks[keyword] = contentJson; } offset += 12 + length; diff --git a/web/scripts/ui.js b/web/scripts/ui.js index 6999c0a73..09861c440 100644 --- a/web/scripts/ui.js +++ b/web/scripts/ui.js @@ -8,14 +8,18 @@ export function $el(tag, propsOrChildren, children) { if (Array.isArray(propsOrChildren)) { element.append(...propsOrChildren); } else { - const parent = propsOrChildren.parent; + const { parent, $: cb, dataset, style } = propsOrChildren; delete propsOrChildren.parent; - const cb = propsOrChildren.$; delete propsOrChildren.$; + delete propsOrChildren.dataset; + delete propsOrChildren.style; - if (propsOrChildren.style) { - Object.assign(element.style, propsOrChildren.style); - delete propsOrChildren.style; + if (style) { + Object.assign(element.style, style); + } + + if (dataset) { + Object.assign(element.dataset, dataset); } Object.assign(element, propsOrChildren); @@ -76,7 +80,7 @@ function dragElement(dragEl, settings) { dragEl.style.left = newPosX + "px"; dragEl.style.right = "unset"; } - + dragEl.style.top = newPosY + "px"; dragEl.style.bottom = "unset"; @@ -145,7 +149,7 @@ function dragElement(dragEl, settings) { } window.addEventListener("resize", () => { - ensureInBounds(); + ensureInBounds(); }); function closeDragElement() { @@ -155,26 +159,33 @@ function dragElement(dragEl, settings) { } } -class ComfyDialog { +export class ComfyDialog { constructor() { this.element = $el("div.comfy-modal", { parent: document.body }, [ - $el("div.comfy-modal-content", [ - $el("p", { $: (p) => (this.textElement = p) }), - $el("button", { - type: "button", - textContent: "Close", - onclick: () => this.close(), - }), - ]), + $el("div.comfy-modal-content", [$el("p", { $: (p) => (this.textElement = p) }), ...this.createButtons()]), ]); } + createButtons() { + return [ + $el("button", { + type: "button", + textContent: "Close", + onclick: () => this.close(), + }), + ]; + } + close() { this.element.style.display = "none"; } show(html) { - this.textElement.innerHTML = html; + if (typeof html === "string") { + this.textElement.innerHTML = html; + } else { + this.textElement.replaceChildren(html); + } this.element.style.display = "flex"; } } @@ -419,7 +430,7 @@ export class ComfyUI { type: "boolean", defaultValue: true, }); - + const fileInput = $el("input", { type: "file", accept: ".json,image/png", diff --git a/web/style.css b/web/style.css index 27bb83bb3..d00a2fbe2 100644 --- a/web/style.css +++ b/web/style.css @@ -202,7 +202,8 @@ button.comfy-queue-btn { margin: 6px 0 !important; } -.comfy-modal.comfy-settings { +.comfy-modal.comfy-settings, +.comfy-modal.comfy-manage-templates { text-align: center; font-family: sans-serif; color: #999; @@ -256,3 +257,16 @@ button.comfy-queue-btn { color: #ddd; border-radius: 12px 0 0 12px; } + +.litegraph .litemenu-entry.has_submenu { + position: relative; + padding-right: 20px; + } + + .litemenu-entry.has_submenu::after { + content: ">"; + position: absolute; + top: 0; + right: 2px; + } + \ No newline at end of file