Merge branch 'comfyanonymous:master' into seedControls

README.md

@@ -1,6 +1,6 @@
 ComfyUI
 =======
-A powerful and modular stable diffusion GUI.
+A powerful and modular stable diffusion GUI and backend.
 -----------
 ![ComfyUI Screenshot](comfyui_screenshot.png)
 

comfy/clip_vision.py

@@ -1,6 +1,7 @@
 from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, CLIPImageProcessor
 from .utils import load_torch_file, transformers_convert
 import os
+import torch
 
 class ClipVisionModel():
     def __init__(self, json_config):
@@ -20,7 +21,8 @@ class ClipVisionModel():
         self.model.load_state_dict(sd, strict=False)
 
     def encode_image(self, image):
-        inputs = self.processor(images=[image[0]], return_tensors="pt")
+        img = torch.clip((255. * image[0]), 0, 255).round().int()
+        inputs = self.processor(images=[img], return_tensors="pt")
         outputs = self.model(**inputs)
         return outputs
 

comfy/model_management.py

@@ -45,6 +45,8 @@ try:
 except:
     OOM_EXCEPTION = Exception
 
+XFORMERS_VERSION = ""
+XFORMERS_ENABLED_VAE = True
 if args.disable_xformers:
     XFORMERS_IS_AVAILABLE = False
 else:
@@ -52,6 +54,17 @@ else:
         import xformers
         import xformers.ops
         XFORMERS_IS_AVAILABLE = True
+        try:
+            XFORMERS_VERSION = xformers.version.__version__
+            print("xformers version:", XFORMERS_VERSION)
+            if XFORMERS_VERSION.startswith("0.0.18"):
+                print()
+                print("WARNING: This version of xformers has a major bug where you will get black images when generating high resolution images.")
+                print("Please downgrade or upgrade xformers to a different version.")
+                print()
+                XFORMERS_ENABLED_VAE = False
+        except:
+            pass
     except:
         XFORMERS_IS_AVAILABLE = False
 
@@ -223,13 +236,8 @@ def xformers_enabled_vae():
     enabled = xformers_enabled()
     if not enabled:
         return False
-    try:
-        #0.0.18 has a bug where Nan is returned when inputs are too big (1152x1920 res images and above)
-        if xformers.version.__version__ == "0.0.18":
-            return False
-    except:
-        pass
-    return enabled
+
+    return XFORMERS_ENABLED_VAE
 
 def pytorch_attention_enabled():
     return ENABLE_PYTORCH_ATTENTION

notebooks/comfyui_colab.ipynb

@@ -47,7 +47,7 @@
         "  !git pull\n",
         "\n",
         "!echo -= Install dependencies =-\n",
-        "!pip install xformers -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cu118"
+        "!pip install xformers!=0.0.18 -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cu118"
       ]
     },
     {
@@ -86,6 +86,11 @@
         "#!wget -c https://huggingface.co/waifu-diffusion/wd-1-5-beta2/resolve/main/checkpoints/wd-1-5-beta2-fp16.safetensors -P ./models/checkpoints/\n",
         "\n",
         "\n",
+        "# unCLIP models\n",
+        "#!wget -c https://huggingface.co/comfyanonymous/illuminatiDiffusionV1_v11_unCLIP/resolve/main/illuminatiDiffusionV1_v11-unclip-h-fp16.safetensors -P ./models/checkpoints/\n",
+        "#!wget -c https://huggingface.co/comfyanonymous/wd-1.5-beta2_unCLIP/resolve/main/wd-1-5-beta2-aesthetic-unclip-h-fp16.safetensors -P ./models/checkpoints/\n",
+        "\n",
+        "\n",
         "# VAE\n",
         "!wget -c https://huggingface.co/stabilityai/sd-vae-ft-mse-original/resolve/main/vae-ft-mse-840000-ema-pruned.safetensors -P ./models/vae/\n",
         "#!wget -c https://huggingface.co/WarriorMama777/OrangeMixs/resolve/main/VAEs/orangemix.vae.pt -P ./models/vae/\n",

web/extensions/core/nodeTemplates.js

@@ -0,0 +1,184 @@
+import { app } from "/scripts/app.js";
+import { ComfyDialog, $el } from "/scripts/ui.js";
+
+// Adds the ability to save and add multiple nodes as a template
+// To save:
+// Select multiple nodes (ctrl + drag to select a region or ctrl+click individual nodes)
+// Right click the canvas
+// Save Node Template -> give it a name
+//
+// To add:
+// Right click the canvas
+// Node templates -> click the one to add
+//
+// To delete/rename:
+// Right click the canvas
+// Node templates -> Manage
+
+const id = "Comfy.NodeTemplates";
+
+class ManageTemplates extends ComfyDialog {
+	constructor() {
+		super();
+		this.element.classList.add("comfy-manage-templates");
+		this.templates = this.load();
+	}
+
+	createButtons() {
+		const btns = super.createButtons();
+		btns[0].textContent = "Cancel";
+		btns.unshift(
+			$el("button", {
+				type: "button",
+				textContent: "Save",
+				onclick: () => this.save(),
+			})
+		);
+		return btns;
+	}
+
+	load() {
+		const templates = localStorage.getItem(id);
+		if (templates) {
+			return JSON.parse(templates);
+		} else {
+			return [];
+		}
+	}
+
+	save() {
+		// Find all visible inputs and save them as our new list
+		const inputs = this.element.querySelectorAll("input");
+		const updated = [];
+
+		for (let i = 0; i < inputs.length; i++) {
+			const input = inputs[i];
+			if (input.parentElement.style.display !== "none") {
+				const t = this.templates[i];
+				t.name = input.value.trim() || input.getAttribute("data-name");
+				updated.push(t);
+			}
+		}
+
+		this.templates = updated;
+		this.store();
+		this.close();
+	}
+
+	store() {
+		localStorage.setItem(id, JSON.stringify(this.templates));
+	}
+
+	show() {
+		// Show list of template names + delete button
+		super.show(
+			$el(
+				"div",
+				{
+					style: {
+						display: "grid",
+						gridTemplateColumns: "1fr auto",
+						gap: "5px",
+					},
+				},
+				this.templates.flatMap((t) => {
+					let nameInput;
+					return [
+						$el(
+							"label",
+							{
+								textContent: "Name: ",
+							},
+							[
+								$el("input", {
+									value: t.name,
+									dataset: { name: t.name },
+									$: (el) => (nameInput = el),
+								}),
+							]
+						),
+						$el("button", {
+							textContent: "Delete",
+							style: {
+								fontSize: "12px",
+								color: "red",
+								fontWeight: "normal",
+							},
+							onclick: (e) => {
+								nameInput.value = "";
+								e.target.style.display = "none";
+								e.target.previousElementSibling.style.display = "none";
+							},
+						}),
+					];
+				})
+			)
+		);
+	}
+}
+
+app.registerExtension({
+	name: id,
+	setup() {
+		const manage = new ManageTemplates();
+
+		const clipboardAction = (cb) => {
+			// We use the clipboard functions but dont want to overwrite the current user clipboard
+			// Restore it after we've run our callback
+			const old = localStorage.getItem("litegrapheditor_clipboard");
+			cb();
+			localStorage.setItem("litegrapheditor_clipboard", old);
+		};
+
+		const orig = LGraphCanvas.prototype.getCanvasMenuOptions;
+		LGraphCanvas.prototype.getCanvasMenuOptions = function () {
+			const options = orig.apply(this, arguments);
+
+			options.push(null);
+			options.push({
+				content: `Save Selected as Template`,
+				disabled: !Object.keys(app.canvas.selected_nodes || {}).length,
+				callback: () => {
+					const name = prompt("Enter name");
+					if (!name || !name.trim()) return;
+
+					clipboardAction(() => {
+						app.canvas.copyToClipboard();
+						manage.templates.push({
+							name,
+							data: localStorage.getItem("litegrapheditor_clipboard"),
+						});
+						manage.store();
+					});
+				},
+			});
+
+			// Map each template to a menu item
+			const subItems = manage.templates.map((t) => ({
+				content: t.name,
+				callback: () => {
+					clipboardAction(() => {
+						localStorage.setItem("litegrapheditor_clipboard", t.data);
+						app.canvas.pasteFromClipboard();
+					});
+				},
+			}));
+
+			if (subItems.length) {
+				subItems.push(null, {
+					content: "Manage",
+					callback: () => manage.show(),
+				});
+
+				options.push({
+					content: "Node Templates",
+					submenu: {
+						options: subItems,
+					},
+				});
+			}
+
+			return options;
+		};
+	},
+});

web/scripts/app.js

@@ -1,5 +1,5 @@
 import { ComfyWidgets } from "./widgets.js";
-import { ComfyUI } from "./ui.js";
+import { ComfyUI, $el } from "./ui.js";
 import { api } from "./api.js";
 import { defaultGraph } from "./defaultGraph.js";
 import { getPngMetadata, importA1111 } from "./pnginfo.js";
@@ -864,12 +864,62 @@ class ComfyApp {
 			graphData = structuredClone(defaultGraph);
 		}
 
-		// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
+		const missingNodeTypes = [];
 		for (let n of graphData.nodes) {
+			// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
 			if (n.type == "T2IAdapterLoader") n.type = "ControlNetLoader";
+
+			// Find missing node types
+			if (!(n.type in LiteGraph.registered_node_types)) {
+				missingNodeTypes.push(n.type);
+			}
 		}
 
-		this.graph.configure(graphData);
+		try {
+			this.graph.configure(graphData);
+		} catch (error) {
+			let errorHint = [];
+			// Try extracting filename to see if it was caused by an extension script
+			const filename = error.fileName || (error.stack || "").match(/(\/extensions\/.*\.js)/)?.[1];
+			const pos = (filename || "").indexOf("/extensions/");
+			if (pos > -1) {
+				errorHint.push(
+					$el("span", { textContent: "This may be due to the following script:" }),
+					$el("br"),
+					$el("span", {
+						style: {
+							fontWeight: "bold",
+						},
+						textContent: filename.substring(pos),
+					})
+				);
+			}
+
+			// Show dialog to let the user know something went wrong loading the data
+			this.ui.dialog.show(
+				$el("div", [
+					$el("p", { textContent: "Loading aborted due to error reloading workflow data" }),
+					$el("pre", {
+						style: { padding: "5px", backgroundColor: "rgba(255,0,0,0.2)" },
+						textContent: error.toString(),
+					}),
+					$el("pre", {
+						style: {
+							padding: "5px",
+							color: "#ccc",
+							fontSize: "10px",
+							maxHeight: "50vh",
+							overflow: "auto",
+							backgroundColor: "rgba(0,0,0,0.2)",
+						},
+						textContent: error.stack || "No stacktrace available",
+					}),
+					...errorHint,
+				]).outerHTML
+			);
+
+			return;
+		}
 
 		for (const node of this.graph._nodes) {
 			const size = node.computeSize();
@@ -897,6 +947,14 @@ class ComfyApp {
 				this.#invokeExtensions("loadedGraphNode", node);
 			}
 		}
+
+		if (missingNodeTypes.length) {
+			this.ui.dialog.show(
+				`When loading the graph, the following node types were not found: <ul>${Array.from(new Set(missingNodeTypes)).map(
+					(t) => `<li>${t}</li>`
+				).join("")}</ul>Nodes that have failed to load will show as red on the graph.`
+			);
+		}
 	}
 
 	/**

web/scripts/pnginfo.js

@@ -32,8 +32,9 @@ export function getPngMetadata(file) {
 					}
 					const keyword = String.fromCharCode(...pngData.slice(offset + 8, keyword_end));
 					// Get the text
-					const text = String.fromCharCode(...pngData.slice(keyword_end + 1, offset + 8 + length));
-					txt_chunks[keyword] = text;
+					const contentArraySegment = pngData.slice(keyword_end + 1, offset + 8 + length);
+					const contentJson = Array.from(contentArraySegment).map(s=>String.fromCharCode(s)).join('')
+					txt_chunks[keyword] = contentJson;
 				}
 
 				offset += 12 + length;

web/scripts/ui.js

@@ -8,14 +8,18 @@ export function $el(tag, propsOrChildren, children) {
 		if (Array.isArray(propsOrChildren)) {
 			element.append(...propsOrChildren);
 		} else {
-			const parent = propsOrChildren.parent;
+			const { parent, $: cb, dataset, style } = propsOrChildren;
 			delete propsOrChildren.parent;
-			const cb = propsOrChildren.$;
 			delete propsOrChildren.$;
+			delete propsOrChildren.dataset;
+			delete propsOrChildren.style;
 
-			if (propsOrChildren.style) {
-				Object.assign(element.style, propsOrChildren.style);
-				delete propsOrChildren.style;
+			if (style) {
+				Object.assign(element.style, style);
+			}
+
+			if (dataset) {
+				Object.assign(element.dataset, dataset);
 			}
 
 			Object.assign(element, propsOrChildren);
@@ -76,7 +80,7 @@ function dragElement(dragEl, settings) {
 			dragEl.style.left = newPosX + "px";
 			dragEl.style.right = "unset";
 		}
-		
+
 		dragEl.style.top = newPosY + "px";
 		dragEl.style.bottom = "unset";
 
@@ -145,7 +149,7 @@ function dragElement(dragEl, settings) {
 	}
 
 	window.addEventListener("resize", () => {
-			ensureInBounds();
+		ensureInBounds();
 	});
 
 	function closeDragElement() {
@@ -155,26 +159,33 @@ function dragElement(dragEl, settings) {
 	}
 }
 
-class ComfyDialog {
+export class ComfyDialog {
 	constructor() {
 		this.element = $el("div.comfy-modal", { parent: document.body }, [
-			$el("div.comfy-modal-content", [
-				$el("p", { $: (p) => (this.textElement = p) }),
-				$el("button", {
-					type: "button",
-					textContent: "Close",
-					onclick: () => this.close(),
-				}),
-			]),
+			$el("div.comfy-modal-content", [$el("p", { $: (p) => (this.textElement = p) }), ...this.createButtons()]),
 		]);
 	}
 
+	createButtons() {
+		return [
+			$el("button", {
+				type: "button",
+				textContent: "Close",
+				onclick: () => this.close(),
+			}),
+		];
+	}
+
 	close() {
 		this.element.style.display = "none";
 	}
 
 	show(html) {
-		this.textElement.innerHTML = html;
+		if (typeof html === "string") {
+			this.textElement.innerHTML = html;
+		} else {
+			this.textElement.replaceChildren(html);
+		}
 		this.element.style.display = "flex";
 	}
 }
@@ -419,7 +430,7 @@ export class ComfyUI {
 			type: "boolean",
 			defaultValue: true,
 		});
-		
+
 		const fileInput = $el("input", {
 			type: "file",
 			accept: ".json,image/png",

web/style.css

@@ -202,7 +202,8 @@ button.comfy-queue-btn {
 	margin: 6px 0 !important;
 }
 
-.comfy-modal.comfy-settings {
+.comfy-modal.comfy-settings, 
+.comfy-modal.comfy-manage-templates {
 	text-align: center;
 	font-family: sans-serif;
 	color: #999;
@@ -256,3 +257,16 @@ button.comfy-queue-btn {
 	color: #ddd;
 	border-radius: 12px 0 0 12px;
 }
+
+.litegraph .litemenu-entry.has_submenu {
+	position: relative;
+	padding-right: 20px;
+ }
+
+ .litemenu-entry.has_submenu::after {
+	content: ">";
+	position: absolute;
+	top: 0;
+	right: 2px;
+ }
+