Merge branch 'comfyanonymous:master' into bugfix/extra_data

comfy/sd.py

@@ -479,8 +479,8 @@ class CLIP:
     def load_from_state_dict(self, sd):
         self.cond_stage_model.load_sd(sd)
 
-    def add_patches(self, patches, strength=1.0):
+    def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):
-        return self.patcher.add_patches(patches, strength)
+        return self.patcher.add_patches(patches, strength_patch, strength_model)
 
     def clip_layer(self, layer_idx):
         self.layer_idx = layer_idx
@@ -514,6 +514,9 @@ class CLIP:
     def unpatch_model(self):
         self.patcher.unpatch_model()
 
+    def get_key_patches(self):
+        return self.patcher.get_key_patches()
+
 class VAE:
     def __init__(self, ckpt_path=None, device=None, config=None):
         if config is None:

comfy_extras/nodes_model_merging.py

@@ -23,6 +23,27 @@ class ModelMergeSimple:
             m.add_patches({k: kp[k]}, 1.0 - ratio, ratio)
         return (m, )
 
+class CLIPMergeSimple:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "clip1": ("CLIP",),
+                              "clip2": ("CLIP",),
+                              "ratio": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+                              }}
+    RETURN_TYPES = ("CLIP",)
+    FUNCTION = "merge"
+
+    CATEGORY = "advanced/model_merging"
+
+    def merge(self, clip1, clip2, ratio):
+        m = clip1.clone()
+        kp = clip2.get_key_patches()
+        for k in kp:
+            if k.endswith(".position_ids") or k.endswith(".logit_scale"):
+                continue
+            m.add_patches({k: kp[k]}, 1.0 - ratio, ratio)
+        return (m, )
+
 class ModelMergeBlocks:
     @classmethod
     def INPUT_TYPES(s):
@@ -94,4 +115,5 @@ NODE_CLASS_MAPPINGS = {
     "ModelMergeSimple": ModelMergeSimple,
     "ModelMergeBlocks": ModelMergeBlocks,
     "CheckpointSave": CheckpointSave,
+    "CLIPMergeSimple": CLIPMergeSimple,
 }

web/extensions/core/clipspace.js

@@ -1,6 +1,6 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
-import { ComfyDialog, $el } from "/scripts/ui.js";
+import { ComfyDialog, $el } from "../../scripts/ui.js";
-import { ComfyApp } from "/scripts/app.js";
+import { ComfyApp } from "../../scripts/app.js";
 
 export class ClipspaceDialog extends ComfyDialog {
 	static items = [];

web/extensions/core/colorPalette.js

@@ -1,5 +1,5 @@
-import {app} from "/scripts/app.js";
+import {app} from "../../scripts/app.js";
-import {$el} from "/scripts/ui.js";
+import {$el} from "../../scripts/ui.js";
 
 // Manage color palettes
 

web/extensions/core/editAttention.js

@@ -1,4 +1,4 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
 
 // Allows you to edit the attention weight by holding ctrl (or cmd) and using the up/down arrow keys
 

web/extensions/core/invertMenuScrolling.js

@@ -1,4 +1,4 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
 
 // Inverts the scrolling of context menus
 

web/extensions/core/keybinds.js

@@ -1,4 +1,4 @@
-import {app} from "/scripts/app.js";
+import {app} from "../../scripts/app.js";
 
 app.registerExtension({
 	name: "Comfy.Keybinds",

web/extensions/core/maskeditor.js

@@ -1,7 +1,8 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
-import { ComfyDialog, $el } from "/scripts/ui.js";
+import { ComfyDialog, $el } from "../../scripts/ui.js";
-import { ComfyApp } from "/scripts/app.js";
+import { ComfyApp } from "../../scripts/app.js";
-import { ClipspaceDialog } from "/extensions/core/clipspace.js";
+import { api } from "../../scripts/api.js"
+import { ClipspaceDialog } from "./clipspace.js";
 
 // Helper function to convert a data URL to a Blob object
 function dataURLToBlob(dataURL) {
@@ -33,7 +34,7 @@ function loadedImageToBlob(image) {
 }
 
 async function uploadMask(filepath, formData) {
-	await fetch('/upload/mask', {
+	await api.fetchApi('/upload/mask', {
 		method: 'POST',
 		body: formData
 	}).then(response => {}).catch(error => {
@@ -41,7 +42,7 @@ async function uploadMask(filepath, formData) {
 	});
 
 	ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']] = new Image();
-	ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = "/view?" + new URLSearchParams(filepath).toString() + app.getPreviewFormatParam();
+	ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = api.apiURL("/view?" + new URLSearchParams(filepath).toString() + app.getPreviewFormatParam());
 
 	if(ComfyApp.clipspace.images)
 		ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']] = filepath;

web/extensions/core/nodeTemplates.js

@@ -1,5 +1,5 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
-import { ComfyDialog, $el } from "/scripts/ui.js";
+import { ComfyDialog, $el } from "../../scripts/ui.js";
 
 // Adds the ability to save and add multiple nodes as a template
 // To save:

web/extensions/core/saveImageExtraOutput.js

@@ -1,4 +1,4 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
 
 // Use widget values and dates in output filenames
 

web/extensions/core/slotDefaults.js

@@ -1,5 +1,5 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
-import { ComfyWidgets } from "/scripts/widgets.js";
+import { ComfyWidgets } from "../../scripts/widgets.js";
 // Adds defaults for quickly adding nodes with middle click on the input/output
 
 app.registerExtension({

web/extensions/core/snapToGrid.js

@@ -1,4 +1,4 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
 
 // Shift + drag/resize to snap to grid
 

web/extensions/core/uploadImage.js

@@ -1,4 +1,4 @@
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
 
 // Adds an upload button to the nodes
 

web/extensions/core/widgetInputs.js

@@ -1,5 +1,5 @@
-import { ComfyWidgets, addValueControlWidget } from "/scripts/widgets.js";
+import { ComfyWidgets, addValueControlWidget } from "../../scripts/widgets.js";
-import { app } from "/scripts/app.js";
+import { app } from "../../scripts/app.js";
 
 const CONVERTED_TYPE = "converted-widget";
 const VALID_TYPES = ["STRING", "combo", "number"];

web/index.html

@@ -4,12 +4,12 @@
 		<meta charset="UTF-8">
 		<title>ComfyUI</title>
 		<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
-		<link rel="stylesheet" type="text/css" href="lib/litegraph.css" />
+		<link rel="stylesheet" type="text/css" href="./lib/litegraph.css" />
-		<link rel="stylesheet" type="text/css" href="style.css" />
+		<link rel="stylesheet" type="text/css" href="./style.css" />
-		<script type="text/javascript" src="lib/litegraph.core.js"></script>
+		<script type="text/javascript" src="./lib/litegraph.core.js"></script>
-		<script type="text/javascript" src="lib/litegraph.extensions.js" defer></script>
+		<script type="text/javascript" src="./lib/litegraph.extensions.js" defer></script>
 		<script type="module">
-			import { app } from "/scripts/app.js";
+			import { app } from "./scripts/app.js";
 			await app.setup();
 			window.app = app;
 			window.graph = app.graph;

web/scripts/api.js

@@ -3,6 +3,16 @@ class ComfyApi extends EventTarget {
 
 	constructor() {
 		super();
+		this.api_host = location.host;
+		this.api_base = location.pathname.split('/').slice(0, -1).join('/');
+	}
+
+	apiURL(route) {
+		return this.api_base + route;
+	}
+
+	fetchApi(route, options) {
+		return fetch(this.apiURL(route), options);
 	}
 
 	addEventListener(type, callback, options) {
@@ -16,7 +26,7 @@ class ComfyApi extends EventTarget {
 	#pollQueue() {
 		setInterval(async () => {
 			try {
-				const resp = await fetch("/prompt");
+				const resp = await this.fetchApi("/prompt");
 				const status = await resp.json();
 				this.dispatchEvent(new CustomEvent("status", { detail: status }));
 			} catch (error) {
@@ -40,7 +50,7 @@ class ComfyApi extends EventTarget {
 			existingSession = "?clientId=" + existingSession;
 		}
 		this.socket = new WebSocket(
-			`ws${window.location.protocol === "https:" ? "s" : ""}://${location.host}/ws${existingSession}`
+			`ws${window.location.protocol === "https:" ? "s" : ""}://${this.api_host}${this.api_base}/ws${existingSession}`
 		);
 		this.socket.binaryType = "arraybuffer";
 
@@ -149,7 +159,7 @@ class ComfyApi extends EventTarget {
 	 * @returns An array of script urls to import
 	 */
 	async getExtensions() {
-		const resp = await fetch("/extensions", { cache: "no-store" });
+		const resp = await this.fetchApi("/extensions", { cache: "no-store" });
 		return await resp.json();
 	}
 
@@ -158,7 +168,7 @@ class ComfyApi extends EventTarget {
 	 * @returns An array of script urls to import
 	 */
 	async getEmbeddings() {
-		const resp = await fetch("/embeddings", { cache: "no-store" });
+		const resp = await this.fetchApi("/embeddings", { cache: "no-store" });
 		return await resp.json();
 	}
 
@@ -167,7 +177,7 @@ class ComfyApi extends EventTarget {
 	 * @returns The node definitions
 	 */
 	async getNodeDefs() {
-		const resp = await fetch("object_info", { cache: "no-store" });
+		const resp = await this.fetchApi("/object_info", { cache: "no-store" });
 		return await resp.json();
 	}
 
@@ -189,7 +199,7 @@ class ComfyApi extends EventTarget {
 			body.number = number;
 		}
 
-		const res = await fetch("/prompt", {
+		const res = await this.fetchApi("/prompt", {
 			method: "POST",
 			headers: {
 				"Content-Type": "application/json",
@@ -224,7 +234,7 @@ class ComfyApi extends EventTarget {
 	 */
 	async getQueue() {
 		try {
-			const res = await fetch("/queue");
+			const res = await this.fetchApi("/queue");
 			const data = await res.json();
 			return {
 				// Running action uses a different endpoint for cancelling
@@ -246,7 +256,7 @@ class ComfyApi extends EventTarget {
 	 */
 	async getHistory() {
 		try {
-			const res = await fetch("/history");
+			const res = await this.fetchApi("/history");
 			return { History: Object.values(await res.json()) };
 		} catch (error) {
 			console.error(error);
@@ -261,7 +271,7 @@ class ComfyApi extends EventTarget {
 	 */
 	async #postItem(type, body) {
 		try {
-			await fetch("/" + type, {
+			await this.fetchApi("/" + type, {
 				method: "POST",
 				headers: {
 					"Content-Type": "application/json",

web/scripts/app.js

@@ -404,7 +404,7 @@ export class ComfyApp {
 						this.images = output.images;
 						imagesChanged = true;
 						imgURLs = imgURLs.concat(output.images.map(params => {
-							return "/view?" + new URLSearchParams(params).toString() + app.getPreviewFormatParam();
+							return api.apiURL("/view?" + new URLSearchParams(params).toString() + app.getPreviewFormatParam());
 						}))
 					}
 				}
@@ -1005,7 +1005,7 @@ export class ComfyApp {
 		const extensions = await api.getExtensions();
 		for (const ext of extensions) {
 			try {
-				await import(ext);
+				await import(api.apiURL(ext));
 			} catch (error) {
 				console.error("Error loading extension", ext, error);
 			}

web/scripts/widgets.js

@@ -1,3 +1,5 @@
+import { api } from "./api.js"
+
 function getNumberDefaults(inputData, defaultStep) {
 	let defaultVal = inputData[1]["default"];
 	let { min, max, step } = inputData[1];
@@ -305,7 +307,7 @@ export const ComfyWidgets = {
 				subfolder = name.substring(0, folder_separator);
 				name = name.substring(folder_separator + 1);
 			}
-			img.src = `/view?filename=${name}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}`;
+			img.src = api.apiURL(`/view?filename=${name}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}`);
 			node.setSizeForImage?.();
 		}
 
@@ -362,7 +364,7 @@ export const ComfyWidgets = {
 				// Wrap file in formdata so it includes filename
 				const body = new FormData();
 				body.append("image", file);
-				const resp = await fetch("/upload/image", {
+				const resp = await api.fetchApi("/upload/image", {
 					method: "POST",
 					body,
 				});

web/types/comfy.d.ts

@@ -1,5 +1,5 @@
 import { LGraphNode, IWidget } from "./litegraph";
-import { ComfyApp } from "/scripts/app";
+import { ComfyApp } from "../../scripts/app";
 
 export interface ComfyExtension {
 	/**