From d712193885ef69076f79dade517b7e2a6a1a482d Mon Sep 17 00:00:00 2001 From: FuamiCake Date: Tue, 1 Aug 2023 01:23:14 -0500 Subject: [PATCH 01/11] Add LatentBlend node, allowing for blending between two Latent inputs. --- nodes.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/nodes.py b/nodes.py index 097f92308..86aed032e 100644 --- a/nodes.py +++ b/nodes.py @@ -1055,6 +1055,48 @@ class LatentComposite: samples_out["samples"] = s return (samples_out,) +class LatentBlend: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "samples_a": ("LATENT",), + "samples_b": ("LATENT",), + "blend_factor": ("FLOAT", { + "default": 0.5, + "min": 0, + "max": 1, + "step": 0.01 + }), + "blend_mode": (["normal"],), + }} + + RETURN_TYPES = ("LATENT",) + FUNCTION = "blend" + + CATEGORY = "_for_testing" + + def blend(self, samples_a, samples_b, blend_factor:float, blend_mode: str): + + samples_out = samples_a.copy() + samples_a = samples_a["samples"] + samples_b = samples_b["samples"] + + if samples_a.shape != samples_b.shape: + samples_b.permute(0, 3, 1, 2) + samples_b = comfy.utils.common_upscale(samples_b, samples_a.shape[3], samples_a.shape[2], 'bicubic', crop='center') + samples_b.permute(0, 2, 3, 1) + + samples_blended = self.blend_mode(samples_a, samples_b, blend_mode) + samples_blended = samples_a * (1 - blend_factor) + samples_blended * blend_factor + samples_out["samples"] = samples_blended + return (samples_out,) + + def blend_mode(self, img1, img2, mode): + if mode == "normal": + return img2 + else: + raise ValueError(f"Unsupported blend mode: {mode}") + class LatentCrop: @classmethod def INPUT_TYPES(s): @@ -1501,6 +1543,7 @@ NODE_CLASS_MAPPINGS = { "KSamplerAdvanced": KSamplerAdvanced, "SetLatentNoiseMask": SetLatentNoiseMask, "LatentComposite": LatentComposite, + "LatentBlend": LatentBlend, "LatentRotate": LatentRotate, "LatentFlip": LatentFlip, "LatentCrop": LatentCrop, @@ -1572,6 +1615,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LatentUpscale": "Upscale Latent", "LatentUpscaleBy": "Upscale Latent By", "LatentComposite": "Latent Composite", + "LatentBlend": "Latent Blend", "LatentFromBatch" : "Latent From Batch", "RepeatLatentBatch": "Repeat Latent Batch", # Image From d1347544bcb66bd87618164af13d9d300aefa200 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 3 Aug 2023 16:51:37 -0400 Subject: [PATCH 02/11] Make context menu filter import from relative path. --- web/extensions/core/contextMenuFilter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/extensions/core/contextMenuFilter.js b/web/extensions/core/contextMenuFilter.js index 662d87e74..e0e8854b3 100644 --- a/web/extensions/core/contextMenuFilter.js +++ b/web/extensions/core/contextMenuFilter.js @@ -1,4 +1,4 @@ -import {app} from "/scripts/app.js"; +import {app} from "../../scripts/app.js"; // Adds filtering to combo context menus From 9534f0f8a5a026654492da378f84d2cdc589ed01 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" <128333288+ltdrdata@users.noreply.github.com> Date: Fri, 4 Aug 2023 09:24:52 +0900 Subject: [PATCH 03/11] allows convert to widget for boolean type (#1063) --- web/extensions/core/widgetInputs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js index 7600ce87b..d9eaf8a0c 100644 --- a/web/extensions/core/widgetInputs.js +++ b/web/extensions/core/widgetInputs.js @@ -2,7 +2,7 @@ import { ComfyWidgets, addValueControlWidget } from "../../scripts/widgets.js"; import { app } from "../../scripts/app.js"; const CONVERTED_TYPE = "converted-widget"; -const VALID_TYPES = ["STRING", "combo", "number"]; +const VALID_TYPES = ["STRING", "combo", "number", "BOOLEAN"]; function isConvertableWidget(widget, config) { return VALID_TYPES.includes(widget.type) || VALID_TYPES.includes(config[0]); From c99d8002f8a479e9505830347cee37f7d603394c Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 3 Aug 2023 20:27:50 -0400 Subject: [PATCH 04/11] Make sure the pooled output stays at the EOS token with added embeddings. --- comfy/sd1_clip.py | 18 +++++++++++++----- web/scripts/widgets.js | 1 - 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index d504bf77d..feca41880 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -91,13 +91,15 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder): def set_up_textual_embeddings(self, tokens, current_embeds): out_tokens = [] - next_new_token = token_dict_size = current_embeds.weight.shape[0] + next_new_token = token_dict_size = current_embeds.weight.shape[0] - 1 embedding_weights = [] for x in tokens: tokens_temp = [] for y in x: if isinstance(y, int): + if y == token_dict_size: #EOS token + y = -1 tokens_temp += [y] else: if y.shape[0] == current_embeds.weight.shape[1]: @@ -110,15 +112,21 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder): tokens_temp += [self.empty_tokens[0][-1]] out_tokens += [tokens_temp] + n = token_dict_size if len(embedding_weights) > 0: - new_embedding = torch.nn.Embedding(next_new_token, current_embeds.weight.shape[1], device=current_embeds.weight.device, dtype=current_embeds.weight.dtype) - new_embedding.weight[:token_dict_size] = current_embeds.weight[:] - n = token_dict_size + new_embedding = torch.nn.Embedding(next_new_token + 1, current_embeds.weight.shape[1], device=current_embeds.weight.device, dtype=current_embeds.weight.dtype) + new_embedding.weight[:token_dict_size] = current_embeds.weight[:-1] for x in embedding_weights: new_embedding.weight[n] = x n += 1 + new_embedding.weight[n] = current_embeds.weight[-1] #EOS embedding self.transformer.set_input_embeddings(new_embedding) - return out_tokens + + processed_tokens = [] + for x in out_tokens: + processed_tokens += [list(map(lambda a: n if a == -1 else a, x))] #The EOS token should always be the largest one + + return processed_tokens def forward(self, tokens): backup_embeds = self.transformer.get_input_embeddings() diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index d5a28badf..d4a15ba84 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -267,7 +267,6 @@ export const ComfyWidgets = { return { widget: node.addWidget(widgetType, inputName, val, () => {}, config) }; }, INT(node, inputName, inputData, app) { - console.log(app); let widgetType = isSlider(inputData[1]["display"], app); const { val, config } = getNumberDefaults(inputData, 1); Object.assign(config, { precision: 0 }); From fa962e86c1cdc3bb9dd57ac028fba0e577346983 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 4 Aug 2023 02:51:28 -0400 Subject: [PATCH 05/11] Make LatentBlend more consistent with other nodes. --- nodes.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/nodes.py b/nodes.py index 86aed032e..92baffe30 100644 --- a/nodes.py +++ b/nodes.py @@ -1059,15 +1059,14 @@ class LatentBlend: @classmethod def INPUT_TYPES(s): return {"required": { - "samples_a": ("LATENT",), - "samples_b": ("LATENT",), + "samples1": ("LATENT",), + "samples2": ("LATENT",), "blend_factor": ("FLOAT", { "default": 0.5, "min": 0, "max": 1, "step": 0.01 }), - "blend_mode": (["normal"],), }} RETURN_TYPES = ("LATENT",) @@ -1075,19 +1074,19 @@ class LatentBlend: CATEGORY = "_for_testing" - def blend(self, samples_a, samples_b, blend_factor:float, blend_mode: str): + def blend(self, samples1, samples2, blend_factor:float, blend_mode: str="normal"): - samples_out = samples_a.copy() - samples_a = samples_a["samples"] - samples_b = samples_b["samples"] + samples_out = samples1.copy() + samples1 = samples1["samples"] + samples2 = samples2["samples"] - if samples_a.shape != samples_b.shape: - samples_b.permute(0, 3, 1, 2) - samples_b = comfy.utils.common_upscale(samples_b, samples_a.shape[3], samples_a.shape[2], 'bicubic', crop='center') - samples_b.permute(0, 2, 3, 1) + if samples1.shape != samples2.shape: + samples2.permute(0, 3, 1, 2) + samples2 = comfy.utils.common_upscale(samples2, samples1.shape[3], samples1.shape[2], 'bicubic', crop='center') + samples2.permute(0, 2, 3, 1) - samples_blended = self.blend_mode(samples_a, samples_b, blend_mode) - samples_blended = samples_a * (1 - blend_factor) + samples_blended * blend_factor + samples_blended = self.blend_mode(samples1, samples2, blend_mode) + samples_blended = samples1 * blend_factor + samples_blended * (1 - blend_factor) samples_out["samples"] = samples_blended return (samples_out,) From d7638c47fc36ea7366d970f56c005216e4793e82 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 4 Aug 2023 03:22:47 -0400 Subject: [PATCH 06/11] Fix ui inconsistency. --- web/scripts/ui.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/scripts/ui.js b/web/scripts/ui.js index 5d4e92542..03a4035b1 100644 --- a/web/scripts/ui.js +++ b/web/scripts/ui.js @@ -480,7 +480,7 @@ class ComfyList { hide() { this.element.style.display = "none"; - this.button.textContent = "See " + this.#text; + this.button.textContent = "View " + this.#text; } toggle() { From 0bbd9dd4d9cc3749f4ec0903f33c5ffb9d7aecfd Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 4 Aug 2023 08:29:25 +0100 Subject: [PATCH 07/11] add system info to stats endpoint --- server.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server.py b/server.py index f61b11a97..fab33be3e 100644 --- a/server.py +++ b/server.py @@ -345,6 +345,11 @@ class PromptServer(): vram_total, torch_vram_total = comfy.model_management.get_total_memory(device, torch_total_too=True) vram_free, torch_vram_free = comfy.model_management.get_free_memory(device, torch_free_too=True) system_stats = { + "system": { + "os": os.name, + "python_version": sys.version, + "embedded_python": os.path.split(os.path.split(sys.executable)[0])[1] == "python_embeded" + }, "devices": [ { "name": device_name, From 43ae9fe7216198ff044ea33ac6f6d21a9ca9c2af Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 4 Aug 2023 08:29:51 +0100 Subject: [PATCH 08/11] add system stats function --- web/scripts/api.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/scripts/api.js b/web/scripts/api.js index d3d15e47e..b1d245d73 100644 --- a/web/scripts/api.js +++ b/web/scripts/api.js @@ -264,6 +264,15 @@ class ComfyApi extends EventTarget { } } + /** + * Gets system & device stats + * @returns System stats such as python version, OS, per device info + */ + async getSystemStats() { + const res = await this.fetchApi("/system_stats"); + return await res.json(); + } + /** * Sends a POST request to the API * @param {*} type The endpoint to post to From b2ea0cbd5c5dfbb734f375acd042bd49cabe84ec Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 4 Aug 2023 08:30:01 +0100 Subject: [PATCH 09/11] add logging --- web/scripts/app.js | 6 + web/scripts/logging.js | 367 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+) create mode 100644 web/scripts/logging.js diff --git a/web/scripts/app.js b/web/scripts/app.js index 8c9e7a27f..11903a2d4 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -1,3 +1,4 @@ +import { ComfyLogging } from "./logging.js"; import { ComfyWidgets } from "./widgets.js"; import { ComfyUI, $el } from "./ui.js"; import { api } from "./api.js"; @@ -31,6 +32,7 @@ export class ComfyApp { constructor() { this.ui = new ComfyUI(this); + this.logging = new ComfyLogging(this); /** * List of extensions that are registered with the app @@ -1023,6 +1025,7 @@ export class ComfyApp { */ async #loadExtensions() { const extensions = await api.getExtensions(); + this.logging.addEntry("Comfy.App", "debug", { Extensions: extensions }); for (const ext of extensions) { try { await import(api.apiURL(ext)); @@ -1306,6 +1309,9 @@ export class ComfyApp { (t) => `
  • ${t}
    • ` ).join("")}Nodes that have failed to load will show as red on the graph.` ); + this.logging.addEntry("Comfy.App", "warn", { + MissingNodes: nodes, + }); } } diff --git a/web/scripts/logging.js b/web/scripts/logging.js new file mode 100644 index 000000000..c73462e1e --- /dev/null +++ b/web/scripts/logging.js @@ -0,0 +1,367 @@ +import { $el, ComfyDialog } from "./ui.js"; +import { api } from "./api.js"; + +$el("style", { + textContent: ` + .comfy-logging-logs { + display: grid; + color: var(--fg-color); + white-space: pre-wrap; + } + .comfy-logging-log { + display: contents; + } + .comfy-logging-title { + background: var(--tr-even-bg-color); + font-weight: bold; + margin-bottom: 5px; + text-align: center; + } + .comfy-logging-log div { + background: var(--row-bg); + padding: 5px; + } + `, + parent: document.body, +}); + +// Stringify function supporting max depth and removal of circular references +// https://stackoverflow.com/a/57193345 +function stringify(val, depth, replacer, space, onGetObjID) { + depth = isNaN(+depth) ? 1 : depth; + var recursMap = new WeakMap(); + function _build(val, depth, o, a, r) { + // (JSON.stringify() has it's own rules, which we respect here by using it for property iteration) + return !val || typeof val != "object" + ? val + : ((r = recursMap.has(val)), + recursMap.set(val, true), + (a = Array.isArray(val)), + r + ? (o = (onGetObjID && onGetObjID(val)) || null) + : JSON.stringify(val, function (k, v) { + if (a || depth > 0) { + if (replacer) v = replacer(k, v); + if (!k) return (a = Array.isArray(v)), (val = v); + !o && (o = a ? [] : {}); + o[k] = _build(v, a ? depth : depth - 1); + } + }), + o === void 0 ? (a ? [] : {}) : o); + } + return JSON.stringify(_build(val, depth), null, space); +} + +const jsonReplacer = (k, v, ui) => { + if (v instanceof Array && v.length === 1) { + v = v[0]; + } + if (v instanceof Date) { + v = v.toISOString(); + if (ui) { + v = v.split("T")[1]; + } + } + if (v instanceof Error) { + let err = ""; + if (v.name) err += v.name + "\n"; + if (v.message) err += v.message + "\n"; + if (v.stack) err += v.stack + "\n"; + if (!err) { + err = v.toString(); + } + v = err; + } + return v; +}; + +const fileInput = $el("input", { + type: "file", + accept: ".json", + style: { display: "none" }, + parent: document.body, +}); + +class ComfyLoggingDialog extends ComfyDialog { + constructor(logging) { + super(); + this.logging = logging; + } + + clear() { + this.logging.clear(); + this.show(); + } + + export() { + const blob = new Blob([stringify([...this.logging.entries], 20, jsonReplacer, "\t")], { + type: "application/json", + }); + const url = URL.createObjectURL(blob); + const a = $el("a", { + href: url, + download: `comfyui-logs-${Date.now()}.json`, + style: { display: "none" }, + parent: document.body, + }); + a.click(); + setTimeout(function () { + a.remove(); + window.URL.revokeObjectURL(url); + }, 0); + } + + import() { + fileInput.onchange = () => { + const reader = new FileReader(); + reader.onload = () => { + fileInput.remove(); + try { + const obj = JSON.parse(reader.result); + if (obj instanceof Array) { + this.show(obj); + } else { + throw new Error("Invalid file selected."); + } + } catch (error) { + alert("Unable to load logs: " + error.message); + } + }; + reader.readAsText(fileInput.files[0]); + }; + fileInput.click(); + } + + createButtons() { + return [ + $el("button", { + type: "button", + textContent: "Clear", + onclick: () => this.clear(), + }), + $el("button", { + type: "button", + textContent: "Export logs...", + onclick: () => this.export(), + }), + $el("button", { + type: "button", + textContent: "View exported logs...", + onclick: () => this.import(), + }), + ...super.createButtons(), + ]; + } + + getTypeColor(type) { + switch (type) { + case "error": + return "red"; + case "warn": + return "orange"; + case "debug": + return "dodgerblue"; + } + } + + show(entries) { + if (!entries) entries = this.logging.entries; + this.element.style.width = "100%"; + const cols = { + source: "Source", + type: "Type", + timestamp: "Timestamp", + message: "Message", + }; + const keys = Object.keys(cols); + const headers = Object.values(cols).map((title) => + $el("div.comfy-logging-title", { + textContent: title, + }) + ); + const rows = entries.map((entry, i) => { + return $el( + "div.comfy-logging-log", + { + $: (el) => el.style.setProperty("--row-bg", `var(--tr-${i % 2 ? "even" : "odd"}-bg-color)`), + }, + keys.map((key) => { + let v = entry[key]; + let color; + if (key === "type") { + color = this.getTypeColor(v); + } else { + v = jsonReplacer(key, v, true); + + if (typeof v === "object") { + v = stringify(v, 5, jsonReplacer, " "); + } + } + + return $el("div", { + style: { + color, + }, + textContent: v, + }); + }) + ); + }); + + const grid = $el( + "div.comfy-logging-logs", + { + style: { + gridTemplateColumns: `repeat(${headers.length}, 1fr)`, + }, + }, + [...headers, ...rows] + ); + const els = [grid]; + if (!this.logging.enabled) { + els.unshift( + $el("h3", { + style: { textAlign: "center" }, + textContent: "Logging is disabled", + }) + ); + } + super.show($el("div", els)); + } +} + +export class ComfyLogging { + /** + * @type Array<{ source: string, type: string, timestamp: Date, message: any }> + */ + entries = []; + + #enabled; + #console = {}; + + get enabled() { + return this.#enabled; + } + + set enabled(value) { + if (value === this.#enabled) return; + if (value) { + this.patchConsole(); + } else { + this.unpatchConsole(); + } + this.#enabled = value; + } + + constructor(app) { + this.app = app; + + this.dialog = new ComfyLoggingDialog(this); + this.addSetting(); + this.catchUnhandled(); + this.addInitData(); + } + + addSetting() { + const settingId = "Comfy.Logging.Enabled"; + const htmlSettingId = settingId.replaceAll(".", "-"); + const setting = this.app.ui.settings.addSetting({ + id: settingId, + name: settingId, + defaultValue: true, + type: (name, setter, value) => { + return $el("tr", [ + $el("td", [ + $el("label", { + textContent: "Logging", + for: htmlSettingId, + }), + ]), + $el("td", [ + $el("input", { + id: htmlSettingId, + type: "checkbox", + checked: value, + onchange: (event) => { + setter((this.enabled = event.target.checked)); + }, + }), + $el("button", { + textContent: "View Logs", + onclick: () => { + this.app.ui.settings.element.close(); + this.dialog.show(); + }, + style: { + fontSize: "14px", + display: "block", + marginTop: "5px", + }, + }), + ]), + ]); + }, + }); + this.enabled = setting.value; + } + + patchConsole() { + // Capture common console outputs + const self = this; + for (const type of ["log", "warn", "error", "debug"]) { + const orig = console[type]; + this.#console[type] = orig; + console[type] = function () { + orig.apply(console, arguments); + self.addEntry("console", type, ...arguments); + }; + } + } + + unpatchConsole() { + // Restore original console functions + for (const type of Object.keys(this.#console)) { + console[type] = this.#console[type]; + } + this.#console = {}; + } + + catchUnhandled() { + // Capture uncaught errors + window.addEventListener("error", (e) => { + this.addEntry("window", "error", e.error ?? "Unknown error"); + return false; + }); + + window.addEventListener("unhandledrejection", (e) => { + this.addEntry("unhandledrejection", "error", e.reason ?? "Unknown error"); + }); + } + + clear() { + this.entries = []; + } + + addEntry(source, type, ...args) { + if (this.enabled) { + this.entries.push({ + source, + type, + timestamp: new Date(), + message: args, + }); + } + } + + log(source, ...args) { + this.addEntry(source, "log", ...args); + } + + async addInitData() { + if (!this.enabled) return; + const source = "ComfyUI.Logging"; + this.addEntry(source, "debug", { UserAgent: navigator.userAgent }); + const systemStats = await api.getSystemStats(); + this.addEntry(source, "debug", systemStats); + } +} From 3d614dde499d7c7fcb29696ce4999967b51757c2 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 4 Aug 2023 03:47:45 -0400 Subject: [PATCH 10/11] Fix bug with reroutes and bypass. --- web/scripts/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index 8c9e7a27f..c4d593f2c 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -1356,7 +1356,7 @@ export class ComfyApp { if (parent.isVirtualNode) { link = parent.getInputLink(link.origin_slot); if (link) { - parent = parent.getInputNode(link.origin_slot); + parent = parent.getInputNode(link.target_slot); if (parent) { found = true; } From 1ce0d8ad68e15c58a0e9793eb873f0238f741f4c Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 4 Aug 2023 12:08:45 -0400 Subject: [PATCH 11/11] Add CMP 30HX card to the nvidia_16_series list. --- comfy/model_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 0ffca06da..4dd15b41c 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -535,7 +535,7 @@ def should_use_fp16(device=None, model_params=0): return False #FP16 is just broken on these cards - nvidia_16_series = ["1660", "1650", "1630", "T500", "T550", "T600", "MX550", "MX450"] + nvidia_16_series = ["1660", "1650", "1630", "T500", "T550", "T600", "MX550", "MX450", "CMP 30HX"] for x in nvidia_16_series: if x in props.name: return False