From bfebe2d6c36e2ab7cf9ce9892abe80fc3057c46f Mon Sep 17 00:00:00 2001 From: reaper47 Date: Fri, 9 Jun 2023 13:29:15 +0200 Subject: [PATCH 01/45] Improve ContextMenuFilter extension --- web/extensions/core/contextMenuFilter.js | 226 ++++++++++++----------- web/style.css | 15 +- 2 files changed, 128 insertions(+), 113 deletions(-) diff --git a/web/extensions/core/contextMenuFilter.js b/web/extensions/core/contextMenuFilter.js index 51e66f924..662d87e74 100644 --- a/web/extensions/core/contextMenuFilter.js +++ b/web/extensions/core/contextMenuFilter.js @@ -1,132 +1,138 @@ -import { app } from "/scripts/app.js"; +import {app} from "/scripts/app.js"; // Adds filtering to combo context menus -const id = "Comfy.ContextMenuFilter"; -app.registerExtension({ - name: id, +const ext = { + name: "Comfy.ContextMenuFilter", init() { const ctxMenu = LiteGraph.ContextMenu; + LiteGraph.ContextMenu = function (values, options) { const ctx = ctxMenu.call(this, values, options); // If we are a dark menu (only used for combo boxes) then add a filter input if (options?.className === "dark" && values?.length > 10) { const filter = document.createElement("input"); - Object.assign(filter.style, { - width: "calc(100% - 10px)", - border: "0", - boxSizing: "border-box", - background: "#333", - border: "1px solid #999", - margin: "0 0 5px 5px", - color: "#fff", - }); + filter.classList.add("comfy-context-menu-filter"); filter.placeholder = "Filter list"; this.root.prepend(filter); - let selectedIndex = 0; - let items = this.root.querySelectorAll(".litemenu-entry"); - let itemCount = items.length; - let selectedItem; + const items = Array.from(this.root.querySelectorAll(".litemenu-entry")); + let displayedItems = [...items]; + let itemCount = displayedItems.length; - // Apply highlighting to the selected item - function updateSelected() { - if (selectedItem) { - selectedItem.style.setProperty("background-color", ""); - selectedItem.style.setProperty("color", ""); - } - selectedItem = items[selectedIndex]; - if (selectedItem) { - selectedItem.style.setProperty("background-color", "#ccc", "important"); - selectedItem.style.setProperty("color", "#000", "important"); - } - } + // We must request an animation frame for the current node of the active canvas to update. + requestAnimationFrame(() => { + const currentNode = LGraphCanvas.active_canvas.current_node; + const clickedComboValue = currentNode.widgets + .filter(w => w.type === "combo" && w.options.values.length === values.length) + .find(w => w.options.values.every((v, i) => v === values[i])) + .value; - const positionList = () => { - const rect = this.root.getBoundingClientRect(); - - // If the top is off screen then shift the element with scaling applied - if (rect.top < 0) { - const scale = 1 - this.root.getBoundingClientRect().height / this.root.clientHeight; - const shift = (this.root.clientHeight * scale) / 2; - this.root.style.top = -shift + "px"; - } - } - - updateSelected(); - - // Arrow up/down to select items - filter.addEventListener("keydown", (e) => { - if (e.key === "ArrowUp") { - if (selectedIndex === 0) { - selectedIndex = itemCount - 1; - } else { - selectedIndex--; - } - updateSelected(); - e.preventDefault(); - } else if (e.key === "ArrowDown") { - if (selectedIndex === itemCount - 1) { - selectedIndex = 0; - } else { - selectedIndex++; - } - updateSelected(); - e.preventDefault(); - } else if ((selectedItem && e.key === "Enter") || e.keyCode === 13 || e.keyCode === 10) { - selectedItem.click(); - } else if(e.key === "Escape") { - this.close(); - } - }); - - filter.addEventListener("input", () => { - // Hide all items that dont match our filter - const term = filter.value.toLocaleLowerCase(); - items = this.root.querySelectorAll(".litemenu-entry"); - // When filtering recompute which items are visible for arrow up/down - // Try and maintain selection - let visibleItems = []; - for (const item of items) { - const visible = !term || item.textContent.toLocaleLowerCase().includes(term); - if (visible) { - item.style.display = "block"; - if (item === selectedItem) { - selectedIndex = visibleItems.length; - } - visibleItems.push(item); - } else { - item.style.display = "none"; - if (item === selectedItem) { - selectedIndex = 0; - } - } - } - items = visibleItems; + let selectedIndex = values.findIndex(v => v === clickedComboValue); + let selectedItem = displayedItems?.[selectedIndex]; updateSelected(); - // If we have an event then we can try and position the list under the source - if (options.event) { - let top = options.event.clientY - 10; - - const bodyRect = document.body.getBoundingClientRect(); - const rootRect = this.root.getBoundingClientRect(); - if (bodyRect.height && top > bodyRect.height - rootRect.height - 10) { - top = Math.max(0, bodyRect.height - rootRect.height - 10); - } - - this.root.style.top = top + "px"; - positionList(); + // Apply highlighting to the selected item + function updateSelected() { + selectedItem?.style.setProperty("background-color", ""); + selectedItem?.style.setProperty("color", ""); + selectedItem = displayedItems[selectedIndex]; + selectedItem?.style.setProperty("background-color", "#ccc", "important"); + selectedItem?.style.setProperty("color", "#000", "important"); } - }); - requestAnimationFrame(() => { - // Focus the filter box when opening - filter.focus(); + const positionList = () => { + const rect = this.root.getBoundingClientRect(); - positionList(); - }); + // If the top is off-screen then shift the element with scaling applied + if (rect.top < 0) { + const scale = 1 - this.root.getBoundingClientRect().height / this.root.clientHeight; + const shift = (this.root.clientHeight * scale) / 2; + this.root.style.top = -shift + "px"; + } + } + + // Arrow up/down to select items + filter.addEventListener("keydown", (event) => { + switch (event.key) { + case "ArrowUp": + event.preventDefault(); + if (selectedIndex === 0) { + selectedIndex = itemCount - 1; + } else { + selectedIndex--; + } + updateSelected(); + break; + case "ArrowRight": + event.preventDefault(); + selectedIndex = itemCount - 1; + updateSelected(); + break; + case "ArrowDown": + event.preventDefault(); + if (selectedIndex === itemCount - 1) { + selectedIndex = 0; + } else { + selectedIndex++; + } + updateSelected(); + break; + case "ArrowLeft": + event.preventDefault(); + selectedIndex = 0; + updateSelected(); + break; + case "Enter": + selectedItem?.click(); + break; + case "Escape": + this.close(); + break; + } + }); + + filter.addEventListener("input", () => { + // Hide all items that don't match our filter + const term = filter.value.toLocaleLowerCase(); + // When filtering, recompute which items are visible for arrow up/down and maintain selection. + displayedItems = items.filter(item => { + const isVisible = !term || item.textContent.toLocaleLowerCase().includes(term); + item.style.display = isVisible ? "block" : "none"; + return isVisible; + }); + + selectedIndex = 0; + if (displayedItems.includes(selectedItem)) { + selectedIndex = displayedItems.findIndex(d => d === selectedItem); + } + itemCount = displayedItems.length; + + updateSelected(); + + // If we have an event then we can try and position the list under the source + if (options.event) { + let top = options.event.clientY - 10; + + const bodyRect = document.body.getBoundingClientRect(); + const rootRect = this.root.getBoundingClientRect(); + if (bodyRect.height && top > bodyRect.height - rootRect.height - 10) { + top = Math.max(0, bodyRect.height - rootRect.height - 10); + } + + this.root.style.top = top + "px"; + positionList(); + } + }); + + requestAnimationFrame(() => { + // Focus the filter box when opening + filter.focus(); + + positionList(); + }); + }) } return ctx; @@ -134,4 +140,6 @@ app.registerExtension({ LiteGraph.ContextMenu.prototype = ctxMenu.prototype; }, -}); +} + +app.registerExtension(ext); diff --git a/web/style.css b/web/style.css index 47571a16e..5fea5bba8 100644 --- a/web/style.css +++ b/web/style.css @@ -50,7 +50,7 @@ body { padding: 30px 30px 10px 30px; background-color: var(--comfy-menu-bg); /* Modal background */ color: var(--error-text); - box-shadow: 0px 0px 20px #888888; + box-shadow: 0 0 20px #888888; border-radius: 10px; top: 50%; left: 50%; @@ -84,7 +84,7 @@ body { font-size: 15px; position: absolute; top: 50%; - right: 0%; + right: 0; text-align: center; z-index: 100; width: 170px; @@ -252,7 +252,7 @@ button.comfy-queue-btn { bottom: 0 !important; left: auto !important; right: 0 !important; - border-radius: 0px; + border-radius: 0; } .comfy-menu span.drag-handle { visibility:hidden @@ -291,7 +291,7 @@ button.comfy-queue-btn { .litegraph .dialog { z-index: 1; - font-family: Arial; + font-family: Arial, sans-serif; } .litegraph .litemenu-entry.has_submenu { @@ -330,6 +330,13 @@ button.comfy-queue-btn { color: var(--input-text) !important; } +.comfy-context-menu-filter { + box-sizing: border-box; + border: 1px solid #999; + margin: 0 0 5px 5px; + width: calc(100% - 10px); +} + /* Search box */ .litegraph.litesearchbox { From 2bcdd6c7d401d2e036325c6f3d16aa7eff7ccf14 Mon Sep 17 00:00:00 2001 From: Jorge Campo <62282406+jorge-campo@users.noreply.github.com> Date: Fri, 9 Jun 2023 22:25:33 +0200 Subject: [PATCH 02/45] Add install instructions for Apple silicon --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9083b7e1..001641496 100644 --- a/README.md +++ b/README.md @@ -121,9 +121,16 @@ After this you should have everything installed and can proceed to running Comfy [Intel Arc](https://github.com/comfyanonymous/ComfyUI/discussions/476) -Mac/MPS: There is basic support in the code but until someone makes some install instruction you are on your own. +You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version. -Directml: ```pip install torch-directml``` Then you can launch ComfyUI with: ```python main.py --directml``` +According to the [DirectML page](https://github.com/microsoft/DirectML#hardware-requirements), `pytorch-directml` package is not avilable for Apple silicon computers. However, you can still run ComfyUI without `pytorch-directml`. + +1. Install pytorch. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide. +1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux. +1. Install the ComfyUI [dependencies](#dependencies). If you have another UI to work with Stable Diffusion (such as Automatic1111), you can use the the packages for this installation. See [the instruction below](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies). +1. Launch ComfyUI by running `python main.py`. + +> **Note**: Remember to add your models, VAE, LoRAs etc. to the corresponding Comfy folders, as discussed in [ComfyUI manual installation](#manual-install-windows-linux). ### I already have another UI for Stable Diffusion installed do I really have to install all of these dependencies? From ba23753670913a6f38a22f84cfb631e44f549a78 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 10 Jun 2023 03:23:01 -0400 Subject: [PATCH 03/45] DirectML is for Windows. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 001641496..e33060307 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,6 @@ After this you should have everything installed and can proceed to running Comfy You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version. -According to the [DirectML page](https://github.com/microsoft/DirectML#hardware-requirements), `pytorch-directml` package is not avilable for Apple silicon computers. However, you can still run ComfyUI without `pytorch-directml`. - 1. Install pytorch. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide. 1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux. 1. Install the ComfyUI [dependencies](#dependencies). If you have another UI to work with Stable Diffusion (such as Automatic1111), you can use the the packages for this installation. See [the instruction below](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies). @@ -132,6 +130,7 @@ According to the [DirectML page](https://github.com/microsoft/DirectML#hardware- > **Note**: Remember to add your models, VAE, LoRAs etc. to the corresponding Comfy folders, as discussed in [ComfyUI manual installation](#manual-install-windows-linux). +DirectML (AMD Cards on Windows): ```pip install torch-directml``` Then you can launch ComfyUI with: ```python main.py --directml``` ### I already have another UI for Stable Diffusion installed do I really have to install all of these dependencies? From 656f62569d4e3858d7f99ec7a44f23e885353285 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 10 Jun 2023 04:19:33 -0400 Subject: [PATCH 04/45] Make the sections in the others install section more clearly separate. --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e33060307..1de9d4c3b 100644 --- a/README.md +++ b/README.md @@ -119,18 +119,22 @@ After this you should have everything installed and can proceed to running Comfy ### Others: -[Intel Arc](https://github.com/comfyanonymous/ComfyUI/discussions/476) +#### [Intel Arc](https://github.com/comfyanonymous/ComfyUI/discussions/476) + +#### Apple Mac silicon You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version. 1. Install pytorch. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide. 1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux. -1. Install the ComfyUI [dependencies](#dependencies). If you have another UI to work with Stable Diffusion (such as Automatic1111), you can use the the packages for this installation. See [the instruction below](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies). +1. Install the ComfyUI [dependencies](#dependencies). If you have another Stable Diffusion UI [you might be able to reuse the dependencies](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies). 1. Launch ComfyUI by running `python main.py`. > **Note**: Remember to add your models, VAE, LoRAs etc. to the corresponding Comfy folders, as discussed in [ComfyUI manual installation](#manual-install-windows-linux). -DirectML (AMD Cards on Windows): ```pip install torch-directml``` Then you can launch ComfyUI with: ```python main.py --directml``` +#### DirectML (AMD Cards on Windows) + +```pip install torch-directml``` Then you can launch ComfyUI with: ```python main.py --directml``` ### I already have another UI for Stable Diffusion installed do I really have to install all of these dependencies? From 7b2f09b5fa7db652a798a4d865b685ea1e6e152c Mon Sep 17 00:00:00 2001 From: reaper47 Date: Sat, 10 Jun 2023 21:53:49 +0200 Subject: [PATCH 05/45] Issue 742: Extension folder should be ignored --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8380a2f7c..38d2ba11b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,8 @@ custom_nodes/ !custom_nodes/example_node.py.example extra_model_paths.yaml /.vs -.idea/ \ No newline at end of file +.idea/ +venv/ +web/extensions/* +!web/extensions/logging.js.example +!web/extensions/core/ \ No newline at end of file From c64ca8c0b22d2f5ac5dff59075b2a5ea4edd9e23 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 11 Jun 2023 04:01:18 -0400 Subject: [PATCH 06/45] Refactor unCLIP noise augment out of samplers.py --- comfy/model_base.py | 31 +++++++++++++++++++++++++++++++ comfy/samplers.py | 43 ++++++++----------------------------------- nodes.py | 8 ++++---- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/comfy/model_base.py b/comfy/model_base.py index 7370c19fd..9adea9a5d 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -60,6 +60,37 @@ class SD21UNCLIP(BaseModel): super().__init__(unet_config, v_prediction) self.noise_augmentor = CLIPEmbeddingNoiseAugmentation(**noise_aug_config) + def encode_adm(self, **kwargs): + unclip_conditioning = kwargs.get("unclip_conditioning", None) + device = kwargs["device"] + + if unclip_conditioning is not None: + adm_inputs = [] + weights = [] + noise_aug = [] + for unclip_cond in unclip_conditioning: + adm_cond = unclip_cond["clip_vision_output"].image_embeds + weight = unclip_cond["strength"] + noise_augment = unclip_cond["noise_augmentation"] + noise_level = round((self.noise_augmentor.max_noise_level - 1) * noise_augment) + c_adm, noise_level_emb = self.noise_augmentor(adm_cond.to(device), noise_level=torch.tensor([noise_level], device=device)) + adm_out = torch.cat((c_adm, noise_level_emb), 1) * weight + weights.append(weight) + noise_aug.append(noise_augment) + adm_inputs.append(adm_out) + + if len(noise_aug) > 1: + adm_out = torch.stack(adm_inputs).sum(0) + #TODO: add a way to control this + noise_augment = 0.05 + noise_level = round((self.noise_augmentor.max_noise_level - 1) * noise_augment) + c_adm, noise_level_emb = self.noise_augmentor(adm_out[:, :self.noise_augmentor.time_embed.dim], noise_level=torch.tensor([noise_level], device=device)) + adm_out = torch.cat((c_adm, noise_level_emb), 1) + else: + adm_out = torch.zeros((1, self.adm_channels)) + + return adm_out + class SDInpaint(BaseModel): def __init__(self, unet_config, v_prediction=False): super().__init__(unet_config, v_prediction) diff --git a/comfy/samplers.py b/comfy/samplers.py index a33d150d0..d3cd901e7 100644 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -460,42 +460,18 @@ def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func): uncond[temp[1]] = [o[0], n] -def encode_adm(conds, batch_size, device, noise_augmentor=None): +def encode_adm(model, conds, batch_size, device): for t in range(len(conds)): x = conds[t] adm_out = None - if noise_augmentor is not None: - if 'adm' in x[1]: - adm_inputs = [] - weights = [] - noise_aug = [] - adm_in = x[1]["adm"] - for adm_c in adm_in: - adm_cond = adm_c[0].image_embeds - weight = adm_c[1] - noise_augment = adm_c[2] - noise_level = round((noise_augmentor.max_noise_level - 1) * noise_augment) - c_adm, noise_level_emb = noise_augmentor(adm_cond.to(device), noise_level=torch.tensor([noise_level], device=device)) - adm_out = torch.cat((c_adm, noise_level_emb), 1) * weight - weights.append(weight) - noise_aug.append(noise_augment) - adm_inputs.append(adm_out) - - if len(noise_aug) > 1: - adm_out = torch.stack(adm_inputs).sum(0) - #TODO: add a way to control this - noise_augment = 0.05 - noise_level = round((noise_augmentor.max_noise_level - 1) * noise_augment) - c_adm, noise_level_emb = noise_augmentor(adm_out[:, :noise_augmentor.time_embed.dim], noise_level=torch.tensor([noise_level], device=device)) - adm_out = torch.cat((c_adm, noise_level_emb), 1) - else: - adm_out = torch.zeros((1, noise_augmentor.time_embed.dim * 2), device=device) + if 'adm' in x[1]: + adm_out = x[1]["adm"] else: - if 'adm' in x[1]: - adm_out = x[1]["adm"].to(device) + params = x[1].copy() + adm_out = model.encode_adm(device=device, **params) if adm_out is not None: x[1] = x[1].copy() - x[1]["adm_encoded"] = torch.cat([adm_out] * batch_size) + x[1]["adm_encoded"] = torch.cat([adm_out] * batch_size).to(device) return conds @@ -603,11 +579,8 @@ class KSampler: precision_scope = contextlib.nullcontext if self.model.is_adm(): - noise_augmentor = None - if hasattr(self.model, 'noise_augmentor'): #unclip - noise_augmentor = self.model.noise_augmentor - positive = encode_adm(positive, noise.shape[0], self.device, noise_augmentor) - negative = encode_adm(negative, noise.shape[0], self.device, noise_augmentor) + positive = encode_adm(self.model, positive, noise.shape[0], self.device) + negative = encode_adm(self.model, negative, noise.shape[0], self.device) extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": self.model_options} diff --git a/nodes.py b/nodes.py index b057504ed..12243fab4 100644 --- a/nodes.py +++ b/nodes.py @@ -623,11 +623,11 @@ class unCLIPConditioning: c = [] for t in conditioning: o = t[1].copy() - x = (clip_vision_output, strength, noise_augmentation) - if "adm" in o: - o["adm"] = o["adm"][:] + [x] + x = {"clip_vision_output": clip_vision_output, "strength": strength, "noise_augmentation": noise_augmentation} + if "unclip_conditioning" in o: + o["unclip_conditioning"] = o["unclip_conditioning"][:] + [x] else: - o["adm"] = [x] + o["unclip_conditioning"] = [x] n = [t[0], o] c.append(n) return (c, ) From c069fc0730f27d42b1e0dc82c698f448cd3d8087 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 11 Jun 2023 23:25:39 -0400 Subject: [PATCH 07/45] Auto switch to tiled VAE encode if regular one runs out of memory. --- comfy/sd.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 3747f53b8..718dccd09 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -544,6 +544,19 @@ class VAE: / 3.0) / 2.0, min=0.0, max=1.0) return output + def encode_tiled_(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64): + steps = pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap) + steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap) + steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x * 2, tile_y // 2, overlap) + pbar = utils.ProgressBar(steps) + + encode_fn = lambda a: self.first_stage_model.encode(2. * a.to(self.device) - 1.).sample() * self.scale_factor + samples = utils.tiled_scale(pixel_samples, encode_fn, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar) + samples += utils.tiled_scale(pixel_samples, encode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar) + samples += utils.tiled_scale(pixel_samples, encode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar) + samples /= 3.0 + return samples + def decode(self, samples_in): model_management.unload_model() self.first_stage_model = self.first_stage_model.to(self.device) @@ -574,28 +587,26 @@ class VAE: def encode(self, pixel_samples): model_management.unload_model() self.first_stage_model = self.first_stage_model.to(self.device) - pixel_samples = pixel_samples.movedim(-1,1).to(self.device) - samples = self.first_stage_model.encode(2. * pixel_samples - 1.).sample() * self.scale_factor + pixel_samples = pixel_samples.movedim(-1,1) + try: + batch_number = 1 + samples = torch.empty((pixel_samples.shape[0], 4, round(pixel_samples.shape[2] // 8), round(pixel_samples.shape[3] // 8)), device="cpu") + for x in range(0, pixel_samples.shape[0], batch_number): + pixels_in = (2. * pixel_samples[x:x+batch_number] - 1.).to(self.device) + samples[x:x+batch_number] = self.first_stage_model.encode(pixels_in).sample().cpu() * self.scale_factor + except model_management.OOM_EXCEPTION as e: + print("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.") + samples = self.encode_tiled_(pixel_samples) + self.first_stage_model = self.first_stage_model.cpu() - samples = samples.cpu() return samples def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64): model_management.unload_model() self.first_stage_model = self.first_stage_model.to(self.device) - pixel_samples = pixel_samples.movedim(-1,1).to(self.device) - - steps = pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap) - steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap) - steps += pixel_samples.shape[0] * utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x * 2, tile_y // 2, overlap) - pbar = utils.ProgressBar(steps) - - samples = utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar) - samples += utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar) - samples += utils.tiled_scale(pixel_samples, lambda a: self.first_stage_model.encode(2. * a - 1.).sample() * self.scale_factor, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar) - samples /= 3.0 + pixel_samples = pixel_samples.movedim(-1,1) + samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap) self.first_stage_model = self.first_stage_model.cpu() - samples = samples.cpu() return samples def broadcast_image_to(tensor, target_batch_size, batched_number): From f8c593105300008018431452567fe0b1262fdc81 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 12 Jun 2023 00:21:50 -0400 Subject: [PATCH 08/45] Split the batch in VAEEncode if there's not enough memory. --- comfy/sd.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/comfy/sd.py b/comfy/sd.py index 718dccd09..d898d0197 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -589,11 +589,14 @@ class VAE: self.first_stage_model = self.first_stage_model.to(self.device) pixel_samples = pixel_samples.movedim(-1,1) try: - batch_number = 1 + free_memory = model_management.get_free_memory(self.device) + batch_number = int((free_memory * 0.7) / (2078 * pixel_samples.shape[2] * pixel_samples.shape[3])) #NOTE: this constant along with the one in the decode above are estimated from the mem usage for the VAE and could change. + batch_number = max(1, batch_number) samples = torch.empty((pixel_samples.shape[0], 4, round(pixel_samples.shape[2] // 8), round(pixel_samples.shape[3] // 8)), device="cpu") for x in range(0, pixel_samples.shape[0], batch_number): pixels_in = (2. * pixel_samples[x:x+batch_number] - 1.).to(self.device) samples[x:x+batch_number] = self.first_stage_model.encode(pixels_in).sample().cpu() * self.scale_factor + except model_management.OOM_EXCEPTION as e: print("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.") samples = self.encode_tiled_(pixel_samples) From 67833c83d88711b185e9db32abefe8a05fe1c665 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 12 Jun 2023 01:14:04 -0400 Subject: [PATCH 09/45] Add ImageScaleBy node. --- nodes.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/nodes.py b/nodes.py index 12243fab4..658e32dad 100644 --- a/nodes.py +++ b/nodes.py @@ -1192,6 +1192,26 @@ class ImageScale: s = s.movedim(1,-1) return (s,) +class ImageScaleBy: + upscale_methods = ["nearest-exact", "bilinear", "area"] + + @classmethod + def INPUT_TYPES(s): + return {"required": { "image": ("IMAGE",), "upscale_method": (s.upscale_methods,), + "scale_by": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 8.0, "step": 0.01}),}} + RETURN_TYPES = ("IMAGE",) + FUNCTION = "upscale" + + CATEGORY = "image/upscaling" + + def upscale(self, image, upscale_method, scale_by): + samples = image.movedim(-1,1) + width = round(samples.shape[3] * scale_by) + height = round(samples.shape[2] * scale_by) + s = comfy.utils.common_upscale(samples, width, height, upscale_method, "disabled") + s = s.movedim(1,-1) + return (s,) + class ImageInvert: @classmethod @@ -1290,6 +1310,7 @@ NODE_CLASS_MAPPINGS = { "LoadImage": LoadImage, "LoadImageMask": LoadImageMask, "ImageScale": ImageScale, + "ImageScaleBy": ImageScaleBy, "ImageInvert": ImageInvert, "ImagePadForOutpaint": ImagePadForOutpaint, "ConditioningAverage ": ConditioningAverage , @@ -1371,6 +1392,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LoadImage": "Load Image", "LoadImageMask": "Load Image (as Mask)", "ImageScale": "Upscale Image", + "ImageScaleBy": "Upscale Image By", "ImageUpscaleWithModel": "Upscale Image (using Model)", "ImageInvert": "Invert Image", "ImagePadForOutpaint": "Pad Image for Outpainting", From 3402ec0c0db83615831dca946d9cc1d167d5a2f8 Mon Sep 17 00:00:00 2001 From: reaper47 Date: Mon, 12 Jun 2023 15:58:05 +0200 Subject: [PATCH 10/45] Issue 752: Fix background --- web/extensions/core/colorPalette.js | 70 +++++++++++++++++++---------- web/index.html | 1 + web/lib/litegraph.extensions.js | 21 +++++++++ 3 files changed, 68 insertions(+), 24 deletions(-) create mode 100644 web/lib/litegraph.extensions.js diff --git a/web/extensions/core/colorPalette.js b/web/extensions/core/colorPalette.js index 84c2a3d10..048161bdd 100644 --- a/web/extensions/core/colorPalette.js +++ b/web/extensions/core/colorPalette.js @@ -1,6 +1,5 @@ -import { app } from "/scripts/app.js"; -import { $el } from "/scripts/ui.js"; -import { api } from "/scripts/api.js"; +import {app} from "/scripts/app.js"; +import {$el} from "/scripts/ui.js"; // Manage color palettes @@ -24,6 +23,8 @@ const colorPalettes = { "TAESD": "#DCC274", // cheesecake }, "litegraph_base": { + "BACKGROUND_IMAGE": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAQBJREFUeNrs1rEKwjAUhlETUkj3vP9rdmr1Ysammk2w5wdxuLgcMHyptfawuZX4pJSWZTnfnu/lnIe/jNNxHHGNn//HNbbv+4dr6V+11uF527arU7+u63qfa/bnmh8sWLBgwYJlqRf8MEptXPBXJXa37BSl3ixYsGDBMliwFLyCV/DeLIMFCxYsWLBMwSt4Be/NggXLYMGCBUvBK3iNruC9WbBgwYJlsGApeAWv4L1ZBgsWLFiwYJmCV/AK3psFC5bBggULloJX8BpdwXuzYMGCBctgwVLwCl7Be7MMFixYsGDBsu8FH1FaSmExVfAxBa/gvVmwYMGCZbBg/W4vAQYA5tRF9QYlv/QAAAAASUVORK5CYII=", + "CLEAR_BACKGROUND_COLOR": "#222", "NODE_TITLE_COLOR": "#999", "NODE_SELECTED_TITLE_COLOR": "#FFF", "NODE_TEXT_SIZE": 14, @@ -77,6 +78,8 @@ const colorPalettes = { "VAE": "#FF7043", // deep orange }, "litegraph_base": { + "BACKGROUND_IMAGE": "data:image/gif;base64,R0lGODlhZABkALMAAAAAAP///+vr6+rq6ujo6Ofn5+bm5uXl5d3d3f///wAAAAAAAAAAAAAAAAAAAAAAACH5BAEAAAkALAAAAABkAGQAAAT/UMhJq7046827HkcoHkYxjgZhnGG6si5LqnIM0/fL4qwwIMAg0CAsEovBIxKhRDaNy2GUOX0KfVFrssrNdpdaqTeKBX+dZ+jYvEaTf+y4W66mC8PUdrE879f9d2mBeoNLfH+IhYBbhIx2jkiHiomQlGKPl4uZe3CaeZifnnijgkESBqipqqusra6vsLGys62SlZO4t7qbuby7CLa+wqGWxL3Gv3jByMOkjc2lw8vOoNSi0czAncXW3Njdx9Pf48/Z4Kbbx+fQ5evZ4u3k1fKR6cn03vHlp7T9/v8A/8Gbp4+gwXoFryXMB2qgwoMMHyKEqA5fxX322FG8tzBcRnMW/zlulPbRncmQGidKjMjyYsOSKEF2FBlJQMCbOHP6c9iSZs+UnGYCdbnSo1CZI5F64kn0p1KnTH02nSoV3dGTV7FFHVqVq1dtWcMmVQZTbNGu72zqXMuW7danVL+6e4t1bEy6MeueBYLXrNO5Ze36jQtWsOG97wIj1vt3St/DjTEORss4nNq2mDP3e7w4r1bFkSET5hy6s2TRlD2/mSxXtSHQhCunXo26NevCpmvD/UU6tuullzULH76q92zdZG/Ltv1a+W+osI/nRmyc+fRi1Xdbh+68+0vv10dH3+77KD/i6IdnX669/frn5Zsjh4/2PXju8+8bzc9/6fj27LFnX11/+IUnXWl7BJfegm79FyB9JOl3oHgSklefgxAC+FmFGpqHIYcCfkhgfCohSKKJVo044YUMttggiBkmp6KFXw1oII24oYhjiDByaKOOHcp3Y5BD/njikSkO+eBREQAAOw==", + "CLEAR_BACKGROUND_COLOR": "white", "NODE_TITLE_COLOR": "#222", "NODE_SELECTED_TITLE_COLOR": "#000", "NODE_TEXT_SIZE": 14, @@ -191,7 +194,7 @@ app.registerExtension({ const nodeData = defs[nodeId]; var inputs = nodeData["input"]["required"]; - if (nodeData["input"]["optional"] != undefined){ + if (nodeData["input"]["optional"] != undefined) { inputs = Object.assign({}, nodeData["input"]["required"], nodeData["input"]["optional"]) } @@ -232,12 +235,9 @@ app.registerExtension({ "id": "my_color_palette_unique_id", "name": "My Color Palette", "colors": { - "node_slot": { - }, - "litegraph_base": { - }, - "comfy_base": { - } + "node_slot": {}, + "litegraph_base": {}, + "comfy_base": {} } }; @@ -266,7 +266,7 @@ app.registerExtension({ }; const addCustomColorPalette = async (colorPalette) => { - if (typeof(colorPalette) !== "object") { + if (typeof (colorPalette) !== "object") { app.ui.dialog.show("Invalid color palette"); return; } @@ -286,7 +286,7 @@ app.registerExtension({ return; } - if (colorPalette.colors.node_slot && typeof(colorPalette.colors.node_slot) !== "object") { + if (colorPalette.colors.node_slot && typeof (colorPalette.colors.node_slot) !== "object") { app.ui.dialog.show("Invalid color palette colors.node_slot"); return; } @@ -301,7 +301,11 @@ app.registerExtension({ } } - els.select.append($el("option", { textContent: colorPalette.name + " (custom)", value: "custom_" + colorPalette.id, selected: true })); + els.select.append($el("option", { + textContent: colorPalette.name + " (custom)", + value: "custom_" + colorPalette.id, + selected: true + })); setColorPalette("custom_" + colorPalette.id); await loadColorPalette(colorPalette); @@ -350,7 +354,7 @@ app.registerExtension({ if (colorPalette.colors.comfy_base) { const rootStyle = document.documentElement.style; for (const key in colorPalette.colors.comfy_base) { - rootStyle.setProperty('--' + key, colorPalette.colors.comfy_base[key]); + rootStyle.setProperty('--' + key, colorPalette.colors.comfy_base[key]); } } app.canvas.draw(true, true); @@ -380,7 +384,7 @@ app.registerExtension({ const fileInput = $el("input", { type: "file", accept: ".json", - style: { display: "none" }, + style: {display: "none"}, parent: document.body, onchange: () => { let file = fileInput.files[0]; @@ -403,17 +407,25 @@ app.registerExtension({ for (const c in colorPalettes) { const colorPalette = colorPalettes[c]; - options.push($el("option", { textContent: colorPalette.name, value: colorPalette.id, selected: colorPalette.id === value })); + options.push($el("option", { + textContent: colorPalette.name, + value: colorPalette.id, + selected: colorPalette.id === value + })); } let customColorPalettes = getCustomColorPalettes(); for (const c in customColorPalettes) { const colorPalette = customColorPalettes[c]; - options.push($el("option", { textContent: colorPalette.name + " (custom)", value: "custom_" + colorPalette.id, selected: "custom_" + colorPalette.id === value })); + options.push($el("option", { + textContent: colorPalette.name + " (custom)", + value: "custom_" + colorPalette.id, + selected: "custom_" + colorPalette.id === value + })); } return $el("div", [ - $el("label", { textContent: name || id }, [ + $el("label", {textContent: name || id}, [ els.select = $el("select", { onchange: (e) => { setter(e.target.value); @@ -427,12 +439,12 @@ app.registerExtension({ const colorPaletteId = app.ui.settings.getSettingValue(id, defaultColorPaletteId); const colorPalette = await completeColorPalette(getColorPalette(colorPaletteId)); const json = JSON.stringify(colorPalette, null, 2); // convert the data to a JSON string - const blob = new Blob([json], { type: "application/json" }); + const blob = new Blob([json], {type: "application/json"}); const url = URL.createObjectURL(blob); const a = $el("a", { href: url, download: colorPaletteId + ".json", - style: { display: "none" }, + style: {display: "none"}, parent: document.body, }); a.click(); @@ -455,12 +467,12 @@ app.registerExtension({ onclick: async () => { const colorPalette = await getColorPaletteTemplate(); const json = JSON.stringify(colorPalette, null, 2); // convert the data to a JSON string - const blob = new Blob([json], { type: "application/json" }); + const blob = new Blob([json], {type: "application/json"}); const url = URL.createObjectURL(blob); const a = $el("a", { href: url, download: "color_palette.json", - style: { display: "none" }, + style: {display: "none"}, parent: document.body, }); a.click(); @@ -496,15 +508,25 @@ app.registerExtension({ return; } - if (colorPalettes[value]) { - await loadColorPalette(colorPalettes[value]); + let palette = colorPalettes[value]; + if (palette) { + await loadColorPalette(palette); } else if (value.startsWith("custom_")) { value = value.substr(7); let customColorPalettes = getCustomColorPalettes(); if (customColorPalettes[value]) { + palette = customColorPalettes[value]; await loadColorPalette(customColorPalettes[value]); } } + + let {BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR} = palette.colors.litegraph_base; + if (BACKGROUND_IMAGE === undefined || CLEAR_BACKGROUND_COLOR === undefined) { + const base = colorPalettes["dark"].colors.litegraph_base; + BACKGROUND_IMAGE = base.BACKGROUND_IMAGE; + CLEAR_BACKGROUND_COLOR = base.CLEAR_BACKGROUND_COLOR; + } + app.canvas.updateBackground(BACKGROUND_IMAGE, CLEAR_BACKGROUND_COLOR); }, }); }, diff --git a/web/index.html b/web/index.html index da0adb6c2..c48d716e1 100644 --- a/web/index.html +++ b/web/index.html @@ -7,6 +7,7 @@ +