From e930a387d62cc819117502993b0b821b1e3f2687 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Mon, 19 May 2025 01:58:41 -0700 Subject: [PATCH 1/8] Update AMD instructions in README. (#8198) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9b5f301c9..15157f527 100644 --- a/README.md +++ b/README.md @@ -197,11 +197,11 @@ Put your VAE in: models/vae ### AMD GPUs (Linux only) AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version: -```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.2.4``` +```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.3``` This is the command to install the nightly with ROCm 6.3 which might have some performance improvements: -```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.3``` +```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4``` ### Intel GPUs (Windows and Linux) From 4f3b50ba510e02fa3fdd8c755ef9ad319b36bd61 Mon Sep 17 00:00:00 2001 From: filtered <176114999+webfiltered@users.noreply.github.com> Date: Tue, 20 May 2025 06:40:55 +1000 Subject: [PATCH 2/8] Update README ROCm text to match link (#8199) - Follow-up on #8198 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15157f527..47514d1b4 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ AMD users can install rocm and pytorch with pip if you don't have it already ins ```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.3``` -This is the command to install the nightly with ROCm 6.3 which might have some performance improvements: +This is the command to install the nightly with ROCm 6.4 which might have some performance improvements: ```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4``` From 7e84bf53737879ace37a68dc93e0df7704a53514 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Tue, 20 May 2025 02:29:23 -0700 Subject: [PATCH 3/8] This doesn't seem to be needed on chroma. (#8209) --- comfy/ldm/chroma/layers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/comfy/ldm/chroma/layers.py b/comfy/ldm/chroma/layers.py index 35da91ee2..18a4a9cfc 100644 --- a/comfy/ldm/chroma/layers.py +++ b/comfy/ldm/chroma/layers.py @@ -109,9 +109,6 @@ class DoubleStreamBlock(nn.Module): txt += txt_mod1.gate * self.txt_attn.proj(txt_attn) txt += txt_mod2.gate * self.txt_mlp((1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift) - if txt.dtype == torch.float16: - txt = torch.nan_to_num(txt, nan=0.0, posinf=65504, neginf=-65504) - return img, txt @@ -163,8 +160,6 @@ class SingleStreamBlock(nn.Module): # compute activation in mlp stream, cat again and run second linear layer output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2)) x += mod.gate * output - if x.dtype == torch.float16: - x = torch.nan_to_num(x, nan=0.0, posinf=65504, neginf=-65504) return x From 87f91307782ce0b401786d8edddd8f618b955141 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Tue, 20 May 2025 02:39:55 -0700 Subject: [PATCH 4/8] Revert "This doesn't seem to be needed on chroma. (#8209)" (#8210) This reverts commit 7e84bf53737879ace37a68dc93e0df7704a53514. --- comfy/ldm/chroma/layers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/comfy/ldm/chroma/layers.py b/comfy/ldm/chroma/layers.py index 18a4a9cfc..35da91ee2 100644 --- a/comfy/ldm/chroma/layers.py +++ b/comfy/ldm/chroma/layers.py @@ -109,6 +109,9 @@ class DoubleStreamBlock(nn.Module): txt += txt_mod1.gate * self.txt_attn.proj(txt_attn) txt += txt_mod2.gate * self.txt_mlp((1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift) + if txt.dtype == torch.float16: + txt = torch.nan_to_num(txt, nan=0.0, posinf=65504, neginf=-65504) + return img, txt @@ -160,6 +163,8 @@ class SingleStreamBlock(nn.Module): # compute activation in mlp stream, cat again and run second linear layer output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2)) x += mod.gate * output + if x.dtype == torch.float16: + x = torch.nan_to_num(x, nan=0.0, posinf=65504, neginf=-65504) return x From 10024a38ea8d7e8950b26500a540cd0323d0e611 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 21 May 2025 04:50:37 -0400 Subject: [PATCH 5/8] ComfyUI version v0.3.35 --- comfyui_version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/comfyui_version.py b/comfyui_version.py index b740b378d..8db3bc803 100644 --- a/comfyui_version.py +++ b/comfyui_version.py @@ -1,3 +1,3 @@ # This file is automatically generated by the build process when version is # updated in pyproject.toml. -__version__ = "0.3.34" +__version__ = "0.3.35" diff --git a/pyproject.toml b/pyproject.toml index 80061b39a..a33fc4370 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ComfyUI" -version = "0.3.34" +version = "0.3.35" readme = "README.md" license = { file = "LICENSE" } requires-python = ">=3.9" From 65da29aaa965afcb0811a9c8dac1cc0facb006d4 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 21 May 2025 01:56:56 -0700 Subject: [PATCH 6/8] Make torch.compile LoRA/key-compatible (#8213) * Make torch compile node use wrapper instead of object_patch for the entire diffusion_models object, allowing key assotiations on diffusion_models to not break (loras, getting attributes, etc.) * Moved torch compile code into comfy_api so it can be used by custom nodes with a degree of confidence * Refactor set_torch_compile_wrapper to support a list of keys instead of just diffusion_model, as well as additional torch.compile args * remove unused import * Moved torch compile kwargs to be stored in model_options instead of attachments; attachments are more intended for things to be 'persisted', AKA not deepcopied * Add some comments * Remove random line of code, not sure how it got there --- comfy_api/torch_helpers/__init__.py | 5 ++ comfy_api/torch_helpers/torch_compile.py | 69 ++++++++++++++++++++++++ comfy_extras/nodes_torch_compile.py | 5 +- 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 comfy_api/torch_helpers/__init__.py create mode 100644 comfy_api/torch_helpers/torch_compile.py diff --git a/comfy_api/torch_helpers/__init__.py b/comfy_api/torch_helpers/__init__.py new file mode 100644 index 000000000..be7ae7a61 --- /dev/null +++ b/comfy_api/torch_helpers/__init__.py @@ -0,0 +1,5 @@ +from .torch_compile import set_torch_compile_wrapper + +__all__ = [ + "set_torch_compile_wrapper", +] diff --git a/comfy_api/torch_helpers/torch_compile.py b/comfy_api/torch_helpers/torch_compile.py new file mode 100644 index 000000000..9223f58db --- /dev/null +++ b/comfy_api/torch_helpers/torch_compile.py @@ -0,0 +1,69 @@ +from __future__ import annotations +import torch + +import comfy.utils +from comfy.patcher_extension import WrappersMP +from typing import TYPE_CHECKING, Callable, Optional +if TYPE_CHECKING: + from comfy.model_patcher import ModelPatcher + from comfy.patcher_extension import WrapperExecutor + + +COMPILE_KEY = "torch.compile" +TORCH_COMPILE_KWARGS = "torch_compile_kwargs" + + +def apply_torch_compile_factory(compiled_module_dict: dict[str, Callable]) -> Callable: + ''' + Create a wrapper that will refer to the compiled_diffusion_model. + ''' + def apply_torch_compile_wrapper(executor: WrapperExecutor, *args, **kwargs): + try: + orig_modules = {} + for key, value in compiled_module_dict.items(): + orig_modules[key] = comfy.utils.get_attr(executor.class_obj, key) + comfy.utils.set_attr(executor.class_obj, key, value) + return executor(*args, **kwargs) + finally: + for key, value in orig_modules.items(): + comfy.utils.set_attr(executor.class_obj, key, value) + return apply_torch_compile_wrapper + + +def set_torch_compile_wrapper(model: ModelPatcher, backend: str, options: Optional[dict[str,str]]=None, + mode: Optional[str]=None, fullgraph=False, dynamic: Optional[bool]=None, + keys: list[str]=["diffusion_model"], *args, **kwargs): + ''' + Perform torch.compile that will be applied at sample time for either the whole model or specific params of the BaseModel instance. + + When keys is None, it will default to using ["diffusion_model"], compiling the whole diffusion_model. + When a list of keys is provided, it will perform torch.compile on only the selected modules. + ''' + # clear out any other torch.compile wrappers + model.remove_wrappers_with_key(WrappersMP.APPLY_MODEL, COMPILE_KEY) + # if no keys, default to 'diffusion_model' + if not keys: + keys = ["diffusion_model"] + # create kwargs dict that can be referenced later + compile_kwargs = { + "backend": backend, + "options": options, + "mode": mode, + "fullgraph": fullgraph, + "dynamic": dynamic, + } + # get a dict of compiled keys + compiled_modules = {} + for key in keys: + compiled_modules[key] = torch.compile( + model=model.get_model_object(key), + **compile_kwargs, + ) + # add torch.compile wrapper + wrapper_func = apply_torch_compile_factory( + compiled_module_dict=compiled_modules, + ) + # store wrapper to run on BaseModel's apply_model function + model.add_wrapper_with_key(WrappersMP.APPLY_MODEL, COMPILE_KEY, wrapper_func) + # keep compile kwargs for reference + model.model_options[TORCH_COMPILE_KWARGS] = compile_kwargs diff --git a/comfy_extras/nodes_torch_compile.py b/comfy_extras/nodes_torch_compile.py index 1fe6f42c7..605536678 100644 --- a/comfy_extras/nodes_torch_compile.py +++ b/comfy_extras/nodes_torch_compile.py @@ -1,4 +1,5 @@ -import torch +from comfy_api.torch_helpers import set_torch_compile_wrapper + class TorchCompileModel: @classmethod @@ -14,7 +15,7 @@ class TorchCompileModel: def patch(self, model, backend): m = model.clone() - m.add_object_patch("diffusion_model", torch.compile(model=m.get_model_object("diffusion_model"), backend=backend)) + set_torch_compile_wrapper(model=m, backend=backend) return (m, ) NODE_CLASS_MAPPINGS = { From 57893c843f44ea9e8a0be79292d19e5a5e16e9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BC=96=E7=A8=8B=E7=95=8C=E7=9A=84=E5=B0=8F=E5=AD=A6?= =?UTF-8?q?=E7=94=9F?= <15620646321@163.com> Date: Wed, 21 May 2025 16:59:42 +0800 Subject: [PATCH 7/8] Code Optimization and Issues Fixes in ComfyUI server (#8196) * Update server.py * Update server.py --- server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server.py b/server.py index cb1c6a8fd..16cd88d91 100644 --- a/server.py +++ b/server.py @@ -226,7 +226,7 @@ class PromptServer(): return response @routes.get("/embeddings") - def get_embeddings(self): + def get_embeddings(request): embeddings = folder_paths.get_filename_list("embeddings") return web.json_response(list(map(lambda a: os.path.splitext(a)[0], embeddings))) @@ -282,7 +282,6 @@ class PromptServer(): a.update(f.read()) b.update(image.file.read()) image.file.seek(0) - f.close() return a.hexdigest() == b.hexdigest() return False From 8bb858e4d39f7f6a6969c584aeeaa1d606a812d6 Mon Sep 17 00:00:00 2001 From: Michael Abrahams Date: Wed, 21 May 2025 05:14:17 -0400 Subject: [PATCH 8/8] Improve performance with large number of queued prompts (#8176) * get_current_queue_volatile * restore get_current_queue method * remove extra import --- execution.py | 9 ++++++++- main.py | 3 +-- server.py | 5 +++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/execution.py b/execution.py index e5d1c69d9..15ff7567c 100644 --- a/execution.py +++ b/execution.py @@ -909,7 +909,6 @@ class PromptQueue: self.currently_running = {} self.history = {} self.flags = {} - server.prompt_queue = self def put(self, item): with self.mutex: @@ -954,6 +953,7 @@ class PromptQueue: self.history[prompt[1]].update(history_result) self.server.queue_updated() + # Note: slow def get_current_queue(self): with self.mutex: out = [] @@ -961,6 +961,13 @@ class PromptQueue: out += [x] return (out, copy.deepcopy(self.queue)) + # read-safe as long as queue items are immutable + def get_current_queue_volatile(self): + with self.mutex: + running = [x for x in self.currently_running.values()] + queued = copy.copy(self.queue) + return (running, queued) + def get_tasks_remaining(self): with self.mutex: return len(self.queue) + len(self.currently_running) diff --git a/main.py b/main.py index 0fde6d221..fb1f8d20b 100644 --- a/main.py +++ b/main.py @@ -260,7 +260,6 @@ def start_comfyui(asyncio_loop=None): asyncio_loop = asyncio.new_event_loop() asyncio.set_event_loop(asyncio_loop) prompt_server = server.PromptServer(asyncio_loop) - q = execution.PromptQueue(prompt_server) hook_breaker_ac10a0.save_functions() nodes.init_extra_nodes(init_custom_nodes=not args.disable_all_custom_nodes, init_api_nodes=not args.disable_api_nodes) @@ -271,7 +270,7 @@ def start_comfyui(asyncio_loop=None): prompt_server.add_routes() hijack_progress(prompt_server) - threading.Thread(target=prompt_worker, daemon=True, args=(q, prompt_server,)).start() + threading.Thread(target=prompt_worker, daemon=True, args=(prompt_server.prompt_queue, prompt_server,)).start() if args.quick_test_for_ci: exit(0) diff --git a/server.py b/server.py index 16cd88d91..1b0a73601 100644 --- a/server.py +++ b/server.py @@ -29,6 +29,7 @@ import comfy.model_management import node_helpers from comfyui_version import __version__ from app.frontend_management import FrontendManager + from app.user_manager import UserManager from app.model_manager import ModelFileManager from app.custom_node_manager import CustomNodeManager @@ -159,7 +160,7 @@ class PromptServer(): self.custom_node_manager = CustomNodeManager() self.internal_routes = InternalRoutes(self) self.supports = ["custom_nodes_from_web"] - self.prompt_queue = None + self.prompt_queue = execution.PromptQueue(self) self.loop = loop self.messages = asyncio.Queue() self.client_session:Optional[aiohttp.ClientSession] = None @@ -620,7 +621,7 @@ class PromptServer(): @routes.get("/queue") async def get_queue(request): queue_info = {} - current_queue = self.prompt_queue.get_current_queue() + current_queue = self.prompt_queue.get_current_queue_volatile() queue_info['queue_running'] = current_queue[0] queue_info['queue_pending'] = current_queue[1] return web.json_response(queue_info)