From 422163c2ba65b18d8208d4661d27c7e312e8d862 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Tue, 6 Jun 2023 22:27:44 +0900 Subject: [PATCH 1/7] bugfix: Fixing the calculation issue when an image widget is added to the size calculation of the text widget. --- web/scripts/app.js | 4 ++++ web/scripts/widgets.js | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index 9df94c9eb..27c67fb49 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -365,6 +365,10 @@ export class ComfyApp { } node.prototype.setSizeForImage = function () { + if (this.inputHeight) { + this.setSize(this.size); + return; + } const minHeight = getImageTop(this) + 220; if (this.size[1] < minHeight) { this.setSize([this.size[0], minHeight]); diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js index d6faaddbf..dfa26aef4 100644 --- a/web/scripts/widgets.js +++ b/web/scripts/widgets.js @@ -115,12 +115,12 @@ function addMultilineWidget(node, name, opts, app) { // See how large each text input can be freeSpace -= widgetHeight; - freeSpace /= multi.length; + freeSpace /= multi.length + (!!node.imgs?.length); if (freeSpace < MIN_SIZE) { // There isnt enough space for all the widgets, increase the size of the node freeSpace = MIN_SIZE; - node.size[1] = y + widgetHeight + freeSpace * multi.length; + node.size[1] = y + widgetHeight + freeSpace * (multi.length + (!!node.imgs?.length)); node.graph.setDirtyCanvas(true); } From 3b5b095d04f73f85b19f7c44aaffe481081f3818 Mon Sep 17 00:00:00 2001 From: reaper47 Date: Tue, 6 Jun 2023 17:40:07 +0200 Subject: [PATCH 2/7] Add .idea/ to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index df6adbe4b..8380a2f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ custom_nodes/ !custom_nodes/example_node.py.example extra_model_paths.yaml /.vs +.idea/ \ No newline at end of file From 0e425603fb8ba12f1e7d09a1f58127347a94de98 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 6 Jun 2023 03:25:49 -0400 Subject: [PATCH 3/7] Small refactor. --- comfy/sd.py | 21 +++++---------------- comfy/utils.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 336fee4a6..04eaaa9fe 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -31,17 +31,6 @@ def load_model_weights(model, sd, verbose=False, load_state_dict_to=[]): if ids.dtype == torch.float32: sd['cond_stage_model.transformer.text_model.embeddings.position_ids'] = ids.round() - keys_to_replace = { - "cond_stage_model.model.positional_embedding": "cond_stage_model.transformer.text_model.embeddings.position_embedding.weight", - "cond_stage_model.model.token_embedding.weight": "cond_stage_model.transformer.text_model.embeddings.token_embedding.weight", - "cond_stage_model.model.ln_final.weight": "cond_stage_model.transformer.text_model.final_layer_norm.weight", - "cond_stage_model.model.ln_final.bias": "cond_stage_model.transformer.text_model.final_layer_norm.bias", - } - - for x in keys_to_replace: - if x in sd: - sd[keys_to_replace[x]] = sd.pop(x) - sd = utils.transformers_convert(sd, "cond_stage_model.model", "cond_stage_model.transformer.text_model", 24) for x in load_state_dict_to: @@ -1073,13 +1062,13 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o "legacy": False } - if len(sd['model.diffusion_model.input_blocks.1.1.proj_in.weight'].shape) == 2: + if len(sd['model.diffusion_model.input_blocks.4.1.proj_in.weight'].shape) == 2: unet_config['use_linear_in_transformer'] = True unet_config["use_fp16"] = fp16 unet_config["model_channels"] = sd['model.diffusion_model.input_blocks.0.0.weight'].shape[0] unet_config["in_channels"] = sd['model.diffusion_model.input_blocks.0.0.weight'].shape[1] - unet_config["context_dim"] = sd['model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn2.to_k.weight'].shape[1] + unet_config["context_dim"] = sd['model.diffusion_model.input_blocks.4.1.transformer_blocks.0.attn2.to_k.weight'].shape[1] sd_config["unet_config"] = {"target": "comfy.ldm.modules.diffusionmodules.openaimodel.UNetModel", "params": unet_config} model_config = {"target": "comfy.ldm.models.diffusion.ddpm.LatentDiffusion", "params": sd_config} @@ -1097,10 +1086,10 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o else: sd_config["conditioning_key"] = "crossattn" - if unet_config["context_dim"] == 1024: - unet_config["num_head_channels"] = 64 #SD2.x - else: + if unet_config["context_dim"] == 768: unet_config["num_heads"] = 8 #SD1.x + else: + unet_config["num_head_channels"] = 64 #SD2.x unclip = 'model.diffusion_model.label_emb.0.0.weight' if unclip in sd_keys: diff --git a/comfy/utils.py b/comfy/utils.py index 291c62e42..585ebda51 100644 --- a/comfy/utils.py +++ b/comfy/utils.py @@ -24,6 +24,18 @@ def load_torch_file(ckpt, safe_load=False): return sd def transformers_convert(sd, prefix_from, prefix_to, number): + keys_to_replace = { + "{}.positional_embedding": "{}.embeddings.position_embedding.weight", + "{}.token_embedding.weight": "{}.embeddings.token_embedding.weight", + "{}.ln_final.weight": "{}.final_layer_norm.weight", + "{}.ln_final.bias": "{}.final_layer_norm.bias", + } + + for k in keys_to_replace: + x = k.format(prefix_from) + if x in sd: + sd[keys_to_replace[k].format(prefix_to)] = sd.pop(x) + resblock_to_replace = { "ln_1": "layer_norm1", "ln_2": "layer_norm2", From 5cf4079923c94ab3a507d89674bca3bf6f3dbc5b Mon Sep 17 00:00:00 2001 From: reaper47 Date: Wed, 7 Jun 2023 15:15:38 +0200 Subject: [PATCH 4/7] Give linux some love --- README.md | 51 ++++++++++++++++++++++++++------------------------- main.py | 20 +++++++++++--------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index d998afe65..55cd25c72 100644 --- a/README.md +++ b/README.md @@ -38,28 +38,28 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git ## Shortcuts -| Keybind | Explanation | -| - | - | -| Ctrl + Enter | Queue up current graph for generation | -| Ctrl + Shift + Enter | Queue up current graph as first for generation | -| Ctrl + S | Save workflow | -| Ctrl + O | Load workflow | -| Ctrl + A | Select all nodes | -| Ctrl + M | Mute/unmute selected nodes | -| Delete/Backspace | Delete selected nodes | -| Ctrl + Delete/Backspace | Delete the current graph | -| Space | Move the canvas around when held and moving the cursor | -| Ctrl/Shift + Click | Add clicked node to selection | -| Ctrl + C/Ctrl + V | Copy and paste selected nodes (without maintaining connections to outputs of unselected nodes) | -| Ctrl + C/Ctrl + Shift + V| Copy and paste selected nodes (maintaining connections from outputs of unselected nodes to inputs of pasted nodes) | -| Shift + Drag | Move multiple selected nodes at the same time | -| Ctrl + D | Load default graph | -| Q | Toggle visibility of the queue | -| H | Toggle visibility of history | -| R | Refresh graph | -| Double-Click LMB | Open node quick search palette | +| Keybind | Explanation | +|---------------------------|--------------------------------------------------------------------------------------------------------------------| +| Ctrl + Enter | Queue up current graph for generation | +| Ctrl + Shift + Enter | Queue up current graph as first for generation | +| Ctrl + S | Save workflow | +| Ctrl + O | Load workflow | +| Ctrl + A | Select all nodes | +| Ctrl + M | Mute/unmute selected nodes | +| Delete/Backspace | Delete selected nodes | +| Ctrl + Delete/Backspace | Delete the current graph | +| Space | Move the canvas around when held and moving the cursor | +| Ctrl/Shift + Click | Add clicked node to selection | +| Ctrl + C/Ctrl + V | Copy and paste selected nodes (without maintaining connections to outputs of unselected nodes) | +| Ctrl + C/Ctrl + Shift + V | Copy and paste selected nodes (maintaining connections from outputs of unselected nodes to inputs of pasted nodes) | +| Shift + Drag | Move multiple selected nodes at the same time | +| Ctrl + D | Load default graph | +| Q | Toggle visibility of the queue | +| H | Toggle visibility of history | +| R | Refresh graph | +| Double-Click LMB | Open node quick search palette | -Ctrl can also be replaced with Cmd instead for MacOS users +Ctrl can also be replaced with Cmd instead for macOS users # Installing @@ -77,7 +77,8 @@ See the [Config file](extra_model_paths.yaml.example) to set the search paths fo ## Colab Notebook -To run it on colab or paperspace you can use my [Colab Notebook](notebooks/comfyui_colab.ipynb) here: [Link to open with google colab](https://colab.research.google.com/github/comfyanonymous/ComfyUI/blob/master/notebooks/comfyui_colab.ipynb) +To run it on colab or paperspace you can use my [Colab Notebook](notebooks/comfyui_colab.ipynb) here: +Link to open with Google colab ## Manual Install (Windows, Linux) @@ -125,7 +126,7 @@ Mac/MPS: There is basic support in the code but until someone makes some install ### I already have another UI for Stable Diffusion installed do I really have to install all of these dependencies? -You don't. If you have another UI installed and working with it's own python venv you can use that venv to run ComfyUI. You can open up your favorite terminal and activate it: +You don't. If you have another UI installed and working with its own python venv you can use that venv to run ComfyUI. You can open up your favorite terminal and activate it: ```source path_to_other_sd_gui/venv/bin/activate``` @@ -135,7 +136,7 @@ With Powershell: ```"path_to_other_sd_gui\venv\Scripts\Activate.ps1"``` With cmd.exe: ```"path_to_other_sd_gui\venv\Scripts\activate.bat"``` -And then you can use that terminal to run Comfyui without installing any dependencies. Note that the venv folder might be called something else depending on the SD UI. +And then you can use that terminal to run ComfyUI without installing any dependencies. Note that the venv folder might be called something else depending on the SD UI. # Running @@ -190,7 +191,7 @@ The default installation includes a fast latent preview method that's low-resolu ## Support and dev channel -[Matrix space: #comfyui_space:matrix.org](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) (it's like discord but open source). +Matrix space: #comfyui_space:matrix.org (it's like discord but open source). # QA diff --git a/main.py b/main.py index 15f75f892..8293c06fc 100644 --- a/main.py +++ b/main.py @@ -37,21 +37,25 @@ def prompt_worker(q, server): e.execute(item[2], item[1], item[3], item[4]) q.task_done(item_id, e.outputs_ui) + async def run(server, address='', port=8188, verbose=True, call_on_start=None): await asyncio.gather(server.start(address, port, verbose, call_on_start), server.publish_loop()) + def hijack_progress(server): def hook(value, total, preview_image_bytes): - server.send_sync("progress", { "value": value, "max": total}, server.client_id) + server.send_sync("progress", {"value": value, "max": total}, server.client_id) if preview_image_bytes is not None: server.send_sync(BinaryEventTypes.PREVIEW_IMAGE, preview_image_bytes, server.client_id) comfy.utils.set_progress_bar_global_hook(hook) + def cleanup_temp(): temp_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "temp") if os.path.exists(temp_dir): shutil.rmtree(temp_dir, ignore_errors=True) + def load_extra_path_config(yaml_path): with open(yaml_path, 'r') as stream: config = yaml.safe_load(stream) @@ -72,6 +76,7 @@ def load_extra_path_config(yaml_path): print("Adding extra search path", x, full_path) folder_paths.add_model_folder_path(x, full_path) + if __name__ == "__main__": cleanup_temp() @@ -92,7 +97,7 @@ if __name__ == "__main__": server.add_routes() hijack_progress(server) - threading.Thread(target=prompt_worker, daemon=True, args=(q,server,)).start() + threading.Thread(target=prompt_worker, daemon=True, args=(q, server,)).start() if args.output_directory: output_dir = os.path.abspath(args.output_directory) @@ -106,15 +111,12 @@ if __name__ == "__main__": if args.auto_launch: def startup_server(address, port): import webbrowser - webbrowser.open("http://{}:{}".format(address, port)) + webbrowser.open(f"http://{address}:{port}") call_on_start = startup_server - if os.name == "nt": - try: - loop.run_until_complete(run(server, address=args.listen, port=args.port, verbose=not args.dont_print_server, call_on_start=call_on_start)) - except KeyboardInterrupt: - pass - else: + try: loop.run_until_complete(run(server, address=args.listen, port=args.port, verbose=not args.dont_print_server, call_on_start=call_on_start)) + except KeyboardInterrupt: + print("\nStopped server") cleanup_temp() From 70e02b443f6b803ba4d08aa9cc0f0286f9e5dd2b Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Wed, 7 Jun 2023 22:56:08 +0900 Subject: [PATCH 5/7] robust patch on pasteFromClipspace --- web/scripts/app.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index 27c67fb49..657ea0246 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -125,10 +125,14 @@ export class ComfyApp { if(ComfyApp.clipspace.imgs && node.imgs) { if(node.images && ComfyApp.clipspace.images) { if(ComfyApp.clipspace['img_paste_mode'] == 'selected') { - app.nodeOutputs[node.id + ""].images = node.images = [ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']]]; + node.images = [ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']]]; } - else - app.nodeOutputs[node.id + ""].images = node.images = ComfyApp.clipspace.images; + else { + node.images = ComfyApp.clipspace.images; + } + + if(app.nodeOutputs[node.id + ""]) + app.nodeOutputs[node.id + ""].images = node.images; } if(ComfyApp.clipspace.imgs) { From 28677342c1d2f2eb86aadd2e8fceac9c2f6196bc Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Thu, 8 Jun 2023 00:06:56 +0900 Subject: [PATCH 6/7] robust paste for image --- web/scripts/app.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index 657ea0246..385a54579 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -165,7 +165,16 @@ export class ComfyApp { if(ComfyApp.clipspace.widgets) { ComfyApp.clipspace.widgets.forEach(({ type, name, value }) => { const prop = Object.values(node.widgets).find(obj => obj.type === type && obj.name === name); - if (prop && prop.type != 'button') { + if (prop && prop.type != 'image') { + if(typeof prop.value == "string" && value.filename) { + prop.value = (value.subfolder?value.subfolder+'/':'') + value.filename + (value.type?` [${value.type}]`:''); + } + else { + prop.value = value; + prop.callback(value); + } + } + else if (prop && prop.type != 'button') { prop.value = value; prop.callback(value); } From 29c50954eace60fe8422c63ff4bf57c389a1ba76 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 8 Jun 2023 02:00:44 -0400 Subject: [PATCH 7/7] Add some quick instructions how to use directml. --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 55cd25c72..78f34a9bb 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,7 @@ See the [Config file](extra_model_paths.yaml.example) to set the search paths fo ## Colab Notebook -To run it on colab or paperspace you can use my [Colab Notebook](notebooks/comfyui_colab.ipynb) here: -Link to open with Google colab +To run it on colab or paperspace you can use my [Colab Notebook](notebooks/comfyui_colab.ipynb) here: [Link to open with google colab](https://colab.research.google.com/github/comfyanonymous/ComfyUI/blob/master/notebooks/comfyui_colab.ipynb) ## Manual Install (Windows, Linux) @@ -124,6 +123,9 @@ After this you should have everything installed and can proceed to running Comfy Mac/MPS: There is basic support in the code but until someone makes some install instruction you are on your own. +Directml: ```pip install torch-directml``` Then you can launch ComfyUI with: ```python main.py --directml``` + + ### I already have another UI for Stable Diffusion installed do I really have to install all of these dependencies? You don't. If you have another UI installed and working with its own python venv you can use that venv to run ComfyUI. You can open up your favorite terminal and activate it: @@ -191,7 +193,7 @@ The default installation includes a fast latent preview method that's low-resolu ## Support and dev channel -Matrix space: #comfyui_space:matrix.org (it's like discord but open source). +[Matrix space: #comfyui_space:matrix.org](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) (it's like discord but open source). # QA