From b2fd1e2acbaa5ae0e7a27057a42b48c90aedd8f9 Mon Sep 17 00:00:00 2001 From: doctorpangloss <@hiddenswitch.com> Date: Sun, 25 Aug 2024 17:08:19 -0700 Subject: [PATCH] Upstream nf4 nodes and prepare for flash-attention 2.6 --- README.md | 7 +- comfy_extras/nodes/nodes_nf4.py | 10 +- tests/inference/workflows/flux-1.json | 174 ++++++++++++++++++++++++++ tests/inference/workflows/flux-2.json | 163 ++++++++++++++++++++++++ 4 files changed, 345 insertions(+), 9 deletions(-) create mode 100644 tests/inference/workflows/flux-1.json create mode 100644 tests/inference/workflows/flux-2.json diff --git a/README.md b/README.md index da62611c6..8070e7bef 100644 --- a/README.md +++ b/README.md @@ -124,12 +124,13 @@ When using Windows, open the **Windows Powershell** app. Then observe you are at ```shell pip install git+https://github.com/hiddenswitch/ComfyUI.git ``` - **Recommended**: Currently, `torch 2.3.0` is the last version that `xformers` is compatible with. On Windows, install it first, along with `xformers`, for maximum compatibility and the best performance without advanced techniques in ComfyUI: + **Recommended**: Currently, `torch 2.4.0` is the last version that `xformers` is compatible with. On Windows, install it first, along with `xformers`, for maximum compatibility and the best performance without advanced techniques in ComfyUI: ```shell - pip install torch==2.3.0+cu121 torchvision --index-url https://download.pytorch.org/whl/cu121 - pip install xformers==0.0.26.post1 + pip install torch==2.4.0+cu121 torchvision --index-url https://download.pytorch.org/whl/cu121 + pip install xformers==0.0.27.post2 pip install --no-build-isolation git+https://github.com/hiddenswitch/ComfyUI.git ``` + For improved performance when using the language models on Windows, CUDA 12.1 and PyTorch 2.3.0, add: ```shell pip install flash-attn @ https://github.com/AppMana/appmana-comfyui-nodes-extramodels/releases/download/v0.0.0-flash_attn/flash_attn-2.5.9.post1-cp311-cp311-win_amd64.whl diff --git a/comfy_extras/nodes/nodes_nf4.py b/comfy_extras/nodes/nodes_nf4.py index df99d1759..d61f8380a 100644 --- a/comfy_extras/nodes/nodes_nf4.py +++ b/comfy_extras/nodes/nodes_nf4.py @@ -1,10 +1,9 @@ -import comfy.sd -import torch import bitsandbytes as bnb -import comfy.ops - +import torch from bitsandbytes.nn.modules import Params4bit, QuantState +import comfy.ops +import comfy.sd from comfy.cmd.folder_paths import get_folder_paths from comfy.model_downloader import get_filename_list_with_downloadable, get_or_download @@ -150,8 +149,7 @@ class OPS(comfy.ops.manual_cast): self.weight = self.weight.to(layer_original_device) return out else: - weight, bias = comfy.ops.cast_bias_weight(self, x) - return functional_linear_4bits(x, weight, bias) + raise ValueError("should not be reached") class CheckpointLoaderNF4: diff --git a/tests/inference/workflows/flux-1.json b/tests/inference/workflows/flux-1.json new file mode 100644 index 000000000..fb46f7958 --- /dev/null +++ b/tests/inference/workflows/flux-1.json @@ -0,0 +1,174 @@ +{ + "1": { + "inputs": { + "noise": [ + "2", + 0 + ], + "guider": [ + "3", + 0 + ], + "sampler": [ + "6", + 0 + ], + "sigmas": [ + "7", + 0 + ], + "latent_image": [ + "9", + 0 + ] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + } + }, + "2": { + "inputs": { + "noise_seed": 0 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + }, + "3": { + "inputs": { + "model": [ + "17", + 0 + ], + "conditioning": [ + "4", + 0 + ] + }, + "class_type": "BasicGuider", + "_meta": { + "title": "BasicGuider" + } + }, + "4": { + "inputs": { + "guidance": 3, + "conditioning": [ + "13", + 0 + ] + }, + "class_type": "FluxGuidance", + "_meta": { + "title": "FluxGuidance" + } + }, + "6": { + "inputs": { + "sampler_name": "euler" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "7": { + "inputs": { + "scheduler": "ddim_uniform", + "steps": 1, + "denoise": 1, + "model": [ + "17", + 0 + ] + }, + "class_type": "BasicScheduler", + "_meta": { + "title": "BasicScheduler" + } + }, + "9": { + "inputs": { + "width": 1344, + "height": 768, + "batch_size": 1 + }, + "class_type": "EmptySD3LatentImage", + "_meta": { + "title": "EmptySD3LatentImage" + } + }, + "10": { + "inputs": { + "samples": [ + "1", + 0 + ], + "vae": [ + "11", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "11": { + "inputs": { + "vae_name": "ae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "13": { + "inputs": { + "text": "A plastic Barbie doll is walking along Sunset Boulevard. Here is a list of essential elements of it:\n\nArt Deco and Streamline Moderne buildings from the 1920s and 1930s.\nThe Sunset Tower Hotel: A striking Art Deco landmark with a pale pink facade and stepped design.\nChateau Marmont: A Gothic-style castle-like hotel with white stucco walls and red tile roof.\nNumerous billboards and large advertisements, often for upcoming films or TV shows.\nPalm trees lining portions of the street", + "clip": [ + "18", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "16": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "10", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "17": { + "inputs": { + "ckpt_name": "flux1-dev-bnb-nf4-v2.safetensors" + }, + "class_type": "CheckpointLoaderNF4", + "_meta": { + "title": "CheckpointLoaderNF4" + } + }, + "18": { + "inputs": { + "clip_name1": "clip_l.safetensors", + "clip_name2": "t5xxl_fp16.safetensors", + "type": "flux" + }, + "class_type": "DualCLIPLoader", + "_meta": { + "title": "DualCLIPLoader" + } + } +} \ No newline at end of file diff --git a/tests/inference/workflows/flux-2.json b/tests/inference/workflows/flux-2.json new file mode 100644 index 000000000..224aa7181 --- /dev/null +++ b/tests/inference/workflows/flux-2.json @@ -0,0 +1,163 @@ +{ + "1": { + "inputs": { + "noise": [ + "2", + 0 + ], + "guider": [ + "3", + 0 + ], + "sampler": [ + "6", + 0 + ], + "sigmas": [ + "7", + 0 + ], + "latent_image": [ + "9", + 0 + ] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + } + }, + "2": { + "inputs": { + "noise_seed": 0 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + }, + "3": { + "inputs": { + "model": [ + "17", + 0 + ], + "conditioning": [ + "4", + 0 + ] + }, + "class_type": "BasicGuider", + "_meta": { + "title": "BasicGuider" + } + }, + "4": { + "inputs": { + "guidance": 3, + "conditioning": [ + "13", + 0 + ] + }, + "class_type": "FluxGuidance", + "_meta": { + "title": "FluxGuidance" + } + }, + "6": { + "inputs": { + "sampler_name": "euler" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "7": { + "inputs": { + "scheduler": "ddim_uniform", + "steps": 1, + "denoise": 1, + "model": [ + "17", + 0 + ] + }, + "class_type": "BasicScheduler", + "_meta": { + "title": "BasicScheduler" + } + }, + "9": { + "inputs": { + "width": 1344, + "height": 768, + "batch_size": 1 + }, + "class_type": "EmptySD3LatentImage", + "_meta": { + "title": "EmptySD3LatentImage" + } + }, + "10": { + "inputs": { + "samples": [ + "1", + 0 + ], + "vae": [ + "11", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "11": { + "inputs": { + "vae_name": "ae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "13": { + "inputs": { + "text": "A plastic Barbie doll is walking along Sunset Boulevard. Here is a list of essential elements of it:\n\nArt Deco and Streamline Moderne buildings from the 1920s and 1930s.\nThe Sunset Tower Hotel: A striking Art Deco landmark with a pale pink facade and stepped design.\nChateau Marmont: A Gothic-style castle-like hotel with white stucco walls and red tile roof.\nNumerous billboards and large advertisements, often for upcoming films or TV shows.\nPalm trees lining portions of the street", + "clip": [ + "17", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "16": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "10", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "17": { + "inputs": { + "ckpt_name": "flux1-dev-bnb-nf4-v2.safetensors" + }, + "class_type": "CheckpointLoaderNF4", + "_meta": { + "title": "CheckpointLoaderNF4" + } + } +} \ No newline at end of file