From 1305fb294ca69d0a44d88c5bf7ce8c682abd0c8a Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 15 Jul 2024 17:36:24 -0400 Subject: [PATCH 01/15] Refactor: Move some code to the comfy/text_encoders folder. --- comfy/sd.py | 20 +++++++++---------- comfy/supported_models.py | 8 ++++---- comfy/text_encoders/aura_t5.py | 4 ++-- comfy/{ => text_encoders}/sa_t5.py | 4 ++-- comfy/{ => text_encoders}/sd3_clip.py | 4 ++-- comfy/{ => text_encoders}/t5.py | 0 comfy/{ => text_encoders}/t5_config_base.json | 0 comfy/{ => text_encoders}/t5_config_xxl.json | 0 .../t5_tokenizer/special_tokens_map.json | 0 .../t5_tokenizer/tokenizer.json | 0 .../t5_tokenizer/tokenizer_config.json | 0 11 files changed, 20 insertions(+), 20 deletions(-) rename comfy/{ => text_encoders}/sa_t5.py (88%) rename comfy/{ => text_encoders}/sd3_clip.py (98%) rename comfy/{ => text_encoders}/t5.py (100%) rename comfy/{ => text_encoders}/t5_config_base.json (100%) rename comfy/{ => text_encoders}/t5_config_xxl.json (100%) rename comfy/{ => text_encoders}/t5_tokenizer/special_tokens_map.json (100%) rename comfy/{ => text_encoders}/t5_tokenizer/tokenizer.json (100%) rename comfy/{ => text_encoders}/t5_tokenizer/tokenizer_config.json (100%) diff --git a/comfy/sd.py b/comfy/sd.py index b5bbcef61..17df5faff 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -19,8 +19,8 @@ from . import model_detection from . import sd1_clip from . import sd2_clip from . import sdxl_clip -from . import sd3_clip -from . import sa_t5 +import comfy.text_encoders.sd3_clip +import comfy.text_encoders.sa_t5 import comfy.text_encoders.aura_t5 import comfy.model_patcher @@ -414,27 +414,27 @@ def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DI weight = clip_data[0]["encoder.block.23.layer.1.DenseReluDense.wi_1.weight"] dtype_t5 = weight.dtype if weight.shape[-1] == 4096: - clip_target.clip = sd3_clip.sd3_clip(clip_l=False, clip_g=False, t5=True, dtype_t5=dtype_t5) - clip_target.tokenizer = sd3_clip.SD3Tokenizer + clip_target.clip = comfy.text_encoders.sd3_clip.sd3_clip(clip_l=False, clip_g=False, t5=True, dtype_t5=dtype_t5) + clip_target.tokenizer = comfy.text_encoders.sd3_clip.SD3Tokenizer elif weight.shape[-1] == 2048: clip_target.clip = comfy.text_encoders.aura_t5.AuraT5Model clip_target.tokenizer = comfy.text_encoders.aura_t5.AuraT5Tokenizer elif "encoder.block.0.layer.0.SelfAttention.k.weight" in clip_data[0]: - clip_target.clip = sa_t5.SAT5Model - clip_target.tokenizer = sa_t5.SAT5Tokenizer + clip_target.clip = comfy.text_encoders.sa_t5.SAT5Model + clip_target.tokenizer = comfy.text_encoders.sa_t5.SAT5Tokenizer else: clip_target.clip = sd1_clip.SD1ClipModel clip_target.tokenizer = sd1_clip.SD1Tokenizer elif len(clip_data) == 2: if clip_type == CLIPType.SD3: - clip_target.clip = sd3_clip.sd3_clip(clip_l=True, clip_g=True, t5=False) - clip_target.tokenizer = sd3_clip.SD3Tokenizer + clip_target.clip = comfy.text_encoders.sd3_clip.sd3_clip(clip_l=True, clip_g=True, t5=False) + clip_target.tokenizer = comfy.text_encoders.sd3_clip.SD3Tokenizer else: clip_target.clip = sdxl_clip.SDXLClipModel clip_target.tokenizer = sdxl_clip.SDXLTokenizer elif len(clip_data) == 3: - clip_target.clip = sd3_clip.SD3ClipModel - clip_target.tokenizer = sd3_clip.SD3Tokenizer + clip_target.clip = comfy.text_encoders.sd3_clip.SD3ClipModel + clip_target.tokenizer = comfy.text_encoders.sd3_clip.SD3Tokenizer clip = CLIP(clip_target, embedding_directory=embedding_directory) for c in clip_data: diff --git a/comfy/supported_models.py b/comfy/supported_models.py index a030f6229..b4d1059ef 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -5,8 +5,8 @@ from . import utils from . import sd1_clip from . import sd2_clip from . import sdxl_clip -from . import sd3_clip -from . import sa_t5 +import comfy.text_encoders.sd3_clip +import comfy.text_encoders.sa_t5 import comfy.text_encoders.aura_t5 from . import supported_models_base @@ -524,7 +524,7 @@ class SD3(supported_models_base.BASE): t5 = True dtype_t5 = state_dict[t5_key].dtype - return supported_models_base.ClipTarget(sd3_clip.SD3Tokenizer, sd3_clip.sd3_clip(clip_l=clip_l, clip_g=clip_g, t5=t5, dtype_t5=dtype_t5)) + return supported_models_base.ClipTarget(comfy.text_encoders.sd3_clip.SD3Tokenizer, comfy.text_encoders.sd3_clip.sd3_clip(clip_l=clip_l, clip_g=clip_g, t5=t5, dtype_t5=dtype_t5)) class StableAudio(supported_models_base.BASE): unet_config = { @@ -555,7 +555,7 @@ class StableAudio(supported_models_base.BASE): return utils.state_dict_prefix_replace(state_dict, replace_prefix) def clip_target(self, state_dict={}): - return supported_models_base.ClipTarget(sa_t5.SAT5Tokenizer, sa_t5.SAT5Model) + return supported_models_base.ClipTarget(comfy.text_encoders.sa_t5.SAT5Tokenizer, comfy.text_encoders.sa_t5.SAT5Model) class AuraFlow(supported_models_base.BASE): unet_config = { diff --git a/comfy/text_encoders/aura_t5.py b/comfy/text_encoders/aura_t5.py index 95f942ef5..6b9e4fe53 100644 --- a/comfy/text_encoders/aura_t5.py +++ b/comfy/text_encoders/aura_t5.py @@ -1,12 +1,12 @@ from comfy import sd1_clip from .llama_tokenizer import LLAMATokenizer -import comfy.t5 +import comfy.text_encoders.t5 import os class PT5XlModel(sd1_clip.SDClipModel): def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None): textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_config_xl.json") - super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 2, "pad": 1}, model_class=comfy.t5.T5, enable_attention_masks=True, zero_out_masked=True) + super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 2, "pad": 1}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True) class PT5XlTokenizer(sd1_clip.SDTokenizer): def __init__(self, embedding_directory=None): diff --git a/comfy/sa_t5.py b/comfy/text_encoders/sa_t5.py similarity index 88% rename from comfy/sa_t5.py rename to comfy/text_encoders/sa_t5.py index acc302f67..038558e7a 100644 --- a/comfy/sa_t5.py +++ b/comfy/text_encoders/sa_t5.py @@ -1,12 +1,12 @@ from comfy import sd1_clip from transformers import T5TokenizerFast -import comfy.t5 +import comfy.text_encoders.t5 import os class T5BaseModel(sd1_clip.SDClipModel): def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None): textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_config_base.json") - super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.t5.T5, enable_attention_masks=True, zero_out_masked=True) + super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True) class T5BaseTokenizer(sd1_clip.SDTokenizer): def __init__(self, embedding_directory=None): diff --git a/comfy/sd3_clip.py b/comfy/text_encoders/sd3_clip.py similarity index 98% rename from comfy/sd3_clip.py rename to comfy/text_encoders/sd3_clip.py index 0713eb285..70127e509 100644 --- a/comfy/sd3_clip.py +++ b/comfy/text_encoders/sd3_clip.py @@ -1,7 +1,7 @@ from comfy import sd1_clip from comfy import sdxl_clip from transformers import T5TokenizerFast -import comfy.t5 +import comfy.text_encoders.t5 import torch import os import comfy.model_management @@ -10,7 +10,7 @@ import logging class T5XXLModel(sd1_clip.SDClipModel): def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None): textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_config_xxl.json") - super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.t5.T5) + super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5) class T5XXLTokenizer(sd1_clip.SDTokenizer): def __init__(self, embedding_directory=None): diff --git a/comfy/t5.py b/comfy/text_encoders/t5.py similarity index 100% rename from comfy/t5.py rename to comfy/text_encoders/t5.py diff --git a/comfy/t5_config_base.json b/comfy/text_encoders/t5_config_base.json similarity index 100% rename from comfy/t5_config_base.json rename to comfy/text_encoders/t5_config_base.json diff --git a/comfy/t5_config_xxl.json b/comfy/text_encoders/t5_config_xxl.json similarity index 100% rename from comfy/t5_config_xxl.json rename to comfy/text_encoders/t5_config_xxl.json diff --git a/comfy/t5_tokenizer/special_tokens_map.json b/comfy/text_encoders/t5_tokenizer/special_tokens_map.json similarity index 100% rename from comfy/t5_tokenizer/special_tokens_map.json rename to comfy/text_encoders/t5_tokenizer/special_tokens_map.json diff --git a/comfy/t5_tokenizer/tokenizer.json b/comfy/text_encoders/t5_tokenizer/tokenizer.json similarity index 100% rename from comfy/t5_tokenizer/tokenizer.json rename to comfy/text_encoders/t5_tokenizer/tokenizer.json diff --git a/comfy/t5_tokenizer/tokenizer_config.json b/comfy/text_encoders/t5_tokenizer/tokenizer_config.json similarity index 100% rename from comfy/t5_tokenizer/tokenizer_config.json rename to comfy/text_encoders/t5_tokenizer/tokenizer_config.json From 136c93cb477e4c5c308791746583499805ab29aa Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 15 Jul 2024 20:01:49 -0400 Subject: [PATCH 02/15] Fix bug with workflow not registering change. There was an issue when only the class type of a node changed with all the inputs staying the same. --- execution.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/execution.py b/execution.py index 76225a962..8b3d116c6 100644 --- a/execution.py +++ b/execution.py @@ -247,6 +247,8 @@ def recursive_output_delete_if_changed(prompt, old_prompt, outputs, current_item to_delete = True elif unique_id not in old_prompt: to_delete = True + elif class_type != old_prompt[unique_id]['class_type']: + to_delete = True elif inputs == old_prompt[unique_id]['inputs']: for x in inputs: input_data = inputs[x] From 33346fd9b8856942187ad8c818498a080a2027b5 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 15 Jul 2024 20:36:03 -0400 Subject: [PATCH 03/15] Fix bug with custom nodes on other drives. --- nodes.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/nodes.py b/nodes.py index 5778f0609..998b316b7 100644 --- a/nodes.py +++ b/nodes.py @@ -1890,29 +1890,29 @@ NODE_DISPLAY_NAME_MAPPINGS = { EXTENSION_WEB_DIRS = {} -def get_relative_module_name(module_path: str) -> str: +def get_module_name(module_path: str) -> str: """ Returns the module name based on the given module path. Examples: - get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node.py") -> "custom_nodes.my_custom_node" - get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node") -> "custom_nodes.my_custom_node" - get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/") -> "custom_nodes.my_custom_node" - get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/__init__.py") -> "custom_nodes.my_custom_node" - get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/__init__") -> "custom_nodes.my_custom_node" - get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/__init__/") -> "custom_nodes.my_custom_node" - get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node.disabled") -> "custom_nodes.my + get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node.py") -> "my_custom_node" + get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node") -> "my_custom_node" + get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/") -> "my_custom_node" + get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/__init__.py") -> "my_custom_node" + get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/__init__") -> "my_custom_node" + get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node/__init__/") -> "my_custom_node" + get_module_name("C:/Users/username/ComfyUI/custom_nodes/my_custom_node.disabled") -> "custom_nodes Args: module_path (str): The path of the module. Returns: str: The module name. """ - relative_path = os.path.relpath(module_path, folder_paths.base_path) + base_path = os.path.basename(module_path) if os.path.isfile(module_path): - relative_path = os.path.splitext(relative_path)[0] - return relative_path.replace(os.sep, '.') + base_path = os.path.splitext(base_path)[0] + return base_path -def load_custom_node(module_path: str, ignore=set()) -> bool: +def load_custom_node(module_path: str, ignore=set(), module_parent="custom_nodes") -> bool: module_name = os.path.basename(module_path) if os.path.isfile(module_path): sp = os.path.splitext(module_path) @@ -1939,7 +1939,7 @@ def load_custom_node(module_path: str, ignore=set()) -> bool: for name, node_cls in module.NODE_CLASS_MAPPINGS.items(): if name not in ignore: NODE_CLASS_MAPPINGS[name] = node_cls - node_cls.RELATIVE_PYTHON_MODULE = get_relative_module_name(module_path) + node_cls.RELATIVE_PYTHON_MODULE = "{}.{}".format(module_parent, get_module_name(module_path)) if hasattr(module, "NODE_DISPLAY_NAME_MAPPINGS") and getattr(module, "NODE_DISPLAY_NAME_MAPPINGS") is not None: NODE_DISPLAY_NAME_MAPPINGS.update(module.NODE_DISPLAY_NAME_MAPPINGS) return True @@ -1974,7 +1974,7 @@ def init_external_custom_nodes(): if os.path.isfile(module_path) and os.path.splitext(module_path)[1] != ".py": continue if module_path.endswith(".disabled"): continue time_before = time.perf_counter() - success = load_custom_node(module_path, base_node_names) + success = load_custom_node(module_path, base_node_names, module_parent="custom_nodes") node_import_times.append((time.perf_counter() - time_before, module_path, success)) if len(node_import_times) > 0: @@ -2040,7 +2040,7 @@ def init_builtin_extra_nodes(): import_failed = [] for node_file in extras_files: - if not load_custom_node(os.path.join(extras_dir, node_file)): + if not load_custom_node(os.path.join(extras_dir, node_file), module_parent="comfy_extras"): import_failed.append(node_file) return import_failed From 99458e8aca3c48c7b2de3c404f66536991c37182 Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Tue, 16 Jul 2024 11:26:11 -0400 Subject: [PATCH 04/15] Add `FrontendManager` to manage non-default front-end impl (#3897) * Add frontend manager * Add tests * nit * Add unit test to github CI * Fix path * nit * ignore * Add logging * Install test deps * Remove 'stable' keyword support * Update test * Add web-root arg * Rename web-root to front-end-root * Add test on non-exist version number * Use repo owner/name to replace hard coded provider list * Inline cmd args * nit * Fix unit test --- .github/workflows/test-ui.yaml | 4 + .gitignore | 3 +- app/__init__.py | 0 app/frontend_management.py | 187 +++++++++++++++++++ comfy/cli_args.py | 35 ++++ pytest.ini | 7 +- server.py | 11 +- tests-unit/README.md | 8 + tests-unit/app_test/__init__.py | 0 tests-unit/app_test/frontend_manager_test.py | 100 ++++++++++ tests-unit/requirements.txt | 1 + 11 files changed, 350 insertions(+), 6 deletions(-) create mode 100644 app/__init__.py create mode 100644 app/frontend_management.py create mode 100644 tests-unit/README.md create mode 100644 tests-unit/app_test/__init__.py create mode 100644 tests-unit/app_test/frontend_manager_test.py create mode 100644 tests-unit/requirements.txt diff --git a/.github/workflows/test-ui.yaml b/.github/workflows/test-ui.yaml index 4b8b97934..d947e9d5f 100644 --- a/.github/workflows/test-ui.yaml +++ b/.github/workflows/test-ui.yaml @@ -24,3 +24,7 @@ jobs: npm run test:generate npm test -- --verbose working-directory: ./tests-ui + - name: Run Unit Tests + run: | + pip install -r tests-unit/requirements.txt + python -m pytest tests-unit diff --git a/.gitignore b/.gitignore index a9beebe73..5092c98f4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ venv/ !/web/extensions/core/ /tests-ui/data/object_info.json /user/ -*.log \ No newline at end of file +*.log +web_custom_versions/ \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/frontend_management.py b/app/frontend_management.py new file mode 100644 index 000000000..0855f4160 --- /dev/null +++ b/app/frontend_management.py @@ -0,0 +1,187 @@ +import argparse +import logging +import os +import re +import tempfile +import zipfile +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path +from typing import TypedDict + +import requests +from typing_extensions import NotRequired +from comfy.cli_args import DEFAULT_VERSION_STRING + + +REQUEST_TIMEOUT = 10 # seconds + + +class Asset(TypedDict): + url: str + + +class Release(TypedDict): + id: int + tag_name: str + name: str + prerelease: bool + created_at: str + published_at: str + body: str + assets: NotRequired[list[Asset]] + + +@dataclass +class FrontEndProvider: + owner: str + repo: str + + @property + def folder_name(self) -> str: + return f"{self.owner}_{self.repo}" + + @property + def release_url(self) -> str: + return f"https://api.github.com/repos/{self.owner}/{self.repo}/releases" + + @cached_property + def all_releases(self) -> list[Release]: + releases = [] + api_url = self.release_url + while api_url: + response = requests.get(api_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() # Raises an HTTPError if the response was an error + releases.extend(response.json()) + # GitHub uses the Link header to provide pagination links. Check if it exists and update api_url accordingly. + if "next" in response.links: + api_url = response.links["next"]["url"] + else: + api_url = None + return releases + + @cached_property + def latest_release(self) -> Release: + latest_release_url = f"{self.release_url}/latest" + response = requests.get(latest_release_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() # Raises an HTTPError if the response was an error + return response.json() + + def get_release(self, version: str) -> Release: + if version == "latest": + return self.latest_release + else: + for release in self.all_releases: + if release["tag_name"] in [version, f"v{version}"]: + return release + raise ValueError(f"Version {version} not found in releases") + + +def download_release_asset_zip(release: Release, destination_path: str) -> None: + """Download dist.zip from github release.""" + asset_url = None + for asset in release.get("assets", []): + if asset["name"] == "dist.zip": + asset_url = asset["url"] + break + + if not asset_url: + raise ValueError("dist.zip not found in the release assets") + + # Use a temporary file to download the zip content + with tempfile.TemporaryFile() as tmp_file: + headers = {"Accept": "application/octet-stream"} + response = requests.get( + asset_url, headers=headers, allow_redirects=True, timeout=REQUEST_TIMEOUT + ) + response.raise_for_status() # Ensure we got a successful response + + # Write the content to the temporary file + tmp_file.write(response.content) + + # Go back to the beginning of the temporary file + tmp_file.seek(0) + + # Extract the zip file content to the destination path + with zipfile.ZipFile(tmp_file, "r") as zip_ref: + zip_ref.extractall(destination_path) + + +class FrontendManager: + DEFAULT_FRONTEND_PATH = str(Path(__file__).parents[1] / "web") + CUSTOM_FRONTENDS_ROOT = str(Path(__file__).parents[1] / "web_custom_versions") + + @classmethod + def parse_version_string(cls, value: str) -> tuple[str, str, str]: + """ + Args: + value (str): The version string to parse. + + Returns: + tuple[str, str]: A tuple containing provider name and version. + + Raises: + argparse.ArgumentTypeError: If the version string is invalid. + """ + VERSION_PATTERN = r"^([a-zA-Z0-9][a-zA-Z0-9-]{0,38})/([a-zA-Z0-9_.-]+)@(\d+\.\d+\.\d+|latest)$" + match_result = re.match(VERSION_PATTERN, value) + if match_result is None: + raise argparse.ArgumentTypeError(f"Invalid version string: {value}") + + return match_result.group(1), match_result.group(2), match_result.group(3) + + @classmethod + def init_frontend_unsafe(cls, version_string: str) -> str: + """ + Initializes the frontend for the specified version. + + Args: + version_string (str): The version string. + + Returns: + str: The path to the initialized frontend. + + Raises: + Exception: If there is an error during the initialization process. + main error source might be request timeout or invalid URL. + """ + if version_string == DEFAULT_VERSION_STRING: + return cls.DEFAULT_FRONTEND_PATH + + repo_owner, repo_name, version = cls.parse_version_string(version_string) + provider = FrontEndProvider(repo_owner, repo_name) + release = provider.get_release(version) + + semantic_version = release["tag_name"].lstrip("v") + web_root = str( + Path(cls.CUSTOM_FRONTENDS_ROOT) / provider.folder_name / semantic_version + ) + if not os.path.exists(web_root): + os.makedirs(web_root, exist_ok=True) + logging.info( + "Downloading frontend(%s) version(%s) to (%s)", + provider.folder_name, + semantic_version, + web_root, + ) + logging.debug(release) + download_release_asset_zip(release, destination_path=web_root) + return web_root + + @classmethod + def init_frontend(cls, version_string: str) -> str: + """ + Initializes the frontend with the specified version string. + + Args: + version_string (str): The version string to initialize the frontend with. + + Returns: + str: The path of the initialized frontend. + """ + try: + return cls.init_frontend_unsafe(version_string) + except Exception as e: + logging.error("Failed to initialize frontend: %s", e) + logging.info("Falling back to the default frontend.") + return cls.DEFAULT_FRONTEND_PATH diff --git a/comfy/cli_args.py b/comfy/cli_args.py index b72bf3998..6251e3d28 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -1,7 +1,10 @@ import argparse import enum +import os +from typing import Optional import comfy.options + class EnumAction(argparse.Action): """ Argparse action for handling Enums @@ -124,6 +127,38 @@ parser.add_argument("--multi-user", action="store_true", help="Enables per-user parser.add_argument("--verbose", action="store_true", help="Enables more debug prints.") +# The default built-in provider hosted under web/ +DEFAULT_VERSION_STRING = "comfyanonymous/ComfyUI@latest" + +parser.add_argument( + "--front-end-version", + type=str, + default=DEFAULT_VERSION_STRING, + help=""" + Specifies the version of the frontend to be used. This command needs internet connectivity to query and + download available frontend implementations from GitHub releases. + + The version string should be in the format of: + [repoOwner]/[repoName]@[version] + where version is one of: "latest" or a valid version number (e.g. "1.0.0") + """, +) + +def is_valid_directory(path: Optional[str]) -> Optional[str]: + """Validate if the given path is a directory.""" + if path is None: + return None + + if not os.path.isdir(path): + raise argparse.ArgumentTypeError(f"{path} is not a valid directory.") + return path + +parser.add_argument( + "--front-end-root", + type=is_valid_directory, + default=None, + help="The local filesystem path to the directory where the frontend is located. Overrides --front-end-version.", +) if comfy.options.args_parsing: args = parser.parse_args() diff --git a/pytest.ini b/pytest.ini index b5a68e0f1..8b7a747e7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,8 @@ [pytest] markers = inference: mark as inference test (deselect with '-m "not inference"') -testpaths = tests -addopts = -s \ No newline at end of file +testpaths = + tests + tests-unit +addopts = -s +pythonpath = . diff --git a/server.py b/server.py index ce7c7532d..03c4c4219 100644 --- a/server.py +++ b/server.py @@ -25,9 +25,10 @@ import mimetypes from comfy.cli_args import args import comfy.utils import comfy.model_management - +from app.frontend_management import FrontendManager from app.user_manager import UserManager + class BinaryEventTypes: PREVIEW_IMAGE = 1 UNENCODED_PREVIEW_IMAGE = 2 @@ -83,8 +84,12 @@ class PromptServer(): max_upload_size = round(args.max_upload_size * 1024 * 1024) self.app = web.Application(client_max_size=max_upload_size, middlewares=middlewares) self.sockets = dict() - self.web_root = os.path.join(os.path.dirname( - os.path.realpath(__file__)), "web") + self.web_root = ( + FrontendManager.init_frontend(args.front_end_version) + if args.front_end_root is None + else args.front_end_root + ) + logging.info(f"[Prompt Server] web root: {self.web_root}") routes = web.RouteTableDef() self.routes = routes self.last_node_id = None diff --git a/tests-unit/README.md b/tests-unit/README.md new file mode 100644 index 000000000..94abd9853 --- /dev/null +++ b/tests-unit/README.md @@ -0,0 +1,8 @@ +# Pytest Unit Tests + +## Install test dependencies + +`pip install -r tests-units/requirements.txt` + +## Run tests +`pytest tests-units/` diff --git a/tests-unit/app_test/__init__.py b/tests-unit/app_test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests-unit/app_test/frontend_manager_test.py b/tests-unit/app_test/frontend_manager_test.py new file mode 100644 index 000000000..637869cfb --- /dev/null +++ b/tests-unit/app_test/frontend_manager_test.py @@ -0,0 +1,100 @@ +import argparse +import pytest +from requests.exceptions import HTTPError + +from app.frontend_management import ( + FrontendManager, + FrontEndProvider, + Release, +) +from comfy.cli_args import DEFAULT_VERSION_STRING + + +@pytest.fixture +def mock_releases(): + return [ + Release( + id=1, + tag_name="1.0.0", + name="Release 1.0.0", + prerelease=False, + created_at="2022-01-01T00:00:00Z", + published_at="2022-01-01T00:00:00Z", + body="Release notes for 1.0.0", + assets=[{"name": "dist.zip", "url": "https://example.com/dist.zip"}], + ), + Release( + id=2, + tag_name="2.0.0", + name="Release 2.0.0", + prerelease=False, + created_at="2022-02-01T00:00:00Z", + published_at="2022-02-01T00:00:00Z", + body="Release notes for 2.0.0", + assets=[{"name": "dist.zip", "url": "https://example.com/dist.zip"}], + ), + ] + + +@pytest.fixture +def mock_provider(mock_releases): + provider = FrontEndProvider( + owner="test-owner", + repo="test-repo", + ) + provider.all_releases = mock_releases + provider.latest_release = mock_releases[1] + FrontendManager.PROVIDERS = [provider] + return provider + + +def test_get_release(mock_provider, mock_releases): + version = "1.0.0" + release = mock_provider.get_release(version) + assert release == mock_releases[0] + + +def test_get_release_latest(mock_provider, mock_releases): + version = "latest" + release = mock_provider.get_release(version) + assert release == mock_releases[1] + + +def test_get_release_invalid_version(mock_provider): + version = "invalid" + with pytest.raises(ValueError): + mock_provider.get_release(version) + + +def test_init_frontend_default(): + version_string = DEFAULT_VERSION_STRING + frontend_path = FrontendManager.init_frontend(version_string) + assert frontend_path == FrontendManager.DEFAULT_FRONTEND_PATH + + +def test_init_frontend_invalid_version(): + version_string = "test-owner/test-repo@1.100.99" + with pytest.raises(HTTPError): + FrontendManager.init_frontend_unsafe(version_string) + + +def test_init_frontend_invalid_provider(): + version_string = "invalid/invalid@latest" + with pytest.raises(HTTPError): + FrontendManager.init_frontend_unsafe(version_string) + + +def test_parse_version_string(): + version_string = "owner/repo@1.0.0" + repo_owner, repo_name, version = FrontendManager.parse_version_string( + version_string + ) + assert repo_owner == "owner" + assert repo_name == "repo" + assert version == "1.0.0" + + +def test_parse_version_string_invalid(): + version_string = "invalid" + with pytest.raises(argparse.ArgumentTypeError): + FrontendManager.parse_version_string(version_string) diff --git a/tests-unit/requirements.txt b/tests-unit/requirements.txt new file mode 100644 index 000000000..0587502f8 --- /dev/null +++ b/tests-unit/requirements.txt @@ -0,0 +1 @@ +pytest>=7.8.0 From e1630391d6d190c3dd9ef4609f0c84af72be54af Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 16 Jul 2024 11:29:38 -0400 Subject: [PATCH 05/15] Allow version names like v0.0.1 for the FrontendManager. --- app/frontend_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/frontend_management.py b/app/frontend_management.py index 0855f4160..5452f6e11 100644 --- a/app/frontend_management.py +++ b/app/frontend_management.py @@ -123,7 +123,7 @@ class FrontendManager: Raises: argparse.ArgumentTypeError: If the version string is invalid. """ - VERSION_PATTERN = r"^([a-zA-Z0-9][a-zA-Z0-9-]{0,38})/([a-zA-Z0-9_.-]+)@(\d+\.\d+\.\d+|latest)$" + VERSION_PATTERN = r"^([a-zA-Z0-9][a-zA-Z0-9-]{0,38})/([a-zA-Z0-9_.-]+)@(v?\d+\.\d+\.\d+|latest)$" match_result = re.match(VERSION_PATTERN, value) if match_result is None: raise argparse.ArgumentTypeError(f"Invalid version string: {value}") From 821f93872ecf3f1769b8abebd640a804839f4ab9 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 16 Jul 2024 15:18:24 -0400 Subject: [PATCH 06/15] Allow model sampling to set number of timesteps. --- comfy/model_sampling.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/comfy/model_sampling.py b/comfy/model_sampling.py index 2d95a83dc..25bb7e043 100644 --- a/comfy/model_sampling.py +++ b/comfy/model_sampling.py @@ -59,8 +59,9 @@ class ModelSamplingDiscrete(torch.nn.Module): beta_schedule = sampling_settings.get("beta_schedule", "linear") linear_start = sampling_settings.get("linear_start", 0.00085) linear_end = sampling_settings.get("linear_end", 0.012) + timesteps = sampling_settings.get("timesteps", 1000) - self._register_schedule(given_betas=None, beta_schedule=beta_schedule, timesteps=1000, linear_start=linear_start, linear_end=linear_end, cosine_s=8e-3) + self._register_schedule(given_betas=None, beta_schedule=beta_schedule, timesteps=timesteps, linear_start=linear_start, linear_end=linear_end, cosine_s=8e-3) self.sigma_data = 1.0 def _register_schedule(self, given_betas=None, beta_schedule="linear", timesteps=1000, From 8270c625301034767c77b7d7b1e5a135172d1ebb Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 16 Jul 2024 17:01:40 -0400 Subject: [PATCH 07/15] Add SetUnionControlNetType to set the type of the union controlnet model. --- comfy/controlnet.py | 8 ++++++- comfy_extras/nodes_controlnet.py | 37 ++++++++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 comfy_extras/nodes_controlnet.py diff --git a/comfy/controlnet.py b/comfy/controlnet.py index b8e27c71f..12e5f16c8 100644 --- a/comfy/controlnet.py +++ b/comfy/controlnet.py @@ -45,6 +45,7 @@ class ControlBase: self.timestep_range = None self.compression_ratio = 8 self.upscale_algorithm = 'nearest-exact' + self.extra_args = {} if device is None: device = comfy.model_management.get_torch_device() @@ -90,6 +91,7 @@ class ControlBase: c.compression_ratio = self.compression_ratio c.upscale_algorithm = self.upscale_algorithm c.latent_format = self.latent_format + c.extra_args = self.extra_args.copy() c.vae = self.vae def inference_memory_requirements(self, dtype): @@ -135,6 +137,10 @@ class ControlBase: o[i] = prev_val + o[i] #TODO: change back to inplace add if shared tensors stop being an issue return out + def set_extra_arg(self, argument, value=None): + self.extra_args[argument] = value + + class ControlNet(ControlBase): def __init__(self, control_model=None, global_average_pooling=False, compression_ratio=8, latent_format=None, device=None, load_device=None, manual_cast_dtype=None): super().__init__(device) @@ -191,7 +197,7 @@ class ControlNet(ControlBase): timestep = self.model_sampling_current.timestep(t) x_noisy = self.model_sampling_current.calculate_input(t, x_noisy) - control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y) + control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y, **self.extra_args) return self.control_merge(control, control_prev, output_dtype) def copy(self): diff --git a/comfy_extras/nodes_controlnet.py b/comfy_extras/nodes_controlnet.py new file mode 100644 index 000000000..e550436b0 --- /dev/null +++ b/comfy_extras/nodes_controlnet.py @@ -0,0 +1,37 @@ + +UNION_CONTROLNET_TYPES = {"auto": -1, + "openpose": 0, + "depth": 1, + "hed/pidi/scribble/ted": 2, + "canny/lineart/anime_lineart/mlsd": 3, + "normal": 4, + "segment": 5, + "tile": 6, + "repaint": 7, + } + +class SetUnionControlNetType: + @classmethod + def INPUT_TYPES(s): + return {"required": {"control_net": ("CONTROL_NET", ), + "type": (list(UNION_CONTROLNET_TYPES.keys()),) + }} + + CATEGORY = "conditioning" + RETURN_TYPES = ("CONTROL_NET",) + + FUNCTION = "set_controlnet_type" + + def set_controlnet_type(self, control_net, type): + control_net = control_net.copy() + type_number = UNION_CONTROLNET_TYPES[type] + if type_number >= 0: + control_net.set_extra_arg("control_type", [type_number]) + else: + control_net.set_extra_arg("control_type", []) + + return (control_net,) + +NODE_CLASS_MAPPINGS = { + "SetUnionControlNetType": SetUnionControlNetType, +} diff --git a/nodes.py b/nodes.py index 998b316b7..89a2f21d2 100644 --- a/nodes.py +++ b/nodes.py @@ -2036,6 +2036,7 @@ def init_builtin_extra_nodes(): "nodes_audio.py", "nodes_sd3.py", "nodes_gits.py", + "nodes_controlnet.py", ] import_failed = [] From 60383f3b64aa4477edfea325879bb04d3695161c Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 16 Jul 2024 17:08:25 -0400 Subject: [PATCH 08/15] Move controlnet nodes to conditioning/controlnet. --- comfy_extras/nodes_controlnet.py | 2 +- comfy_extras/nodes_sd3.py | 2 +- nodes.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/comfy_extras/nodes_controlnet.py b/comfy_extras/nodes_controlnet.py index e550436b0..ef7cfc6ab 100644 --- a/comfy_extras/nodes_controlnet.py +++ b/comfy_extras/nodes_controlnet.py @@ -17,7 +17,7 @@ class SetUnionControlNetType: "type": (list(UNION_CONTROLNET_TYPES.keys()),) }} - CATEGORY = "conditioning" + CATEGORY = "conditioning/controlnet" RETURN_TYPES = ("CONTROL_NET",) FUNCTION = "set_controlnet_type" diff --git a/comfy_extras/nodes_sd3.py b/comfy_extras/nodes_sd3.py index 548b1ad6a..0aafa2426 100644 --- a/comfy_extras/nodes_sd3.py +++ b/comfy_extras/nodes_sd3.py @@ -92,7 +92,7 @@ class ControlNetApplySD3(nodes.ControlNetApplyAdvanced): "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}) }} - CATEGORY = "_for_testing/sd3" + CATEGORY = "conditioning/controlnet" NODE_CLASS_MAPPINGS = { "TripleCLIPLoader": TripleCLIPLoader, diff --git a/nodes.py b/nodes.py index 89a2f21d2..fbdcb6c91 100644 --- a/nodes.py +++ b/nodes.py @@ -748,7 +748,7 @@ class ControlNetApply: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "apply_controlnet" - CATEGORY = "conditioning" + CATEGORY = "conditioning/controlnet" def apply_controlnet(self, conditioning, control_net, image, strength): if strength == 0: @@ -783,7 +783,7 @@ class ControlNetApplyAdvanced: RETURN_NAMES = ("positive", "negative") FUNCTION = "apply_controlnet" - CATEGORY = "conditioning" + CATEGORY = "conditioning/controlnet" def apply_controlnet(self, positive, negative, control_net, image, strength, start_percent, end_percent, vae=None): if strength == 0: From f2298799ba58e915a1f22fec336d455c0913f409 Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Tue, 16 Jul 2024 18:20:39 -0400 Subject: [PATCH 09/15] Fix annotation (#4035) --- app/frontend_management.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/frontend_management.py b/app/frontend_management.py index 5452f6e11..fb57b23f3 100644 --- a/app/frontend_management.py +++ b/app/frontend_management.py @@ -1,3 +1,4 @@ +from __future__ import annotations import argparse import logging import os From c5a48b15bddab089f0054af0ed72eea868263ef6 Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Tue, 16 Jul 2024 18:27:09 -0400 Subject: [PATCH 10/15] Make default hash lib configurable without code changes via CLI argument (#3947) * cli_args: Add --duplicate-check-hash-function. * server.py: compare_image_hash configurable hash function Uses an argument added in cli_args to specify the type of hashing to default to for duplicate hash checking. Uses an `eval()` to identify the specific hashlib class to utilize, but ultimately safely operates because we have specific options and only those options/choices in the arg parser. So we don't have any unsafe input there. * Add hasher() to node_helpers * hashlib selection moved to node_helpers * default-hashing-function instead of dupe checking hasher This makes a default-hashing-function option instead of previous selected option. * Use args.default_hashing_function * Use safer handling for node_helpers.hasher() Uses a safer handling method than `eval` to evaluate default hashing function. * Stray parentheses are evil. * Indentation fix. Somehow when I hit save I didn't notice I missed a space to make indentation work proper. Oops! --- comfy/cli_args.py | 1 + node_helpers.py | 13 +++++++++++++ server.py | 7 +++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 6251e3d28..2397de3d6 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -112,6 +112,7 @@ vram_group.add_argument("--lowvram", action="store_true", help="Split the unet i vram_group.add_argument("--novram", action="store_true", help="When lowvram isn't enough.") vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") +parser.add_argument("--default-hashing-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.") parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.") parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.") diff --git a/node_helpers.py b/node_helpers.py index 43b9e829f..fee628790 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -1,3 +1,7 @@ +import hashlib + +from comfy.cli_args import args + from PIL import ImageFile, UnidentifiedImageError def conditioning_set_values(conditioning, values={}): @@ -22,3 +26,12 @@ def pillow(fn, arg): if prev_value is not None: ImageFile.LOAD_TRUNCATED_IMAGES = prev_value return x + +def hasher(): + hashfuncs = { + "md5": hashlib.md5, + "sha1": hashlib.sha1, + "sha256": hashlib.sha256, + "sha512": hashlib.sha512 + } + return hashfuncs[args.default_hashing_function] diff --git a/server.py b/server.py index 03c4c4219..23ca2fd33 100644 --- a/server.py +++ b/server.py @@ -25,6 +25,7 @@ import mimetypes from comfy.cli_args import args import comfy.utils import comfy.model_management +import node_helpers from app.frontend_management import FrontendManager from app.user_manager import UserManager @@ -161,10 +162,12 @@ class PromptServer(): return type_dir, dir_type def compare_image_hash(filepath, image): + hasher = node_helpers.hasher() + # function to compare hashes of two images to see if it already exists, fix to #3465 if os.path.exists(filepath): - a = hashlib.sha256() - b = hashlib.sha256() + a = hasher() + b = hasher() with open(filepath, "rb") as f: a.update(f.read()) b.update(image.file.read()) From 1cde6b2eff5721175725e9cc5f80191d8276e1e8 Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Tue, 16 Jul 2024 21:15:08 -0400 Subject: [PATCH 11/15] Disallow use of eval with pylint (#4033) --- .github/workflows/pylint.yml | 23 +++++++++++++++++++++++ .pylintrc | 3 +++ 2 files changed, 26 insertions(+) create mode 100644 .github/workflows/pylint.yml create mode 100644 .pylintrc diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 000000000..5effbea35 --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,23 @@ +name: Python Linting + +on: [push, pull_request] + +jobs: + pylint: + name: Run Pylint + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install Pylint + run: pip install pylint + + - name: Run Pylint + run: pylint --rcfile=.pylintrc $(find . -type f -name "*.py") diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..a5da56e57 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,3 @@ +[MESSAGES CONTROL] +disable=all +enable=eval-used From 281ad42df4fae74ed11c1e7ff14cff6b7db6498c Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 17 Jul 2024 10:16:31 -0400 Subject: [PATCH 12/15] Fix lowvram union controlnet bug. --- comfy/cldm/cldm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py index 4a58c823f..1d7294bd6 100644 --- a/comfy/cldm/cldm.py +++ b/comfy/cldm/cldm.py @@ -361,7 +361,7 @@ class ControlNet(nn.Module): controlnet_cond = self.input_hint_block(hint[idx], emb, context) feat_seq = torch.mean(controlnet_cond, dim=(2, 3)) if idx < len(control_type): - feat_seq += self.task_embedding[control_type[idx]] + feat_seq += self.task_embedding[control_type[idx]].to(dtype=feat_seq.dtype, device=feat_seq.device) inputs.append(feat_seq.unsqueeze(1)) condition_list.append(controlnet_cond) From 6f7869f365ca4ec34b017f9002b9dcb9bce56b1a Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 17 Jul 2024 13:05:38 -0400 Subject: [PATCH 13/15] Get clip vision image size from config. --- comfy/clip_vision.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/comfy/clip_vision.py b/comfy/clip_vision.py index acc86be85..ac39c6227 100644 --- a/comfy/clip_vision.py +++ b/comfy/clip_vision.py @@ -34,6 +34,7 @@ class ClipVisionModel(): with open(json_config) as f: config = json.load(f) + self.image_size = config.get("image_size", 224) self.load_device = comfy.model_management.text_encoder_device() offload_device = comfy.model_management.text_encoder_offload_device() self.dtype = comfy.model_management.text_encoder_dtype(self.load_device) @@ -50,7 +51,7 @@ class ClipVisionModel(): def encode_image(self, image): comfy.model_management.load_model_gpu(self.patcher) - pixel_values = clip_preprocess(image.to(self.load_device)).float() + pixel_values = clip_preprocess(image.to(self.load_device), size=self.image_size).float() out = self.model(pixel_values=pixel_values, intermediate_output=-2) outputs = Output() From 855789403b346c7d7772e2c7ed0a6a327f56047f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=96=B5=E5=93=A9=E4=B8=AA=E5=92=AA?= Date: Thu, 18 Jul 2024 01:12:50 +0800 Subject: [PATCH 14/15] support clip-vit-large-patch14-336 (#4042) * support clip-vit-large-patch14-336 * support clip-vit-large-patch14-336 --- comfy/clip_vision.py | 5 ++++- comfy/clip_vision_config_vitl_336.json | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 comfy/clip_vision_config_vitl_336.json diff --git a/comfy/clip_vision.py b/comfy/clip_vision.py index ac39c6227..20dc3345d 100644 --- a/comfy/clip_vision.py +++ b/comfy/clip_vision.py @@ -94,7 +94,10 @@ def load_clipvision_from_sd(sd, prefix="", convert_keys=False): elif "vision_model.encoder.layers.30.layer_norm1.weight" in sd: json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_h.json") elif "vision_model.encoder.layers.22.layer_norm1.weight" in sd: - json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl.json") + if sd["vision_model.embeddings.position_embedding.weight"].shape[0] == 577: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl_336.json") + else: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl.json") else: return None diff --git a/comfy/clip_vision_config_vitl_336.json b/comfy/clip_vision_config_vitl_336.json new file mode 100644 index 000000000..f26945273 --- /dev/null +++ b/comfy/clip_vision_config_vitl_336.json @@ -0,0 +1,18 @@ +{ + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "quick_gelu", + "hidden_size": 1024, + "image_size": 336, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-5, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 24, + "patch_size": 14, + "projection_dim": 768, + "torch_dtype": "float32" +} From 374e093e09c94b528d7c9dfc337c65cc5c433ee3 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 17 Jul 2024 13:11:10 -0700 Subject: [PATCH 15/15] Disable audio widget trying to get previews (#4044) --- web/extensions/core/uploadAudio.js | 1 - 1 file changed, 1 deletion(-) diff --git a/web/extensions/core/uploadAudio.js b/web/extensions/core/uploadAudio.js index 0ac9cb807..6cc3863a1 100644 --- a/web/extensions/core/uploadAudio.js +++ b/web/extensions/core/uploadAudio.js @@ -17,7 +17,6 @@ function getResourceURL(subfolder, filename, type = "input") { "filename=" + encodeURIComponent(filename), "type=" + type, "subfolder=" + subfolder, - app.getPreviewFormatParam().substring(1), app.getRandParam().substring(1) ].join("&")