From de8e8e3b0d08cc27b3981c97ba45ed3444601d6f Mon Sep 17 00:00:00 2001 From: Simon Lui <502929+simonlui@users.noreply.github.com> Date: Thu, 19 Sep 2024 18:11:42 +0900 Subject: [PATCH 1/4] Fix xpu Pytorch nightly build from calling optimize which doesn't exist. (#4978) --- comfy/model_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 77c05510e..22a584a2e 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -326,7 +326,7 @@ class LoadedModel: self.model_unload() raise e - if is_intel_xpu() and not args.disable_ipex_optimize and self.real_model is not None: + if is_intel_xpu() and not args.disable_ipex_optimize and 'ipex' in globals() and self.real_model is not None: with torch.no_grad(): self.real_model = ipex.optimize(self.real_model.eval(), inplace=True, graph_mode=True, concat_linear=True) From ad66f7c7d8c3dc2985f2fba4e40b503cb45be03a Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 19 Sep 2024 05:01:00 -0400 Subject: [PATCH 2/4] Add model_options to load_controlnet function. --- comfy/controlnet.py | 56 +++++++++++++++++++++++++-------------------- comfy/sd.py | 2 +- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/comfy/controlnet.py b/comfy/controlnet.py index 1ea00eccf..4bd075e9d 100644 --- a/comfy/controlnet.py +++ b/comfy/controlnet.py @@ -335,7 +335,7 @@ class ControlLoraOps: class ControlLora(ControlNet): - def __init__(self, control_weights, global_average_pooling=False, device=None): + def __init__(self, control_weights, global_average_pooling=False, device=None, model_options={}): #TODO? model_options ControlBase.__init__(self, device) self.control_weights = control_weights self.global_average_pooling = global_average_pooling @@ -392,19 +392,22 @@ class ControlLora(ControlNet): def inference_memory_requirements(self, dtype): return comfy.utils.calculate_parameters(self.control_weights) * comfy.model_management.dtype_size(dtype) + ControlBase.inference_memory_requirements(self, dtype) -def controlnet_config(sd): +def controlnet_config(sd, model_options={}): model_config = comfy.model_detection.model_config_from_unet(sd, "", True) supported_inference_dtypes = model_config.supported_inference_dtypes controlnet_config = model_config.unet_config - unet_dtype = comfy.model_management.unet_dtype(supported_dtypes=supported_inference_dtypes) + unet_dtype = model_options.get("dtype", comfy.model_management.unet_dtype(supported_dtypes=supported_inference_dtypes)) load_device = comfy.model_management.get_torch_device() manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device) - if manual_cast_dtype is not None: - operations = comfy.ops.manual_cast - else: - operations = comfy.ops.disable_weight_init + + operations = model_options.get("custom_operations", None) + if operations is None: + if manual_cast_dtype is not None: + operations = comfy.ops.manual_cast + else: + operations = comfy.ops.disable_weight_init offload_device = comfy.model_management.unet_offload_device() return model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device @@ -419,9 +422,9 @@ def controlnet_load_state_dict(control_model, sd): logging.debug("unexpected controlnet keys: {}".format(unexpected)) return control_model -def load_controlnet_mmdit(sd): +def load_controlnet_mmdit(sd, model_options={}): new_sd = comfy.model_detection.convert_diffusers_mmdit(sd, "") - model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(new_sd) + model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(new_sd, model_options=model_options) num_blocks = comfy.model_detection.count_blocks(new_sd, 'joint_blocks.{}.') for k in sd: new_sd[k] = sd[k] @@ -440,8 +443,8 @@ def load_controlnet_mmdit(sd): return control -def load_controlnet_hunyuandit(controlnet_data): - model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(controlnet_data) +def load_controlnet_hunyuandit(controlnet_data, model_options={}): + model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(controlnet_data, model_options=model_options) control_model = comfy.ldm.hydit.controlnet.HunYuanControlNet(operations=operations, device=offload_device, dtype=unet_dtype) control_model = controlnet_load_state_dict(control_model, controlnet_data) @@ -451,17 +454,17 @@ def load_controlnet_hunyuandit(controlnet_data): control = ControlNet(control_model, compression_ratio=1, latent_format=latent_format, load_device=load_device, manual_cast_dtype=manual_cast_dtype, extra_conds=extra_conds, strength_type=StrengthType.CONSTANT) return control -def load_controlnet_flux_xlabs_mistoline(sd, mistoline=False): - model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(sd) +def load_controlnet_flux_xlabs_mistoline(sd, mistoline=False, model_options={}): + model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(sd, model_options=model_options) control_model = comfy.ldm.flux.controlnet.ControlNetFlux(mistoline=mistoline, operations=operations, device=offload_device, dtype=unet_dtype, **model_config.unet_config) control_model = controlnet_load_state_dict(control_model, sd) extra_conds = ['y', 'guidance'] control = ControlNet(control_model, load_device=load_device, manual_cast_dtype=manual_cast_dtype, extra_conds=extra_conds) return control -def load_controlnet_flux_instantx(sd): +def load_controlnet_flux_instantx(sd, model_options={}): new_sd = comfy.model_detection.convert_diffusers_mmdit(sd, "") - model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(new_sd) + model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(new_sd, model_options=model_options) for k in sd: new_sd[k] = sd[k] @@ -487,13 +490,13 @@ def convert_mistoline(sd): return comfy.utils.state_dict_prefix_replace(sd, {"single_controlnet_blocks.": "controlnet_single_blocks."}) -def load_controlnet(ckpt_path, model=None): +def load_controlnet(ckpt_path, model=None, model_options={}): controlnet_data = comfy.utils.load_torch_file(ckpt_path, safe_load=True) if 'after_proj_list.18.bias' in controlnet_data.keys(): #Hunyuan DiT - return load_controlnet_hunyuandit(controlnet_data) + return load_controlnet_hunyuandit(controlnet_data, model_options=model_options) if "lora_controlnet" in controlnet_data: - return ControlLora(controlnet_data) + return ControlLora(controlnet_data, model_options=model_options) controlnet_config = None supported_inference_dtypes = None @@ -550,13 +553,13 @@ def load_controlnet(ckpt_path, model=None): controlnet_data = new_sd elif "controlnet_blocks.0.weight" in controlnet_data: if "double_blocks.0.img_attn.norm.key_norm.scale" in controlnet_data: - return load_controlnet_flux_xlabs_mistoline(controlnet_data) + return load_controlnet_flux_xlabs_mistoline(controlnet_data, model_options=model_options) elif "pos_embed_input.proj.weight" in controlnet_data: - return load_controlnet_mmdit(controlnet_data) #SD3 diffusers controlnet + return load_controlnet_mmdit(controlnet_data, model_options=model_options) #SD3 diffusers controlnet elif "controlnet_x_embedder.weight" in controlnet_data: - return load_controlnet_flux_instantx(controlnet_data) + return load_controlnet_flux_instantx(controlnet_data, model_options=model_options) elif "controlnet_blocks.0.linear.weight" in controlnet_data: #mistoline flux - return load_controlnet_flux_xlabs_mistoline(convert_mistoline(controlnet_data), mistoline=True) + return load_controlnet_flux_xlabs_mistoline(convert_mistoline(controlnet_data), mistoline=True, model_options=model_options) pth_key = 'control_model.zero_convs.0.0.weight' pth = False @@ -568,7 +571,7 @@ def load_controlnet(ckpt_path, model=None): elif key in controlnet_data: prefix = "" else: - net = load_t2i_adapter(controlnet_data) + net = load_t2i_adapter(controlnet_data, model_options=model_options) if net is None: logging.error("error checkpoint does not contain controlnet or t2i adapter data {}".format(ckpt_path)) return net @@ -587,7 +590,10 @@ def load_controlnet(ckpt_path, model=None): manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device) if manual_cast_dtype is not None: controlnet_config["operations"] = comfy.ops.manual_cast - controlnet_config["dtype"] = unet_dtype + if "custom_operations" in model_options: + controlnet_config["operations"] = model_options["custom_operations"] + if "dtype" in model_options: + controlnet_config["dtype"] = model_options["dtype"] controlnet_config["device"] = comfy.model_management.unet_offload_device() controlnet_config.pop("out_channels") controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1] @@ -685,7 +691,7 @@ class T2IAdapter(ControlBase): self.copy_to(c) return c -def load_t2i_adapter(t2i_data): +def load_t2i_adapter(t2i_data, model_options={}): #TODO: model_options compression_ratio = 8 upscale_algorithm = 'nearest-exact' diff --git a/comfy/sd.py b/comfy/sd.py index 8c5b058ce..99859d241 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -645,7 +645,7 @@ def load_diffusion_model_state_dict(sd, model_options={}): #load unet in diffuse manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes) model_config.set_inference_dtype(unet_dtype, manual_cast_dtype) - model_config.custom_operations = model_options.get("custom_operations", None) + model_config.custom_operations = model_options.get("custom_operations", model_config.custom_operations) model = model_config.get_model(new_sd, "") model = model.to(offload_device) model.load_model_weights(new_sd, "") From 68bb885d228f90d2d1db192a4afa50f6a60a6e23 Mon Sep 17 00:00:00 2001 From: "Alex \"mcmonkey\" Goodwin" <4000772+mcmonkey4eva@users.noreply.github.com> Date: Thu, 19 Sep 2024 21:59:55 +0900 Subject: [PATCH 3/4] add 'is_default' to model paths config (#4979) * add 'is_default' to model paths config including impl and doc in example file * update weirdly overspecific test expectations * oh there's two * sigh --- extra_model_paths.yaml.example | 2 ++ folder_paths.py | 7 +++++-- tests-unit/utils/extra_config_test.py | 4 ++-- utils/extra_config.py | 5 ++++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/extra_model_paths.yaml.example b/extra_model_paths.yaml.example index d4fd55c87..b55913a5a 100644 --- a/extra_model_paths.yaml.example +++ b/extra_model_paths.yaml.example @@ -25,6 +25,8 @@ a111: #comfyui: # base_path: path/to/comfyui/ +# # You can use is_default to mark that these folders should be listed first, and used as the default dirs for eg downloads +# #is_default: true # checkpoints: models/checkpoints/ # clip: models/clip/ # clip_vision: models/clip_vision/ diff --git a/folder_paths.py b/folder_paths.py index 0ad8bc184..1f03c08d8 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -195,11 +195,14 @@ def exists_annotated_filepath(name) -> bool: return os.path.exists(filepath) -def add_model_folder_path(folder_name: str, full_folder_path: str) -> None: +def add_model_folder_path(folder_name: str, full_folder_path: str, is_default: bool = False) -> None: global folder_names_and_paths folder_name = map_legacy(folder_name) if folder_name in folder_names_and_paths: - folder_names_and_paths[folder_name][0].append(full_folder_path) + if is_default: + folder_names_and_paths[folder_name][0].insert(0, full_folder_path) + else: + folder_names_and_paths[folder_name][0].append(full_folder_path) else: folder_names_and_paths[folder_name] = ([full_folder_path], set()) diff --git a/tests-unit/utils/extra_config_test.py b/tests-unit/utils/extra_config_test.py index 06349560d..0effd89e8 100644 --- a/tests-unit/utils/extra_config_test.py +++ b/tests-unit/utils/extra_config_test.py @@ -71,7 +71,7 @@ def test_load_extra_model_paths_expands_userpath( load_extra_path_config(dummy_yaml_file_name) expected_calls = [ - ('checkpoints', os.path.join(mock_expanded_home, 'App', 'subfolder1')), + ('checkpoints', os.path.join(mock_expanded_home, 'App', 'subfolder1'), False), ] assert mock_add_model_folder_path.call_count == len(expected_calls) @@ -111,7 +111,7 @@ def test_load_extra_model_paths_expands_appdata( expected_base_path = 'C:/Users/TestUser/AppData/Roaming/ComfyUI' expected_calls = [ - ('checkpoints', os.path.join(expected_base_path, 'models/checkpoints')), + ('checkpoints', os.path.join(expected_base_path, 'models/checkpoints'), False), ] assert mock_add_model_folder_path.call_count == len(expected_calls) diff --git a/utils/extra_config.py b/utils/extra_config.py index 7ac3650ac..908765902 100644 --- a/utils/extra_config.py +++ b/utils/extra_config.py @@ -14,6 +14,9 @@ def load_extra_path_config(yaml_path): if "base_path" in conf: base_path = conf.pop("base_path") base_path = os.path.expandvars(os.path.expanduser(base_path)) + is_default = False + if "is_default" in conf: + is_default = conf.pop("is_default") for x in conf: for y in conf[x].split("\n"): if len(y) == 0: @@ -22,4 +25,4 @@ def load_extra_path_config(yaml_path): if base_path is not None: full_path = os.path.join(base_path, full_path) logging.info("Adding extra search path {} {}".format(x, full_path)) - folder_paths.add_model_folder_path(x, full_path) + folder_paths.add_model_folder_path(x, full_path, is_default) From 3326bdfd4e9135fc4b5324ddec6561b8c008be0c Mon Sep 17 00:00:00 2001 From: "Alex \"mcmonkey\" Goodwin" <4000772+mcmonkey4eva@users.noreply.github.com> Date: Thu, 19 Sep 2024 22:52:55 +0900 Subject: [PATCH 4/4] add internal /folder_paths route (#4980) returns a json maps of folder paths --- api_server/routes/internal/internal_routes.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api_server/routes/internal/internal_routes.py b/api_server/routes/internal/internal_routes.py index 8c46215f0..63704f13a 100644 --- a/api_server/routes/internal/internal_routes.py +++ b/api_server/routes/internal/internal_routes.py @@ -1,6 +1,6 @@ from aiohttp import web from typing import Optional -from folder_paths import models_dir, user_directory, output_directory +from folder_paths import models_dir, user_directory, output_directory, folder_names_and_paths from api_server.services.file_service import FileService import app.logger @@ -36,6 +36,13 @@ class InternalRoutes: async def get_logs(request): return web.json_response(app.logger.get_logs()) + @self.routes.get('/folder_paths') + async def get_folder_paths(request): + response = {} + for key in folder_names_and_paths: + response[key] = folder_names_and_paths[key][0] + return web.json_response(response) + def get_app(self): if self._app is None: self._app = web.Application()