From e8c782c8e1f7200ccc921856f9bd513f5c28432b Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Wed, 5 Mar 2025 22:27:24 +0900 Subject: [PATCH] feat: pip_auto_fix.list for custom PIPFixer fixed: always reinstall comfyui-frontend-package https://github.com/ltdrdata/ComfyUI-Manager/discussions/980#discussioncomment-12400709 --- README.md | 5 +++ cm-cli.py | 17 ++++---- glob/manager_core.py | 6 +-- glob/manager_util.py | 101 +++++++++++++++++++++++++++++++++++++++++-- prestartup_script.py | 2 +- pyproject.toml | 2 +- 6 files changed, 115 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d288c87b..c759847a 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ In `ComfyUI-Manager` V3.0 and later, configuration files and dynamically generat * Configurable channel lists: `/default/ComfyUI-Manager/channels.ini` * Configurable pip overrides: `/default/ComfyUI-Manager/pip_overrides.json` * Configurable pip blacklist: `/default/ComfyUI-Manager/pip_blacklist.list` +* Configurable pip auto fix: `/default/ComfyUI-Manager/pip_auto_fix.list` * Saved snapshot files: `/default/ComfyUI-Manager/snapshots` * Startup script files: `/default/ComfyUI-Manager/startup-scripts` * Component files: `/default/ComfyUI-Manager/components` @@ -306,6 +307,10 @@ The following settings are applied based on the section marked as `is_default`. * Prevent the installation of specific pip packages * List the package names one per line in the `pip_blacklist.list` file. +* Automatically Restoring pip Installation + * If you list pip spec requirements in `pip_auto_fix.list`, similar to `requirements.txt`, it will automatically restore the specified versions when starting ComfyUI or when versions get mismatched during various custom node installations. + * `--index-url` can be used. + * Use `aria2` as downloader * [howto](docs/en/use_aria2.md) diff --git a/cm-cli.py b/cm-cli.py index fa2267aa..46757fb0 100644 --- a/cm-cli.py +++ b/cm-cli.py @@ -647,7 +647,7 @@ def install( cmd_ctx.set_channel_mode(channel, mode) cmd_ctx.set_no_deps(no_deps) - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) for_each_nodes(nodes, act=install_node) pip_fixer.fix_broken() @@ -685,7 +685,7 @@ def reinstall( cmd_ctx.set_channel_mode(channel, mode) cmd_ctx.set_no_deps(no_deps) - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) for_each_nodes(nodes, act=reinstall_node) pip_fixer.fix_broken() @@ -739,7 +739,7 @@ def update( if 'all' in nodes: asyncio.run(auto_save_snapshot()) - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) for x in nodes: if x.lower() in ['comfyui', 'comfy', 'all']: @@ -840,7 +840,7 @@ def fix( if 'all' in nodes: asyncio.run(auto_save_snapshot()) - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) for_each_nodes(nodes, fix_node, allow_all=True) pip_fixer.fix_broken() @@ -1119,7 +1119,7 @@ def restore_snapshot( print(f"[bold red]ERROR: `{snapshot_path}` is not exists.[/bold red]") exit(1) - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) try: asyncio.run(core.restore_snapshot(snapshot_path, extras)) except Exception: @@ -1151,7 +1151,7 @@ def restore_dependencies( total = len(node_paths) i = 1 - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) for x in node_paths: print("----------------------------------------------------------------------------------------------------") print(f"Restoring [{i}/{total}]: {x}") @@ -1170,7 +1170,7 @@ def post_install( ): path = os.path.expanduser(path) - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) unified_manager.execute_install_script('', path, instant_execution=True) pip_fixer.fix_broken() @@ -1214,8 +1214,7 @@ def install_deps( print(f"[bold red]Invalid json file: {deps}[/bold red]") exit(1) - - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, core.manager_files_path) for k in json_obj['custom_nodes'].keys(): state = core.simple_check_custom_node(k) if state == 'installed': diff --git a/glob/manager_core.py b/glob/manager_core.py index bca7f9a7..1cdc2963 100644 --- a/glob/manager_core.py +++ b/glob/manager_core.py @@ -43,7 +43,7 @@ import manager_downloader from node_package import InstalledNodePackage -version_code = [3, 29] +version_code = [3, 30] version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') @@ -847,7 +847,7 @@ class UnifiedManager: else: if os.path.exists(requirements_path) and not no_deps: print("Install: pip packages") - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, manager_files_path) res = True lines = manager_util.robust_readlines(requirements_path) for line in lines: @@ -1902,7 +1902,7 @@ def execute_install_script(url, repo_path, lazy_mode=False, instant_execution=Fa else: if os.path.exists(requirements_path) and not no_deps: print("Install: pip packages") - pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) + pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, manager_files_path) with open(requirements_path, "r") as requirements_file: for line in requirements_file: #handle comments diff --git a/glob/manager_util.py b/glob/manager_util.py index 63932a87..3b4ef6c7 100644 --- a/glob/manager_util.py +++ b/glob/manager_util.py @@ -2,6 +2,7 @@ description: `manager_util` is the lightest module shared across the prestartup_script, main code, and cm-cli of ComfyUI-Manager. """ +import traceback import aiohttp import json @@ -13,6 +14,7 @@ import sys import re import logging import platform +import shlex cache_lock = threading.Lock() @@ -257,6 +259,46 @@ def clear_pip_cache(): pip_map = None +def parse_requirement_line(line): + tokens = shlex.split(line) + if not tokens: + return None + + package_spec = tokens[0] + + pattern = re.compile( + r'^(?P[A-Za-z0-9_.+-]+)' + r'(?P==|>=|<=|!=|~=|>|<)?' + r'(?P[A-Za-z0-9_.+-]*)$' + ) + m = pattern.match(package_spec) + if not m: + return None + + package = m.group('package') + operator = m.group('operator') or None + version = m.group('version') or None + + index_url = None + if '--index-url' in tokens: + idx = tokens.index('--index-url') + if idx + 1 < len(tokens): + index_url = tokens[idx + 1] + + res = {'package': package} + + if operator is not None: + res['operator'] = operator + + if version is not None: + res['version'] = StrictVersion(version) + + if index_url is not None: + res['index_url'] = index_url + + return res + + torch_torchvision_torchaudio_version_map = { '2.6.0': ('0.21.0', '2.6.0'), '2.5.1': ('0.20.0', '2.5.0'), @@ -276,10 +318,12 @@ torch_torchvision_torchaudio_version_map = { } + class PIPFixer: - def __init__(self, prev_pip_versions, comfyui_path): + def __init__(self, prev_pip_versions, comfyui_path, manager_files_path): self.prev_pip_versions = { **prev_pip_versions } self.comfyui_path = comfyui_path + self.manager_files_path = manager_files_path def torch_rollback(self): spec = self.prev_pip_versions['torch'].split('+') @@ -374,14 +418,15 @@ class PIPFixer: if StrictVersion(np) >= StrictVersion('2'): cmd = make_pip_cmd(['install', "numpy<2"]) subprocess.check_output(cmd , universal_newlines=True) + + logging.info("[ComfyUI-Manager] 'numpy' dependency were fixed") except Exception as e: logging.error("[ComfyUI-Manager] Failed to restore numpy") logging.error(e) # fix missing frontend try: - front = new_pip_versions.get('comfyui_frontend_package') - if front is None: + if 'comfyui-frontend-package' not in new_pip_versions: requirements_path = os.path.join(self.comfyui_path, 'requirements.txt') with open(requirements_path, 'r') as file: @@ -390,10 +435,58 @@ class PIPFixer: front_line = next((line.strip() for line in lines if line.startswith('comfyui-frontend-package')), None) cmd = make_pip_cmd(['install', front_line]) subprocess.check_output(cmd , universal_newlines=True) + + logging.info("[ComfyUI-Manager] 'comfyui-frontend-package' dependency were fixed") except Exception as e: - logging.error("[ComfyUI-Manager] Failed to restore comfyui_frontend_package") + logging.error("[ComfyUI-Manager] Failed to restore comfyui-frontend-package") logging.error(e) + # restore based on custom list + pip_auto_fix_path = os.path.join(self.manager_files_path, "pip_auto_fix.list") + if os.path.exists(pip_auto_fix_path): + with open(pip_auto_fix_path, 'r', encoding="UTF-8", errors="ignore") as f: + fixed_list = [] + + for x in f.readlines(): + try: + parsed = parse_requirement_line(x) + need_to_reinstall = True + + if parsed['package'] in new_pip_versions: + if 'version' in parsed and 'operator' in parsed: + cur = StrictVersion(new_pip_versions[parsed['package']]) + dest = parsed['version'] + op = parsed['operator'] + if cur == dest: + if op in ['==', '>=', '<=']: + need_to_reinstall = False + elif cur < dest: + if op in ['<=', '<', '~=', '!=']: + need_to_reinstall = False + elif cur > dest: + if op in ['>=', '>', '~=', '!=']: + need_to_reinstall = False + + if need_to_reinstall: + cmd_args = ['install'] + if 'version' in parsed and 'operator' in parsed: + cmd_args.append(parsed['package']+parsed['operator']+parsed['version'].version_string) + + if 'index_url' in parsed: + cmd_args.append('--index-url') + cmd_args.append(parsed['index_url']) + + cmd = make_pip_cmd(cmd_args) + subprocess.check_output(cmd, universal_newlines=True) + + fixed_list.append(parsed['package']) + except Exception as e: + traceback.print_exc() + logging.error(f"[ComfyUI-Manager] Failed to restore '{x}'") + logging.error(e) + + if len(fixed_list) > 0: + logging.info(f"[ComfyUI-Manager] dependencies in pip_auto_fix.json were fixed: {fixed_list}") def sanitize(data): return data.replace("<", "<").replace(">", ">") diff --git a/prestartup_script.py b/prestartup_script.py index 6c989ff2..71900969 100644 --- a/prestartup_script.py +++ b/prestartup_script.py @@ -507,7 +507,7 @@ check_bypass_ssl() # Perform install processed_install = set() script_list_path = os.path.join(folder_paths.user_directory, "default", "ComfyUI-Manager", "startup-scripts", "install-scripts.txt") -pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path) +pip_fixer = manager_util.PIPFixer(manager_util.get_installed_packages(), comfy_path, manager_files_path) def is_installed(name): diff --git a/pyproject.toml b/pyproject.toml index 26a4a87b..cd01bd6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-manager" description = "ComfyUI-Manager provides features to install and manage custom nodes for ComfyUI, as well as various functionalities to assist with ComfyUI." -version = "3.29" +version = "3.30" license = { file = "LICENSE.txt" } dependencies = ["GitPython", "PyGithub", "matrix-client==0.4.0", "transformers", "huggingface-hub>0.20", "typer", "rich", "typing-extensions", "toml", "uv", "chardet"]