From ad1b4a9a86ba90ed9ecdf991948ecdbbfa2c6a73 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Tue, 18 Feb 2025 23:41:44 +0900 Subject: [PATCH] feat: reverse proxy https://github.com/ltdrdata/ComfyUI-Manager/pull/795/files --- README.md | 23 ++++++++++ glob/git_utils.py | 24 ++++++---- glob/manager_core.py | 92 ++++++++++++++++---------------------- glob/manager_downloader.py | 1 + pyproject.toml | 2 +- 5 files changed, 80 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 5cc0f189..d288c87b 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,29 @@ The following settings are applied based on the section marked as `is_default`. * This option can be used if performance issues occur in a Colab+GDrive environment. +## Environment Variables + +The following features can be configured using environment variables: + +* **COMFYUI_PATH**: The installation path of ComfyUI +* **GITHUB_ENDPOINT**: Reverse proxy configuration for environments with limited access to GitHub +* **HF_ENDPOINT**: Reverse proxy configuration for environments with limited access to Hugging Face + + +### Example 1: +Redirecting `https://github.com/ltdrdata/ComfyUI-Impact-Pack` to `https://mirror.ghproxy.com/https://github.com/ltdrdata/ComfyUI-Impact-Pack` + +``` +GITHUB_ENDPOINT=https://mirror.ghproxy.com/https://github.com +``` + +#### Example 2: +Changing `https://huggingface.co/path/to/somewhere` to `https://some-hf-mirror.com/path/to/somewhere` + +``` +HF_ENDPOINT=https://some-hf-mirror.com +``` + ## Scanner When you run the `scan.sh` script: diff --git a/glob/git_utils.py b/glob/git_utils.py index 720edd4b..29946862 100644 --- a/glob/git_utils.py +++ b/glob/git_utils.py @@ -2,6 +2,9 @@ import os import configparser +GITHUB_ENDPOINT = os.getenv('GITHUB_ENDPOINT') + + def is_git_repo(path: str) -> bool: """ Check if the path is a git repository. """ # NOTE: Checking it through `git.Repo` must be avoided. @@ -46,16 +49,21 @@ def git_url(fullpath): return None + def normalize_url(url) -> str: - url = url.replace("git@github.com:", "https://github.com/") - if url.endswith('.git'): - url = url[:-4] + if 'github' in url or (GITHUB_ENDPOINT is not None and GITHUB_ENDPOINT in url): + author = os.path.basename(os.path.dirname(url)) + repo_name = os.path.basename(url) + url = f"https://github.com/{author}/{repo_name}" return url -def normalize_url_http(url) -> str: - url = url.replace("https://github.com/", "git@github.com:") - if url.endswith('.git'): - url = url[:-4] - return url \ No newline at end of file +def get_url_for_clone(url): + url = normalize_url(url) + + if GITHUB_ENDPOINT is not None and url.startswith('https://github.com/'): + url = GITHUB_ENDPOINT + url[18:] # url[18:] -> remove `https://github.com` + + return url + \ No newline at end of file diff --git a/glob/manager_core.py b/glob/manager_core.py index 311cdf4a..57000c14 100644 --- a/glob/manager_core.py +++ b/glob/manager_core.py @@ -42,7 +42,7 @@ import manager_downloader from node_package import InstalledNodePackage -version_code = [3, 23] +version_code = [3, 24] version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') @@ -505,6 +505,8 @@ class UnifiedManager: def resolve_from_path(self, fullpath): url = git_utils.git_url(fullpath) if url: + url = git_utils.normalize_url(url) + cnr = self.get_cnr_by_repo(url) commit_hash = git_utils.get_commit_hash(fullpath) if cnr: @@ -1239,15 +1241,16 @@ class UnifiedManager: if url.endswith("/"): url = url[:-1] try: - print(f"Download: git clone '{url}'") - # Clone the repository from the remote URL + clone_url = git_utils.get_url_for_clone(url) + print(f"Download: git clone '{clone_url}'") + if not instant_execution and platform.system() == 'Windows': - res = manager_funcs.run_script([sys.executable, git_script_path, "--clone", get_default_custom_nodes_path(), url, repo_path], cwd=get_default_custom_nodes_path()) + res = manager_funcs.run_script([sys.executable, git_script_path, "--clone", get_default_custom_nodes_path(), clone_url, repo_path], cwd=get_default_custom_nodes_path()) if res != 0: - return result.fail(f"Failed to clone repo: {url}") + return result.fail(f"Failed to clone repo: {clone_url}") else: - repo = git.Repo.clone_from(url, repo_path, recursive=True, progress=GitProgress()) + repo = git.Repo.clone_from(clone_url, repo_path, recursive=True, progress=GitProgress()) repo.git.clear_cache() repo.close() @@ -2043,12 +2046,14 @@ async def gitclone_install(url, instant_execution=False, msg_prefix='', no_deps= print(f"CLONE into '{repo_path}'") # Clone the repository from the remote URL + clone_url = git_utils.get_url_for_clone(url) + if not instant_execution and platform.system() == 'Windows': - res = manager_funcs.run_script([sys.executable, git_script_path, "--clone", get_default_custom_nodes_path(), url, repo_path], cwd=get_default_custom_nodes_path()) + res = manager_funcs.run_script([sys.executable, git_script_path, "--clone", get_default_custom_nodes_path(), clone_url, repo_path], cwd=get_default_custom_nodes_path()) if res != 0: - return result.fail(f"Failed to clone '{url}' into '{repo_path}'") + return result.fail(f"Failed to clone '{clone_url}' into '{repo_path}'") else: - repo = git.Repo.clone_from(url, repo_path, recursive=True, progress=GitProgress()) + repo = git.Repo.clone_from(clone_url, repo_path, recursive=True, progress=GitProgress()) repo.git.clear_cache() repo.close() @@ -2973,7 +2978,14 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): print("cm-cli: unexpected [0001]") # for nightly restore - git_info = info.get('git_custom_nodes') + _git_info = info.get('git_custom_nodes') + git_info = {} + + # normalize github repo + for k, v in _git_info.items(): + norm_k = git_utils.normalize_url(k) + git_info[norm_k] = v + if git_info is not None: todo_disable = [] todo_enable = [] @@ -2986,20 +2998,13 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if v[0] == 'nightly' and cnr_repo_map.get(k): repo_url = cnr_repo_map.get(k) + normalized_url = git_utils.normalize_url(repo_url) - normalized_url1 = git_utils.normalize_url(repo_url) - normalized_url2 = git_utils.normalize_url_http(repo_url) - - if normalized_url1 not in git_info and normalized_url2 not in git_info: + if normalized_url not in git_info: todo_disable.append(k) else: - if normalized_url1 in git_info: - commit_hash = git_info[normalized_url1]['hash'] - todo_checkout.append((v[1], commit_hash)) - - if normalized_url2 in git_info: - commit_hash = git_info[normalized_url2]['hash'] - todo_checkout.append((v[1], commit_hash)) + commit_hash = git_info[normalized_url]['hash'] + todo_checkout.append((v[1], commit_hash)) for k, v in unified_manager.nightly_inactive_nodes.items(): if 'comfyui-manager' in k: @@ -3007,18 +3012,12 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if cnr_repo_map.get(k): repo_url = cnr_repo_map.get(k) - normalized_url1 = git_utils.normalize_url(repo_url) - normalized_url2 = git_utils.normalize_url_http(repo_url) + normalized_url = git_utils.normalize_url(repo_url) - if normalized_url1 in git_info: - commit_hash = git_info[normalized_url1]['hash'] + if normalized_url in git_info: + commit_hash = git_info[normalized_url]['hash'] todo_enable.append((k, commit_hash)) - processed_urls.append(normalized_url1) - - if normalized_url2 in git_info: - commit_hash = git_info[normalized_url2]['hash'] - todo_enable.append((k, commit_hash)) - processed_urls.append(normalized_url2) + processed_urls.append(normalized_url) for x in todo_disable: unified_manager.unified_disable(x, False) @@ -3071,21 +3070,14 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if repo_url is None: continue - normalized_url1 = git_utils.normalize_url(repo_url) - normalized_url2 = git_utils.normalize_url_http(repo_url) + normalized_url = git_utils.normalize_url(repo_url) - if normalized_url1 not in git_info and normalized_url2 not in git_info: + if normalized_url not in git_info: todo_disable.append(k2) else: - if normalized_url1 in git_info: - commit_hash = git_info[normalized_url1]['hash'] - todo_checkout.append((k2, commit_hash)) - processed_urls.append(normalized_url1) - - if normalized_url2 in git_info: - commit_hash = git_info[normalized_url2]['hash'] - todo_checkout.append((k2, commit_hash)) - processed_urls.append(normalized_url2) + commit_hash = git_info[normalized_url]['hash'] + todo_checkout.append((k2, commit_hash)) + processed_urls.append(normalized_url) for k2, v2 in unified_manager.unknown_inactive_nodes.items(): repo_url = resolve_giturl_from_path(v2[1]) @@ -3093,18 +3085,12 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if repo_url is None: continue - normalized_url1 = git_utils.normalize_url(repo_url) - normalized_url2 = git_utils.normalize_url_http(repo_url) + normalized_url = git_utils.normalize_url(repo_url) - if normalized_url1 in git_info: - commit_hash = git_info[normalized_url1]['hash'] + if normalized_url in git_info: + commit_hash = git_info[normalized_url]['hash'] todo_enable.append((k2, commit_hash)) - processed_urls.append(normalized_url1) - - if normalized_url2 in git_info: - commit_hash = git_info[normalized_url2]['hash'] - todo_enable.append((k2, commit_hash)) - processed_urls.append(normalized_url2) + processed_urls.append(normalized_url) for x in todo_disable: unified_manager.unified_disable(x, True) diff --git a/glob/manager_downloader.py b/glob/manager_downloader.py index cce62e50..2d76b913 100644 --- a/glob/manager_downloader.py +++ b/glob/manager_downloader.py @@ -11,6 +11,7 @@ from tqdm.auto import tqdm aria2 = os.getenv('COMFYUI_MANAGER_ARIA2_SERVER') HF_ENDPOINT = os.getenv('HF_ENDPOINT') + if aria2 is not None: secret = os.getenv('COMFYUI_MANAGER_ARIA2_SECRET') url = urlparse(aria2) diff --git a/pyproject.toml b/pyproject.toml index 58d5ab52..e2ef4b8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-manager" description = "ComfyUI-Manager provides features to install and manage custom nodes for ComfyUI, as well as various functionalities to assist with ComfyUI." -version = "3.23" +version = "3.24" license = { file = "LICENSE.txt" } dependencies = ["GitPython", "PyGithub", "matrix-client==0.4.0", "transformers", "huggingface-hub>0.20", "typer", "rich", "typing-extensions"]