From 06286426c0150f648e270b81b56c5c98821af5b8 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Mon, 6 Nov 2023 23:16:00 +0900 Subject: [PATCH] improve: scanner.py - parallel downalod update DB --- custom-node-list.json | 6 +++--- extension-node-map.json | 2 +- node_db/new/custom-node-list.json | 10 ++++++++++ node_db/new/extension-node-map.json | 2 +- scanner.py | 9 ++++++--- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/custom-node-list.json b/custom-node-list.json index 0570f2b6..280dca44 100644 --- a/custom-node-list.json +++ b/custom-node-list.json @@ -1606,14 +1606,14 @@ "description": "Nodes:TC_EqualizeCLAHE, TC_SizeApproximation, TC_ImageResize, TC_ImageScale, TC_ColorFill." }, { - "author": "TeaCrab", - "title": "ComfyUI-TeaNodes", + "author": "nagolinc", + "title": "ComfyUI_FastVAEDecorder_SDXL", "reference": "https://github.com/nagolinc/ComfyUI_FastVAEDecorder_SDXL", "files": [ "https://github.com/nagolinc/ComfyUI_FastVAEDecorder_SDXL" ], "install_type": "git-clone", - "description": "Nodes:FastLatentToImage" + "description": "Based off of: Birch-san/diffusers-play/approx_vae. This ComfyUI node allows you to quickly preview SDXL 1.0 latents." }, { "author": "bradsec", diff --git a/extension-node-map.json b/extension-node-map.json index 1a73b951..e5981245 100644 --- a/extension-node-map.json +++ b/extension-node-map.json @@ -3351,7 +3351,7 @@ "FastLatentToImage" ], { - "title_aux": "ComfyUI-TeaNodes" + "title_aux": "ComfyUI_FastVAEDecorder_SDXL" } ], "https://github.com/nicolai256/comfyUI_Nodes_nicolai256/raw/main/yugioh-presets.py": [ diff --git a/node_db/new/custom-node-list.json b/node_db/new/custom-node-list.json index 27c814b1..3a6fba72 100644 --- a/node_db/new/custom-node-list.json +++ b/node_db/new/custom-node-list.json @@ -1,5 +1,15 @@ { "custom_nodes": [ + { + "author": "nagolinc", + "title": "ComfyUI_FastVAEDecorder_SDXL", + "reference": "https://github.com/nagolinc/ComfyUI_FastVAEDecorder_SDXL", + "files": [ + "https://github.com/nagolinc/ComfyUI_FastVAEDecorder_SDXL" + ], + "install_type": "git-clone", + "description": "Based off of: Birch-san/diffusers-play/approx_vae. This ComfyUI node allows you to quickly preview SDXL 1.0 latents." + }, { "author": "jags111", "title": "ComfyUI_Jags_VectorMagic", diff --git a/node_db/new/extension-node-map.json b/node_db/new/extension-node-map.json index 1a73b951..e5981245 100644 --- a/node_db/new/extension-node-map.json +++ b/node_db/new/extension-node-map.json @@ -3351,7 +3351,7 @@ "FastLatentToImage" ], { - "title_aux": "ComfyUI-TeaNodes" + "title_aux": "ComfyUI_FastVAEDecorder_SDXL" } ], "https://github.com/nicolai256/comfyUI_Nodes_nicolai256/raw/main/yugioh-presets.py": [ diff --git a/scanner.py b/scanner.py index 936a1c14..37664b9a 100644 --- a/scanner.py +++ b/scanner.py @@ -171,14 +171,14 @@ def update_custom_nodes(): node_info[name] = (url, title) clone_or_pull_git_repository(url) - max_threads = 10 - with concurrent.futures.ThreadPoolExecutor(max_threads) as executor: + with concurrent.futures.ThreadPoolExecutor(10) as executor: for url, title in git_url_titles: executor.submit(process_git_url_title, url, title) py_url_titles = get_py_urls_from_json('custom-node-list.json') - for url, title in py_url_titles: + def download_and_store_info(url_title): + url, title = url_title name = os.path.basename(url) if name.endswith(".py"): node_info[name] = (url, title) @@ -187,6 +187,9 @@ def update_custom_nodes(): download_url(url, ".tmp") except: print(f"[ERROR] Cannot download '{url}'") + + with concurrent.futures.ThreadPoolExecutor(10) as executor: + executor.map(download_and_store_info, py_url_titles) return node_info