From db6025a278c60b52ea266b561b6bd548d39dcec4 Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Fri, 2 Jun 2023 16:56:49 +0900 Subject: [PATCH] update db scanner.py: prevent redundant --- alter-list.json | 2 +- extension-node-map.json | 238 +--------------------------------------- scanner.py | 25 +++-- 3 files changed, 22 insertions(+), 243 deletions(-) diff --git a/alter-list.json b/alter-list.json index c83bd43e..d7b9fd45 100644 --- a/alter-list.json +++ b/alter-list.json @@ -66,7 +66,7 @@ "description": "This node takes two models, merges individual blocks together at various ratios, and automatically rates each merge, keeping the ratio with the highest score. " }, { - "id":"https://github.com/asd417/tomeSD_for_Comfy/raw/main/tomeloader.py", + "id":"https://github.com/ltdrdata/ComfyUI-tomeSD-installer", "tags":"tomesd", "description": "This extension is a ComfyUI wrapper for 'tomesd' that improves generation speed through token merging." }, diff --git a/extension-node-map.json b/extension-node-map.json index a9bc217f..f610cfca 100644 --- a/extension-node-map.json +++ b/extension-node-map.json @@ -53,123 +53,63 @@ "MultiLatentComposite" ], "https://github.com/Derfuu/Derfuu_ComfyUI_ModdedNodes": [ - "ABSNode_DF", - "ABSNode_DF", "ABSNode_DF", "Absolute value", "Ceil", "CeilNode_DF", - "CeilNode_DF", - "CeilNode_DF", "Conditioning area scale by ratio", "ConditioningSetArea with tuples", "ConditioningSetAreaEXT_DF", - "ConditioningSetAreaEXT_DF", - "ConditioningSetAreaEXT_DF", "ConditioningSetArea_DF", - "ConditioningSetArea_DF", - "ConditioningSetArea_DF", - "CosNode_DF", - "CosNode_DF", "CosNode_DF", "Cosines", "Divide", "DivideNode_DF", - "DivideNode_DF", - "DivideNode_DF", - "EmptyLatentImage_DF", - "EmptyLatentImage_DF", "EmptyLatentImage_DF", "Float", "Float debug print", "Float2Tuple_DF", - "Float2Tuple_DF", - "Float2Tuple_DF", "FloatDebugPrint_DF", - "FloatDebugPrint_DF", - "FloatDebugPrint_DF", - "FloatNode_DF", - "FloatNode_DF", "FloatNode_DF", "Floor", "FloorNode_DF", - "FloorNode_DF", - "FloorNode_DF", "Get image size", "Get latent size", "GetImageSize_DF", - "GetImageSize_DF", - "GetImageSize_DF", - "GetLatentSize_DF", - "GetLatentSize_DF", "GetLatentSize_DF", "Image scale by ratio", "Image scale to side", "ImageScale_Ratio_DF", - "ImageScale_Ratio_DF", - "ImageScale_Ratio_DF", - "ImageScale_Side_DF", - "ImageScale_Side_DF", "ImageScale_Side_DF", "Int debug print", "Int to float", "Int to tuple", "Int2Float_DF", - "Int2Float_DF", - "Int2Float_DF", - "IntDebugPrint_DF", - "IntDebugPrint_DF", "IntDebugPrint_DF", "Integer", "IntegerNode_DF", - "IntegerNode_DF", - "IntegerNode_DF", "Latent Scale by ratio", "Latent Scale to side", "LatentComposite with tuples", "LatentScale_Ratio_DF", - "LatentScale_Ratio_DF", - "LatentScale_Ratio_DF", "LatentScale_Side_DF", - "LatentScale_Side_DF", - "LatentScale_Side_DF", - "MultilineStringNode_DF", - "MultilineStringNode_DF", "MultilineStringNode_DF", "Multiply", "MultiplyNode_DF", - "MultiplyNode_DF", - "MultiplyNode_DF", - "PowNode_DF", - "PowNode_DF", "PowNode_DF", "Power", "Random", "RandomFloat_DF", - "RandomFloat_DF", - "RandomFloat_DF", - "SinNode_DF", - "SinNode_DF", "SinNode_DF", "Sinus", "SqrtNode_DF", - "SqrtNode_DF", - "SqrtNode_DF", "Square root", "String debug print", "StringNode_DF", - "StringNode_DF", - "StringNode_DF", "Subtract", "SubtractNode_DF", - "SubtractNode_DF", - "SubtractNode_DF", "Sum", "SumNode_DF", - "SumNode_DF", - "SumNode_DF", - "TanNode_DF", - "TanNode_DF", "TanNode_DF", "Tangent", "Text", @@ -180,138 +120,60 @@ "Tuple to floats", "Tuple to ints", "Tuple2Float_DF", - "Tuple2Float_DF", - "Tuple2Float_DF", "TupleDebugPrint_DF", - "TupleDebugPrint_DF", - "TupleDebugPrint_DF", - "TupleNode_DF", - "TupleNode_DF", "TupleNode_DF" ], "https://github.com/EllangoK/ComfyUI-post-processing-nodes": [ "ArithmeticBlend", - "ArithmeticBlend", - "ArithmeticBlend", - "AsciiArt", - "AsciiArt", "AsciiArt", "Blend", - "Blend", - "Blend", - "Blur", - "Blur", "Blur", "CannyEdgeMask", - "CannyEdgeMask", - "CannyEdgeMask", - "ChromaticAberration", - "ChromaticAberration", "ChromaticAberration", "ColorCorrect", - "ColorCorrect", - "ColorCorrect", "ColorTint", - "ColorTint", - "ColorTint", - "Dissolve", - "Dissolve", "Dissolve", "Dither", - "Dither", - "DodgeAndBurn", - "DodgeAndBurn", "DodgeAndBurn", "FilmGrain", - "FilmGrain", - "FilmGrain", "Glow", - "Glow", - "Glow", - "HSVThresholdMask", - "HSVThresholdMask", "HSVThresholdMask", "KMeansQuantize", - "KMeansQuantize", - "KuwaharaBlur", - "KuwaharaBlur", "KuwaharaBlur", "Parabolize", - "Parabolize", - "Parabolize", - "PencilSketch", - "PencilSketch", "PencilSketch", "PixelSort", - "PixelSort", - "PixelSort", - "Pixelize", - "Pixelize", "Pixelize", "Quantize", - "Quantize", - "Quantize", - "Sharpen", - "Sharpen", "Sharpen", "SineWave", - "SineWave", - "SineWave", "Solarize", - "Solarize", - "Solarize", - "Vignette", - "Vignette", "Vignette" ], "https://github.com/Fannovel16/comfy_controlnet_preprocessors": [ "AnimeLineArtPreprocessor", - "AnimeLineArtPreprocessor", - "BAE-NormalMapPreprocessor", "BAE-NormalMapPreprocessor", "BinaryPreprocessor", - "BinaryPreprocessor", - "CannyEdgePreprocessor", "CannyEdgePreprocessor", "ColorPreprocessor", - "ColorPreprocessor", - "FakeScribblePreprocessor", "FakeScribblePreprocessor", "HEDPreprocessor", - "HEDPreprocessor", - "LeReS-DepthMapPreprocessor", "LeReS-DepthMapPreprocessor", "LineArtPreprocessor", - "LineArtPreprocessor", - "M-LSDPreprocessor", "M-LSDPreprocessor", "Manga2Anime-LineArtPreprocessor", - "Manga2Anime-LineArtPreprocessor", - "MediaPipe-FaceMeshPreprocessor", "MediaPipe-FaceMeshPreprocessor", "MediaPipe-HandPosePreprocessor", - "MediaPipe-HandPosePreprocessor", - "MiDaS-DepthMapPreprocessor", "MiDaS-DepthMapPreprocessor", "MiDaS-NormalMapPreprocessor", - "MiDaS-NormalMapPreprocessor", - "OneFormer-ADE20K-SemSegPreprocessor", "OneFormer-ADE20K-SemSegPreprocessor", "OneFormer-COCO-SemSegPreprocessor", - "OneFormer-COCO-SemSegPreprocessor", - "OpenposePreprocessor", "OpenposePreprocessor", "PiDiNetPreprocessor", - "PiDiNetPreprocessor", - "ScribblePreprocessor", "ScribblePreprocessor", "SemSegPreprocessor", - "SemSegPreprocessor", - "TilePreprocessor", "TilePreprocessor", "UniFormer-SemSegPreprocessor", - "UniFormer-SemSegPreprocessor", - "Zoe-DepthMapPreprocessor", "Zoe-DepthMapPreprocessor" ], "https://github.com/Fannovel16/comfy_video": [ @@ -352,58 +214,31 @@ ], "https://github.com/Nourepide/ComfyUI-Allore": [ "AlphaChanelByMask", - "AlphaChanelByMask", - "AlphaChanelRestore", "AlphaChanelRestore", "ClipClamp", - "ClipClamp", - "ClipVisionClamp", "ClipVisionClamp", "ClipVisionOutputClamp", - "ClipVisionOutputClamp", - "ConditioningClamp", "ConditioningClamp", "ControlNetClamp", - "ControlNetClamp", - "GligenClamp", "GligenClamp", "ImageClamp", - "ImageClamp", - "ImageCompositeAbsolute", "ImageCompositeAbsolute", "ImageCompositeRelative", - "ImageCompositeRelative", - "ImageContainer", "ImageContainer", "ImageContainerInheritanceAdd", - "ImageContainerInheritanceAdd", - "ImageContainerInheritanceMax", "ImageContainerInheritanceMax", "ImageContainerInheritanceScale", - "ImageContainerInheritanceScale", - "ImageContainerInheritanceSum", "ImageContainerInheritanceSum", "ImageSegmentation", - "ImageSegmentation", - "ImageSegmentationCustom", "ImageSegmentationCustom", "ImageSegmentationCustomAdvanced", - "ImageSegmentationCustomAdvanced", - "ImageText", "ImageText", "ImageTextOutlined", - "ImageTextOutlined", - "LatentClamp", "LatentClamp", "MaskClamp", - "MaskClamp", - "ModelClamp", "ModelClamp", "StyleModelClamp", - "StyleModelClamp", "UpscaleModelClamp", - "UpscaleModelClamp", - "VaeClamp", "VaeClamp" ], "https://github.com/SLAPaper/ComfyUI-Image-Selector": [ @@ -561,6 +396,7 @@ "SAM Model Loader", "SAM Parameters", "SAM Parameters Combine", + "Samples Passthrough (Stat System)", "Save Text File", "Seed", "String to Text", @@ -597,10 +433,6 @@ "https://github.com/YinBailiang/MergeBlockWeighted_fo_ComfyUI": [ "MergeBlockWeighted" ], - "https://github.com/asd417/tomeSD_for_Comfy/raw/main/tomeloader.py": [ - "CheckpointTomeLoader", - "TestNode" - ], "https://github.com/biegert/ComfyUI-CLIPSeg/raw/main/custom_nodes/clipseg.py": [ "CLIPSeg", "CombineSegMasks" @@ -644,28 +476,16 @@ ], "https://github.com/diontimmer/ComfyUI-Vextra-Nodes": [ "Add Text To Image", - "Add Text To Image", - "Apply Instagram Filter", "Apply Instagram Filter", "Create Solid Color", - "Create Solid Color", - "Flatten Colors", "Flatten Colors", "Generate Noise Image", - "Generate Noise Image", - "GlitchThis Effect", "GlitchThis Effect", "Hue Rotation", - "Hue Rotation", - "Load Picture Index", "Load Picture Index", "Pixel Sort", - "Pixel Sort", - "Play Sound At Execution", "Play Sound At Execution", "Prettify Prompt Using distilgpt2", - "Prettify Prompt Using distilgpt2", - "Swap Color Mode", "Swap Color Mode" ], "https://github.com/flyingshutter/As_ComfyUI_CustomNodes": [ @@ -761,6 +581,10 @@ "TwoSamplersForMaskUpscalerProvider", "TwoSamplersForMaskUpscalerProviderPipe" ], + "https://github.com/ltdrdata/ComfyUI-tomeSD-installer": [ + "CheckpointTomeLoader", + "TestNode" + ], "https://github.com/m957ymj75urz/ComfyUI-Custom-Nodes/raw/main/clip-text-encode-split/clip_text_encode_split.py": [ "RawText", "RawTextCombine", @@ -769,108 +593,56 @@ ], "https://github.com/omar92/ComfyUI-QualityOfLifeSuit_Omar92": [ "CLIPStringEncode _O", - "CLIPStringEncode _O", - "Chat completion _O", "Chat completion _O", "ChatGPT Simple _O", - "ChatGPT Simple _O", - "ChatGPT _O", "ChatGPT _O", "ChatGPT compact _O", - "ChatGPT compact _O", - "Chat_Completion _O", "Chat_Completion _O", "Chat_Message _O", - "Chat_Message _O", - "Chat_Message_fromString _O", "Chat_Message_fromString _O", "Concat Text _O", - "Concat Text _O", - "ConcatRandomNSP_O", "ConcatRandomNSP_O", "Debug String _O", - "Debug String _O", - "Debug Text _O", "Debug Text _O", "Debug Text route _O", - "Debug Text route _O", - "Edit_image _O", "Edit_image _O", "Equation1param _O", - "Equation1param _O", - "Equation2params _O", "Equation2params _O", "GetImage_(Width&Height) _O", - "GetImage_(Width&Height) _O", - "GetLatent_(Width&Height) _O", "GetLatent_(Width&Height) _O", "ImageScaleFactor _O", - "ImageScaleFactor _O", - "ImageScaleFactorSimple _O", "ImageScaleFactorSimple _O", "LatentUpscaleFactor _O", - "LatentUpscaleFactor _O", - "LatentUpscaleFactorSimple _O", "LatentUpscaleFactorSimple _O", "LatentUpscaleMultiply", - "LatentUpscaleMultiply", - "Note _O", "Note _O", "RandomNSP _O", - "RandomNSP _O", - "Replace Text _O", "Replace Text _O", "String _O", - "String _O", - "Text _O", "Text _O", "Text2Image _O", - "Text2Image _O", - "Trim Text _O", "Trim Text _O", "VAEDecodeParallel _O", - "VAEDecodeParallel _O", - "combine_chat_messages _O", "combine_chat_messages _O", "compine_chat_messages _O", - "compine_chat_messages _O", - "concat Strings _O", "concat Strings _O", "create image _O", - "create image _O", - "create_image _O", "create_image _O", "debug Completeion _O", - "debug Completeion _O", - "debug messages_O", "debug messages_O", "float _O", - "float _O", - "floatToInt _O", "floatToInt _O", "floatToText _O", - "floatToText _O", - "int _O", "int _O", "intToFloat _O", - "intToFloat _O", - "load_openAI _O", "load_openAI _O", "replace String _O", - "replace String _O", - "replace String advanced _O", "replace String advanced _O", "saveTextToFile _O", - "saveTextToFile _O", - "seed _O", "seed _O", "selectLatentFromBatch _O", - "selectLatentFromBatch _O", - "string2Image _O", "string2Image _O", "trim String _O", - "trim String _O", - "variation_image _O", "variation_image _O" ], "https://github.com/paulo-coronado/comfy_clip_blip_node": [ diff --git a/scanner.py b/scanner.py index 4e2f99e7..bda1f229 100644 --- a/scanner.py +++ b/scanner.py @@ -18,14 +18,14 @@ def scan_in_file(filename): else: return [] - nodes = [] + nodes = set() class_dict = {} key_value_pairs = re.findall(r"\"([^\"]*)\"\s*:\s*([^,\n]*)", dict_text) for key, value in key_value_pairs: class_dict[key] = value.strip() for key, value in class_dict.items(): - nodes.append(key) + nodes.add(key) return nodes @@ -107,12 +107,13 @@ def clone_or_pull_git_repository(git_url): repo = Repo(repo_dir) origin = repo.remote(name="origin") origin.pull() + repo.git.submodule('update', '--init', '--recursive') print(f"Pulling {repo_name}...") except Exception as e: print(f"Pulling {repo_name} failed: {e}") else: try: - repo = Repo.clone_from(git_url, repo_dir) + repo = Repo.clone_from(git_url, repo_dir, recursive=True) print(f"Cloning {repo_name}...") except Exception as e: print(f"Cloning {repo_name} failed: {e}") @@ -154,13 +155,14 @@ def gen_json(node_info): for dirname in node_dirs: py_files = get_py_file_paths(dirname) - nodes = [] + nodes = set() for py in py_files: - nodes.extend(scan_in_file(py)) + nodes.update(scan_in_file(py)) dirname = os.path.basename(dirname) - if nodes != []: + if len(nodes) > 0: + nodes = list(nodes) nodes.sort() git_url = node_info[dirname] @@ -169,12 +171,17 @@ def gen_json(node_info): for file in node_files: nodes = scan_in_file(file) - if nodes != []: + if len(nodes) > 0: + nodes = list(nodes) nodes.sort() file = os.path.basename(file) - url = node_info[file] - data[url] = nodes + + if file in node_info: + url = node_info[file] + data[url] = nodes + else: + print(f"Missing info: {url}") json_path = f"extension-node-map.json" with open(json_path, "w") as file: