diff --git a/build_static_extensions.py b/build_static_extensions.py new file mode 100644 index 000000000..d42c96cf0 --- /dev/null +++ b/build_static_extensions.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +""" +Static Extensions Builder for ComfyUI + +This script replicates ComfyUI's custom node web directory scanning logic +and uploads all web files to OSS with the same directory structure. + +Usage: + python build_static_extensions.py [--oss] [--output-dir ./static_extensions] [--clean] +""" + +import os +import sys +import glob +import shutil +import argparse +import logging +import importlib.util +from pathlib import Path + +# Add ComfyUI modules to path +sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy")) + +import folder_paths + +# Set up logging +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +logger = logging.getLogger(__name__) + + +# OSS Configuration - Try bucket-specific endpoint +OSS_CONFIG = { + 'endpoint': 'https://oss-cn-shanghai.aliyuncs.com', # Bucket-specific endpoint + 'access_key_id': '', + 'access_key_secret': '', + 'bucket_name': 'tusi-assets', + 'base_path': 'comfyui/howell5/static_extensions/', +} + + +def get_oss_client(): + """Initialize and return OSS client. Returns None if not configured.""" + try: + # Check if OSS is configured + if not all([OSS_CONFIG['endpoint'], OSS_CONFIG['access_key_id'], + OSS_CONFIG['access_key_secret'], OSS_CONFIG['bucket_name']]): + logger.warning("OSS not configured. Falling back to local mode.") + return None + + # Try to import oss2 - if not available, fall back to local mode + try: + import oss2 + from oss2.credentials import StaticCredentialsProvider + except ImportError: + logger.warning("oss2 package not found. Install with: pip install oss2") + return None + + # Initialize OSS client with V4 auth and region + # Use StaticCredentialsProvider for direct key/secret + credentials_provider = StaticCredentialsProvider( + OSS_CONFIG['access_key_id'], + OSS_CONFIG['access_key_secret'] + ) + auth = oss2.ProviderAuthV4(credentials_provider) + + # Extract region from endpoint or use default + region = "cn-shanghai" # extracted from oss-cn-shanghai.aliyuncs.com + + bucket = oss2.Bucket( + auth, + OSS_CONFIG['endpoint'], + OSS_CONFIG['bucket_name'], + region=region + ) + + # Test connection with a simple list operation + logger.info(f"Testing OSS connection to {OSS_CONFIG['endpoint']}/{OSS_CONFIG['bucket_name']}") + + try: + result = bucket.list_objects(max_keys=1) + logger.info(f"OSS connection established to bucket: {OSS_CONFIG['bucket_name']}") + return bucket + except Exception as list_error: + logger.error(f"OSS list test failed: {list_error}") + logger.info("Trying without connection test...") + # Return bucket anyway, let the upload operations handle errors + return bucket + + except Exception as e: + logger.warning(f"Failed to initialize OSS client: {e}. Falling back to local mode.") + return None + + +def upload_to_oss(bucket, local_file_path, oss_key): + """Upload file to OSS.""" + try: + bucket.put_object_from_file(oss_key, local_file_path) + return True + except Exception as e: + logger.error(f"Failed to upload {local_file_path} to OSS: {e}") + return False + + +def get_module_name(module_path: str) -> str: + """ + Returns the module name based on the given module path. + Copied from nodes.py + """ + base_path = os.path.basename(module_path) + if os.path.isfile(module_path): + base_path = os.path.splitext(base_path)[0] + return base_path + + +def scan_custom_node_web_dirs(): + """ + Scan custom nodes and identify their web directories. + Replicates the logic from nodes.py load_custom_node() + """ + extension_web_dirs = {} + + # Get all custom node paths + node_paths = folder_paths.get_folder_paths("custom_nodes") + + for custom_node_path in node_paths: + if not os.path.exists(custom_node_path): + continue + + possible_modules = os.listdir(os.path.realpath(custom_node_path)) + + # Filter out __pycache__ and .disabled files + if "__pycache__" in possible_modules: + possible_modules.remove("__pycache__") + + for possible_module in possible_modules: + module_path = os.path.join(custom_node_path, possible_module) + + # Skip non-python files that aren't directories + if os.path.isfile(module_path) and os.path.splitext(module_path)[1] != ".py": + continue + + # Skip disabled modules + if module_path.endswith(".disabled"): + continue + + module_name = get_module_name(module_path) + logger.info(f"Scanning custom node: {module_name}") + + # Try to detect web directory + web_dir = None + + # Method 1: Try to load module and check WEB_DIRECTORY attribute + try: + if os.path.isfile(module_path): + module_spec = importlib.util.spec_from_file_location(f"temp_{module_name}", module_path) + module_dir = os.path.split(module_path)[0] + else: + init_file = os.path.join(module_path, "__init__.py") + if os.path.exists(init_file): + module_spec = importlib.util.spec_from_file_location(f"temp_{module_name}", init_file) + module_dir = module_path + else: + logger.warning(f"No __init__.py found in {module_path}, skipping") + continue + + if module_spec and module_spec.loader: + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + + if hasattr(module, "WEB_DIRECTORY") and getattr(module, "WEB_DIRECTORY") is not None: + web_dir_name = getattr(module, "WEB_DIRECTORY") + web_dir = os.path.abspath(os.path.join(module_dir, web_dir_name)) + logger.info(f"Found WEB_DIRECTORY: {web_dir}") + + except Exception as e: + logger.warning(f"Could not load module {module_path}: {e}") + + # Method 2: Try common web directory names if no WEB_DIRECTORY found + if not web_dir or not os.path.isdir(web_dir): + common_web_dirs = ["web", "js", "frontend", "static", "assets"] + for web_dir_name in common_web_dirs: + potential_web_dir = os.path.join(module_path if os.path.isdir(module_path) else os.path.dirname(module_path), web_dir_name) + if os.path.isdir(potential_web_dir): + web_dir = potential_web_dir + logger.info(f"Found web directory by convention: {web_dir}") + break + + # Register web directory if found + if web_dir and os.path.isdir(web_dir): + extension_web_dirs[module_name] = web_dir + logger.info(f"Registered web directory for {module_name}: {web_dir}") + else: + logger.debug(f"No web directory found for {module_name}") + + return extension_web_dirs + + +def copy_web_files(extension_web_dirs, output_dir, file_patterns=None, use_oss=False): + """ + Copy all web files from custom nodes to static extensions directory or upload to OSS. + """ + if file_patterns is None: + # Include common web file types + file_patterns = ['**/*.js', '**/*.css', '**/*.html', '**/*.json', + '**/*.png', '**/*.jpg', '**/*.jpeg', '**/*.gif', + '**/*.svg', '**/*.ico', '**/*.woff', '**/*.woff2', + '**/*.ttf', '**/*.eot'] + + # Initialize OSS client if requested + oss_bucket = None + if use_oss: + oss_bucket = get_oss_client() + if not oss_bucket: + logger.warning("OSS upload failed to initialize, falling back to local mode") + use_oss = False + + output_path = Path(output_dir) + + # Create output directory for local mode or as temp storage for OSS + output_path.mkdir(parents=True, exist_ok=True) + + total_files_processed = 0 + total_files_uploaded = 0 + + for module_name, web_dir in extension_web_dirs.items(): + logger.info(f"Processing {module_name} from {web_dir}") + + module_output_dir = output_path / module_name + module_output_dir.mkdir(parents=True, exist_ok=True) + + files_processed = 0 + files_uploaded = 0 + + # Process files matching patterns + for pattern in file_patterns: + files = glob.glob(os.path.join(glob.escape(web_dir), pattern), recursive=True) + + for file_path in files: + # Calculate relative path from web_dir + rel_path = os.path.relpath(file_path, web_dir) + dest_path = module_output_dir / rel_path + + # Create parent directories + dest_path.parent.mkdir(parents=True, exist_ok=True) + + try: + if use_oss and oss_bucket: + # Upload to OSS + oss_key = f"{OSS_CONFIG['base_path']}{module_name}/{rel_path}" + if upload_to_oss(oss_bucket, file_path, oss_key): + files_uploaded += 1 + logger.debug(f"Uploaded to OSS: {oss_key}") + else: + # Fallback to local copy if OSS upload fails + shutil.copy2(file_path, dest_path) + logger.debug(f"OSS upload failed, copied locally: {rel_path}") + else: + # Local copy + shutil.copy2(file_path, dest_path) + logger.debug(f"Copied locally: {rel_path}") + + files_processed += 1 + + except Exception as e: + logger.error(f"Failed to process {file_path}: {e}") + + if use_oss and oss_bucket: + logger.info(f"Processed {files_processed} files for {module_name} (uploaded to OSS: {files_uploaded})") + else: + logger.info(f"Copied {files_processed} files locally for {module_name}") + + total_files_processed += files_processed + total_files_uploaded += files_uploaded + + return total_files_processed, total_files_uploaded + + +def clean_output_dir(output_dir): + """Remove existing output directory.""" + if os.path.exists(output_dir): + logger.info(f"Cleaning output directory: {output_dir}") + shutil.rmtree(output_dir) + + +def build_static_extensions(output_dir="./static_extensions", clean=False, use_oss=False, verbose=False): + """ + Build static extensions from custom nodes. Can be called programmatically. + Returns (total_files_processed, total_files_uploaded) or None if no extensions found. + """ + if verbose: + logging.getLogger().setLevel(logging.DEBUG) + + output_dir = os.path.abspath(output_dir) + + logger.info("Starting static extensions build...") + if use_oss: + logger.info("OSS upload mode enabled") + else: + logger.info(f"Local output directory: {output_dir}") + + # Clean output directory if requested + if clean: + clean_output_dir(output_dir) + + # Scan custom nodes for web directories + logger.info("Scanning custom nodes...") + extension_web_dirs = scan_custom_node_web_dirs() + + if not extension_web_dirs: + logger.warning("No custom nodes with web directories found!") + return None + + logger.info(f"Found {len(extension_web_dirs)} custom nodes with web directories:") + for module_name, web_dir in extension_web_dirs.items(): + logger.info(f" - {module_name}: {web_dir}") + + # Process web files + logger.info("Processing web files...") + total_files_processed, total_files_uploaded = copy_web_files(extension_web_dirs, output_dir, use_oss=use_oss) + + if use_oss and total_files_uploaded > 0: + logger.info(f"Build completed! Processed {total_files_processed} files, uploaded {total_files_uploaded} to OSS") + else: + logger.info(f"Build completed! Processed {total_files_processed} files to {output_dir}") + + # Generate summary + logger.info("\nStatic extensions structure:") + for module_name in extension_web_dirs.keys(): + if use_oss: + logger.info(f" - {OSS_CONFIG['base_path']}{module_name}/") + else: + module_dir = os.path.join(output_dir, module_name) + if os.path.exists(module_dir): + file_count = sum(len(files) for _, _, files in os.walk(module_dir)) + logger.info(f" - /extensions/{module_name}/ ({file_count} files)") + + return total_files_processed, total_files_uploaded + + +def main(): + parser = argparse.ArgumentParser(description="Build static extensions from ComfyUI custom nodes") + parser.add_argument("--output-dir", default="./static_extensions", + help="Output directory for static extensions (default: ./static_extensions)") + parser.add_argument("--clean", action="store_true", + help="Clean output directory before building") + parser.add_argument("--oss", action="store_true", + help="Upload files to OSS instead of local directory") + parser.add_argument("--verbose", "-v", action="store_true", + help="Enable verbose logging") + + args = parser.parse_args() + + build_static_extensions( + output_dir=args.output_dir, + clean=args.clean, + use_oss=args.oss, + verbose=args.verbose + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/main.py b/main.py index 0cd815d9e..f40d9267c 100644 --- a/main.py +++ b/main.py @@ -359,6 +359,27 @@ def start_comfyui(asyncio_loop=None): init_api_nodes=not args.disable_api_nodes )) hook_breaker_ac10a0.restore_functions() + + # Build static extensions after loading custom nodes + try: + import build_static_extensions + # Check if we should build static extensions (skip if all custom nodes disabled) + if (not args.disable_all_custom_nodes) or len(args.whitelist_custom_nodes) > 0: + logging.info("Building static extensions...") + result = build_static_extensions.build_static_extensions( + output_dir="./static_extensions", + clean=False, # Don't clean by default to avoid deleting existing files + use_oss=True, # Enable OSS upload by default + verbose=False + ) + if result: + total_processed, total_uploaded = result + logging.info(f"Static extensions build completed: {total_processed} files processed, {total_uploaded} uploaded to OSS") + else: + logging.debug("No static extensions found to build") + except Exception as e: + logging.warning(f"Failed to build static extensions: {e}") + logging.debug("Static extensions build error", exc_info=True) cuda_malloc_warning() setup_database()