diff --git a/comfy/api_server/routes/internal/internal_routes.py b/comfy/api_server/routes/internal/internal_routes.py index 9b0c4dc61..5d0d4f3f2 100644 --- a/comfy/api_server/routes/internal/internal_routes.py +++ b/comfy/api_server/routes/internal/internal_routes.py @@ -70,6 +70,12 @@ class InternalRoutes: def get_app(self): if self._app is None: + # Manually apply instrumentation to ensure web.Application is instrumented + from opentelemetry.instrumentation.aiohttp_server import AioHttpServerInstrumentor + instrumentor = AioHttpServerInstrumentor() + if not instrumentor.is_instrumented_by_opentelemetry: + instrumentor.instrument() + self._app = web.Application() self.setup_routes() self._app.add_routes(self.routes) diff --git a/comfy/cmd/main_pre.py b/comfy/cmd/main_pre.py index 7c06e4cee..5788f74c9 100644 --- a/comfy/cmd/main_pre.py +++ b/comfy/cmd/main_pre.py @@ -15,7 +15,6 @@ import shutil import warnings import fsspec -from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor from .. import options from ..app import logger @@ -130,6 +129,8 @@ def _create_tracer(): from opentelemetry.processor.baggage import BaggageSpanProcessor, ALLOW_ALL_BAGGAGE_KEYS from opentelemetry.instrumentation.aiohttp_server import AioHttpServerInstrumentor from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor + from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor + from ..tracing_compatibility import ProgressSpanSampler from ..tracing_compatibility import patch_spanbuilder_set_channel @@ -163,6 +164,8 @@ def _create_tracer(): provider.add_span_processor(BaggageSpanProcessor(ALLOW_ALL_BAGGAGE_KEYS)) + trace.set_tracer_provider(provider) + # makes this behave better as a library return trace.get_tracer(args.otel_service_name, tracer_provider=provider) diff --git a/comfy/cmd/server.py b/comfy/cmd/server.py index 7a4be89a8..bc9c7a8e6 100644 --- a/comfy/cmd/server.py +++ b/comfy/cmd/server.py @@ -252,6 +252,14 @@ class PromptServer(ExecutorToClientProgress): middlewares.append(create_origin_only_middleware()) max_upload_size = round(args.max_upload_size * 1024 * 1024) + + # Manually apply instrumentation to ensure web.Application is instrumented + # This works around the import caching bug in opentelemetry-instrumentation-aiohttp-server + from opentelemetry.instrumentation.aiohttp_server import AioHttpServerInstrumentor + instrumentor = AioHttpServerInstrumentor() + if not instrumentor.is_instrumented_by_opentelemetry: + instrumentor.instrument() + self.app: web.Application = web.Application(client_max_size=max_upload_size, handler_args={'max_field_size': 16380}, middlewares=middlewares) diff --git a/comfy/distributed/distributed_prompt_worker.py b/comfy/distributed/distributed_prompt_worker.py index d831ba734..998e2ad2d 100644 --- a/comfy/distributed/distributed_prompt_worker.py +++ b/comfy/distributed/distributed_prompt_worker.py @@ -53,6 +53,12 @@ class DistributedPromptWorker: return web.Response(text="UNHEALTHY: RabbitMQ connection is not healthy", status=503) async def _start_health_check_server(self): + # Manually apply instrumentation to ensure web.Application is instrumented + from opentelemetry.instrumentation.aiohttp_server import AioHttpServerInstrumentor + instrumentor = AioHttpServerInstrumentor() + if not instrumentor.is_instrumented_by_opentelemetry: + instrumentor.instrument() + app = web.Application() app.router.add_get('/health', self._health_check) diff --git a/pyproject.toml b/pyproject.toml index f6e6f375c..174afa215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,8 +33,8 @@ dependencies = [ "peft>=0.10.0", "torchinfo", "safetensors>=0.4.2", - "aiohttp>=3.11.8", - "yarl>=1.9.4", + "aiohttp>=3.13.2", + "yarl>=1.22.0", "accelerate>=0.25.0", "pyyaml>=6.0", "scikit-image>=0.20.0", @@ -119,7 +119,8 @@ dependencies = [ "requests_cache", "universal_pathlib", # yanked propcache is omitted - "propcache!=0.4.0", + # we want this upgraded + "propcache>=0.4.1", ] [build-system]