Merge 1ff364873a into 97f58baaaf

* feat(api-nodes-bytedance): use the virtual portrait library for regular images

Signed-off-by: bigcat88 <bigcat88@icloud.com>

* fix: include shape in image dedup hash

Signed-off-by: bigcat88 <bigcat88@icloud.com>

---------

Signed-off-by: bigcat88 <bigcat88@icloud.com>

CODEOWNERS

@@ -1,2 +1,2 @@
 # Admins
-* @comfyanonymous @kosinkadink @guill
+* @comfyanonymous @kosinkadink @guill @alexisrolland @rattus128

comfy/lora.py

@@ -342,6 +342,12 @@ def model_lora_keys_unet(model, key_map={}):
                 key_map["base_model.model.{}".format(key_lora)] = k  # Official base model loras
                 key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = k  # LyCORIS/LoKR format
 
+    if isinstance(model, comfy.model_base.ErnieImage):
+        for k in sdk:
+            if k.startswith("diffusion_model.") and k.endswith(".weight"):
+                key_lora = k[len("diffusion_model."):-len(".weight")]
+                key_map["transformer.{}".format(key_lora)] = k
+
     return key_map
 
 

comfy/utils.py

@@ -119,6 +119,76 @@ def load_safetensors(ckpt):
     return sd, header.get("__metadata__", {}),
 
 
+def load_safetensors_no_mmap(ckpt, device=None, return_metadata=False):
+    # Load a .safetensors / .sft file without ever mmap'ing it.
+    #
+    # safetensors.safe_open() (and therefore safetensors.torch.load_file) always
+    # mmaps the underlying file in Rust. On systems with unified CPU/GPU memory
+    # like NVIDIA Grace Blackwell / DGX Spark, Apple Silicon, AMD APUs, etc.
+    # this is fatal for large models: the OS page-cache pages backing the mmap
+    # and any subsequent device copy both reside in the same physical memory
+    # pool, doubling peak memory and causing OOM well before the hardware
+    # limit is reached.
+    # See: https://github.com/Comfy-Org/ComfyUI/issues/10896
+    #      https://github.com/safetensors/safetensors/issues/758
+    #      https://github.com/safetensors/safetensors/pull/759
+    #
+    # This is a temporary workaround until upstream safetensors exposes a
+    # public ``mmap=False`` option. Here we parse the safetensors header
+    # ourselves and read each tensor straight from disk into a per-tensor
+    # ``bytearray`` via ``readinto``, then zero-copy-wrap it as a torch tensor
+    # with ``torch.frombuffer``. Peak memory is one model copy (plus, if a
+    # non-CPU device is requested, the bytes of a single tensor in flight
+    # while it is being moved).
+    if device is None:
+        device = torch.device("cpu")
+
+    sd = {}
+    metadata = None
+    with open(ckpt, "rb") as f:
+        header_bytes = f.read(8)
+        if len(header_bytes) != 8:
+            raise ValueError("HeaderTooLarge: file is too small to be a valid safetensors file: {}".format(ckpt))
+        header_size = struct.unpack("<Q", header_bytes)[0]
+        header_data = f.read(header_size)
+        if len(header_data) != header_size:
+            raise ValueError("MetadataIncompleteBuffer: truncated header in {}".format(ckpt))
+        header = json.loads(header_data.decode("utf-8"))
+        data_base_offset = 8 + header_size
+
+        if return_metadata:
+            metadata = header.get("__metadata__", {})
+
+        for name, info in header.items():
+            if name == "__metadata__":
+                continue
+
+            dtype = _TYPES[info["dtype"]]
+            shape = info["shape"]
+            start, end = info["data_offsets"]
+            num_bytes = end - start
+
+            if num_bytes == 0:
+                tensor = torch.empty(shape, dtype=dtype)
+            else:
+                buf = bytearray(num_bytes)
+                f.seek(data_base_offset + start)
+                view = memoryview(buf)
+                offset = 0
+                while offset < num_bytes:
+                    n = f.readinto(view[offset:])
+                    if not n:
+                        raise ValueError("MetadataIncompleteBuffer: unexpected EOF reading tensor {!r} from {}".format(name, ckpt))
+                    offset += n
+                tensor = torch.frombuffer(buf, dtype=dtype).reshape(shape)
+
+            if device.type != "cpu":
+                tensor = tensor.to(device=device)
+            sd[name] = tensor
+
+    return sd, metadata
+
+
 def load_torch_file(ckpt, safe_load=False, device=None, return_metadata=False):
     if device is None:
         device = torch.device("cpu")
@@ -129,14 +199,15 @@ def load_torch_file(ckpt, safe_load=False, device=None, return_metadata=False):
                 sd, metadata = load_safetensors(ckpt)
                 if not return_metadata:
                     metadata = None
+            elif DISABLE_MMAP:
+                sd, metadata = load_safetensors_no_mmap(ckpt, device=device, return_metadata=return_metadata)
+                if not return_metadata:
+                    metadata = None
             else:
                 with safetensors.safe_open(ckpt, framework="pt", device=device.type) as f:
                     sd = {}
                     for k in f.keys():
-                        tensor = f.get_tensor(k)
-                        if DISABLE_MMAP:  # TODO: Not sure if this is the best way to bypass the mmap issues
-                            tensor = tensor.to(device=device, copy=True)
-                        sd[k] = tensor
+                        sd[k] = f.get_tensor(k)
                     if return_metadata:
                         metadata = f.metadata()
         except Exception as e:

comfy_api/latest/_input_impl/video_types.py

@@ -290,7 +290,7 @@ class VideoFromFile(VideoInput):
                                     alphas = []
                                     alpha_channel = True
                                     break
-                            if frame.format.name in ("yuvj420p", "rgb24", "rgba", "pal8"):
+                            if frame.format.name in ("yuvj420p", "yuvj422p", "yuvj444p", "rgb24", "rgba", "pal8"):
                                 process_image_format = lambda a: a.float() / 255.0
                                 if alpha_channel:
                                     image_format = 'rgba'

comfy_api_nodes/apis/bytedance.py

@@ -157,6 +157,11 @@ class SeedanceCreateAssetResponse(BaseModel):
     asset_id: str = Field(...)
 
 
+class SeedanceVirtualLibraryCreateAssetRequest(BaseModel):
+    url: str = Field(..., description="Publicly accessible URL of the image asset to upload.")
+    hash: str = Field(..., description="Dedup key. Re-submitting the same hash returns the existing asset id.")
+
+
 # Dollars per 1K tokens, keyed by (model_id, has_video_input).
 SEEDANCE2_PRICE_PER_1K_TOKENS = {
     ("dreamina-seedance-2-0-260128", False): 0.007,

comfy_api_nodes/nodes_bytedance.py

@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import math
 import re
@@ -20,6 +21,7 @@ from comfy_api_nodes.apis.bytedance import (
     SeedanceCreateAssetResponse,
     SeedanceCreateVisualValidateSessionResponse,
     SeedanceGetVisualValidateSessionResponse,
+    SeedanceVirtualLibraryCreateAssetRequest,
     Seedream4Options,
     Seedream4TaskCreationRequest,
     TaskAudioContent,
@@ -271,6 +273,30 @@ async def _wait_for_asset_active(cls: type[IO.ComfyNode], asset_id: str, group_i
     )
 
 
+async def _seedance_virtual_library_upload_image_asset(
+    cls: type[IO.ComfyNode],
+    image: torch.Tensor,
+    *,
+    wait_label: str = "Uploading image",
+) -> str:
+    """Upload an image into the caller's per-customer Seedance virtual library."""
+    public_url = await upload_image_to_comfyapi(cls, image, wait_label=wait_label)
+    normalized = image.detach().cpu().contiguous().to(torch.float32)
+    digest = hashlib.sha256()
+    digest.update(str(tuple(normalized.shape)).encode("utf-8"))
+    digest.update(b"\0")
+    digest.update(normalized.numpy().tobytes())
+    image_hash = digest.hexdigest()
+    create_resp = await sync_op(
+        cls,
+        ApiEndpoint(path="/proxy/seedance/virtual-library/assets", method="POST"),
+        response_model=SeedanceCreateAssetResponse,
+        data=SeedanceVirtualLibraryCreateAssetRequest(url=public_url, hash=image_hash),
+    )
+    await _wait_for_asset_active(cls, create_resp.asset_id, group_id="virtual-library")
+    return f"asset://{create_resp.asset_id}"
+
+
 def _seedance2_price_extractor(model_id: str, has_video_input: bool):
     """Returns a price_extractor closure for Seedance 2.0 poll_op."""
     rate = SEEDANCE2_PRICE_PER_1K_TOKENS.get((model_id, has_video_input))
@@ -1507,7 +1533,9 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode):
         if first_frame_asset_id:
             first_frame_url = image_assets[first_frame_asset_id]
         else:
-            first_frame_url = await upload_image_to_comfyapi(cls, first_frame, wait_label="Uploading first frame.")
+            first_frame_url = await _seedance_virtual_library_upload_image_asset(
+                cls, first_frame, wait_label="Uploading first frame."
+            )
 
         content: list[TaskTextContent | TaskImageContent] = [
             TaskTextContent(text=model["prompt"]),
@@ -1527,7 +1555,9 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode):
             content.append(
                 TaskImageContent(
                     image_url=TaskImageContentUrl(
-                        url=await upload_image_to_comfyapi(cls, last_frame, wait_label="Uploading last frame.")
+                        url=await _seedance_virtual_library_upload_image_asset(
+                            cls, last_frame, wait_label="Uploading last frame."
+                        )
                     ),
                     role="last_frame",
                 ),
@@ -1805,9 +1835,9 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
             content.append(
                 TaskImageContent(
                     image_url=TaskImageContentUrl(
-                        url=await upload_image_to_comfyapi(
+                        url=await _seedance_virtual_library_upload_image_asset(
                             cls,
-                            image=reference_images[key],
+                            reference_images[key],
                             wait_label=f"Uploading image {i}",
                         ),
                     ),

requirements.txt

@@ -1,5 +1,5 @@
 comfyui-frontend-package==1.42.15
-comfyui-workflow-templates==0.9.63
+comfyui-workflow-templates==0.9.65
 comfyui-embedded-docs==0.4.4
 torch
 torchsde