Merge b0eb165473 into 51bf508a0b

* fix(Video): don't crash on videos with undecodable audio streams

Signed-off-by: bigcat88 <bigcat88@icloud.com>

* Update comfy_api_nodes/util/upload_helpers.py

---------

Signed-off-by: bigcat88 <bigcat88@icloud.com>
Co-authored-by: Alexis Rolland <alexisrolland@hotmail.com>

CLAUDE.md

@@ -1 +0,0 @@
-AGENTS.md

comfy/ldm/joyimage/model.py

@@ -0,0 +1,535 @@
+# https://github.com/jdopensource/JoyAI-Image-Edit (Apache 2.0)
+import math
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import comfy.ops
+import comfy.patcher_extension
+from comfy.ldm.lightricks.model import TimestepEmbedding, Timesteps
+from comfy.ldm.modules.attention import optimized_attention
+
+
+def _apply_rotary_emb(
+    xq: torch.Tensor,
+    xk: torch.Tensor,
+    freqs_cis: Tuple[torch.Tensor, torch.Tensor],
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    ndim = xq.ndim
+    shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(xq.shape)]
+    cos = freqs_cis[0].view(*shape)
+    sin = freqs_cis[1].view(*shape)
+
+    def _rotate_half(x):
+        x_real, x_imag = x.float().reshape(*x.shape[:-1], -1, 2).unbind(-1)
+        return torch.stack([-x_imag, x_real], dim=-1).flatten(3)
+
+    xq_out = (xq.float() * cos + _rotate_half(xq) * sin).type_as(xq)
+    xk_out = (xk.float() * cos + _rotate_half(xk) * sin).type_as(xk)
+    return xq_out, xk_out
+
+
+class JoyImageModulate(nn.Module):
+    def __init__(self, hidden_size: int, factor: int, dtype=None, device=None):
+        super().__init__()
+        self.factor = factor
+        self.modulate_table = nn.Parameter(
+            torch.empty(1, factor, hidden_size, dtype=dtype, device=device)
+        )
+
+    def forward(self, x: torch.Tensor) -> list:
+        if x.ndim != 3:
+            x = x.unsqueeze(1)
+        table = comfy.ops.cast_to_input(self.modulate_table, x)
+        return [o.squeeze(1) for o in (table + x).chunk(self.factor, dim=1)]
+
+
+class JoyImageFeedForward(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        inner_dim: int,
+        dtype=None,
+        device=None,
+        operations=None,
+    ):
+        super().__init__()
+        self.net = nn.ModuleList([
+            _GeluApproximate(dim, inner_dim, dtype=dtype, device=device, operations=operations),
+            nn.Identity(),
+            operations.Linear(inner_dim, dim, bias=True, dtype=dtype, device=device),
+        ])
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        for module in self.net:
+            x = module(x)
+        return x
+
+
+class _GeluApproximate(nn.Module):
+    def __init__(self, dim_in: int, dim_out: int, dtype=None, device=None, operations=None):
+        super().__init__()
+        self.proj = operations.Linear(dim_in, dim_out, bias=True, dtype=dtype, device=device)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return F.gelu(self.proj(x), approximate="tanh")
+
+
+class JoyImageAttention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_attention_heads: int,
+        attention_head_dim: int,
+        eps: float = 1e-6,
+        dtype=None,
+        device=None,
+        operations=None,
+    ):
+        super().__init__()
+        self.num_attention_heads = num_attention_heads
+        inner_dim = num_attention_heads * attention_head_dim
+
+        self.img_attn_qkv = operations.Linear(dim, inner_dim * 3, bias=True, dtype=dtype, device=device)
+        self.img_attn_q_norm = operations.RMSNorm(attention_head_dim, eps=eps, dtype=dtype, device=device)
+        self.img_attn_k_norm = operations.RMSNorm(attention_head_dim, eps=eps, dtype=dtype, device=device)
+        self.img_attn_proj = operations.Linear(inner_dim, dim, bias=True, dtype=dtype, device=device)
+
+        self.txt_attn_qkv = operations.Linear(dim, inner_dim * 3, bias=True, dtype=dtype, device=device)
+        self.txt_attn_q_norm = operations.RMSNorm(attention_head_dim, eps=eps, dtype=dtype, device=device)
+        self.txt_attn_k_norm = operations.RMSNorm(attention_head_dim, eps=eps, dtype=dtype, device=device)
+        self.txt_attn_proj = operations.Linear(inner_dim, dim, bias=True, dtype=dtype, device=device)
+
+    def forward(
+        self,
+        img: torch.Tensor,
+        txt: torch.Tensor,
+        image_rotary_emb: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], Optional[Tuple[torch.Tensor, torch.Tensor]]]],
+        transformer_options={},
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        heads = self.num_attention_heads
+
+        img_q, img_k, img_v = self.img_attn_qkv(img).chunk(3, dim=-1)
+        txt_q, txt_k, txt_v = self.txt_attn_qkv(txt).chunk(3, dim=-1)
+
+        img_q = img_q.unflatten(-1, (heads, -1))
+        img_k = img_k.unflatten(-1, (heads, -1))
+        img_v = img_v.unflatten(-1, (heads, -1))
+        txt_q = txt_q.unflatten(-1, (heads, -1))
+        txt_k = txt_k.unflatten(-1, (heads, -1))
+        txt_v = txt_v.unflatten(-1, (heads, -1))
+
+        img_q = self.img_attn_q_norm(img_q)
+        img_k = self.img_attn_k_norm(img_k)
+        txt_q = self.txt_attn_q_norm(txt_q)
+        txt_k = self.txt_attn_k_norm(txt_k)
+
+        if image_rotary_emb is not None:
+            vis_freqs, txt_freqs = image_rotary_emb
+            if vis_freqs is not None:
+                img_q, img_k = _apply_rotary_emb(img_q, img_k, vis_freqs)
+            if txt_freqs is not None:
+                txt_q, txt_k = _apply_rotary_emb(txt_q, txt_k, txt_freqs)
+
+        joint_q = torch.cat([img_q, txt_q], dim=1)
+        joint_k = torch.cat([img_k, txt_k], dim=1)
+        joint_v = torch.cat([img_v, txt_v], dim=1)
+
+        joint_q = joint_q.flatten(2, 3)
+        joint_k = joint_k.flatten(2, 3)
+        joint_v = joint_v.flatten(2, 3)
+
+        joint_out = optimized_attention(joint_q, joint_k, joint_v, heads=heads, transformer_options=transformer_options)
+        joint_out = joint_out.to(joint_q.dtype)
+
+        seq_img = img.shape[1]
+        img_out = joint_out[:, :seq_img, :]
+        txt_out = joint_out[:, seq_img:, :]
+
+        img_out = self.img_attn_proj(img_out)
+        txt_out = self.txt_attn_proj(txt_out)
+        return img_out, txt_out
+
+
+class JoyImageTransformerBlock(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_attention_heads: int,
+        attention_head_dim: int,
+        mlp_width_ratio: float = 4.0,
+        eps: float = 1e-6,
+        dtype=None,
+        device=None,
+        operations=None,
+    ):
+        super().__init__()
+        self.dim = dim
+        self.num_attention_heads = num_attention_heads
+        self.attention_head_dim = attention_head_dim
+        mlp_hidden_dim = int(dim * mlp_width_ratio)
+
+        self.img_mod = JoyImageModulate(dim, factor=6, dtype=dtype, device=device)
+        self.img_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+        self.img_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+        self.img_mlp = JoyImageFeedForward(dim, inner_dim=mlp_hidden_dim, dtype=dtype, device=device, operations=operations)
+
+        self.txt_mod = JoyImageModulate(dim, factor=6, dtype=dtype, device=device)
+        self.txt_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+        self.txt_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+        self.txt_mlp = JoyImageFeedForward(dim, inner_dim=mlp_hidden_dim, dtype=dtype, device=device, operations=operations)
+
+        self.attn = JoyImageAttention(
+            dim=dim,
+            num_attention_heads=num_attention_heads,
+            attention_head_dim=attention_head_dim,
+            eps=eps,
+            dtype=dtype,
+            device=device,
+            operations=operations,
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        temb: torch.Tensor,
+        image_rotary_emb: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], Optional[Tuple[torch.Tensor, torch.Tensor]]]] = None,
+        transformer_options={},
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        (
+            img_mod1_shift,
+            img_mod1_scale,
+            img_mod1_gate,
+            img_mod2_shift,
+            img_mod2_scale,
+            img_mod2_gate,
+        ) = self.img_mod(temb)
+        (
+            txt_mod1_shift,
+            txt_mod1_scale,
+            txt_mod1_gate,
+            txt_mod2_shift,
+            txt_mod2_scale,
+            txt_mod2_gate,
+        ) = self.txt_mod(temb)
+
+        img_normed = self.img_norm1(hidden_states)
+        txt_normed = self.txt_norm1(encoder_hidden_states)
+        img_modulated = img_normed * (1 + img_mod1_scale.unsqueeze(1)) + img_mod1_shift.unsqueeze(1)
+        txt_modulated = txt_normed * (1 + txt_mod1_scale.unsqueeze(1)) + txt_mod1_shift.unsqueeze(1)
+
+        img_attn, txt_attn = self.attn(img_modulated, txt_modulated, image_rotary_emb, transformer_options=transformer_options)
+
+        hidden_states = hidden_states + img_attn * img_mod1_gate.unsqueeze(1)
+        encoder_hidden_states = encoder_hidden_states + txt_attn * txt_mod1_gate.unsqueeze(1)
+
+        img_ffn_normed = self.img_norm2(hidden_states)
+        txt_ffn_normed = self.txt_norm2(encoder_hidden_states)
+        img_ffn_input = img_ffn_normed * (1 + img_mod2_scale.unsqueeze(1)) + img_mod2_shift.unsqueeze(1)
+        txt_ffn_input = txt_ffn_normed * (1 + txt_mod2_scale.unsqueeze(1)) + txt_mod2_shift.unsqueeze(1)
+        hidden_states = hidden_states + self.img_mlp(img_ffn_input) * img_mod2_gate.unsqueeze(1)
+        encoder_hidden_states = encoder_hidden_states + self.txt_mlp(txt_ffn_input) * txt_mod2_gate.unsqueeze(1)
+
+        return hidden_states, encoder_hidden_states
+
+
+class JoyImageTimeTextImageEmbedding(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        time_freq_dim: int,
+        time_proj_dim: int,
+        text_embed_dim: int,
+        dtype=None,
+        device=None,
+        operations=None,
+    ):
+        super().__init__()
+        self.timesteps_proj = Timesteps(num_channels=time_freq_dim, flip_sin_to_cos=True, downscale_freq_shift=0)
+        self.time_embedder = TimestepEmbedding(
+            in_channels=time_freq_dim,
+            time_embed_dim=dim,
+            dtype=dtype,
+            device=device,
+            operations=operations,
+        )
+        self.act_fn = nn.SiLU()
+        self.time_proj = operations.Linear(dim, time_proj_dim, bias=True, dtype=dtype, device=device)
+        self.text_embedder = _PixArtAlphaTextProjection(
+            text_embed_dim, dim, dtype=dtype, device=device, operations=operations,
+        )
+
+    def forward(self, timestep: torch.Tensor, encoder_hidden_states: torch.Tensor):
+        timestep = self.timesteps_proj(timestep)
+        temb = self.time_embedder(timestep.to(dtype=encoder_hidden_states.dtype)).type_as(encoder_hidden_states)
+        timestep_proj = self.time_proj(self.act_fn(temb))
+        encoder_hidden_states = self.text_embedder(encoder_hidden_states)
+        return temb, timestep_proj, encoder_hidden_states
+
+
+class _PixArtAlphaTextProjection(nn.Module):
+    def __init__(self, in_features: int, hidden_size: int, dtype=None, device=None, operations=None):
+        super().__init__()
+        self.linear_1 = operations.Linear(in_features, hidden_size, bias=True, dtype=dtype, device=device)
+        self.act_1 = nn.GELU(approximate="tanh")
+        self.linear_2 = operations.Linear(hidden_size, hidden_size, bias=True, dtype=dtype, device=device)
+
+    def forward(self, caption: torch.Tensor) -> torch.Tensor:
+        return self.linear_2(self.act_1(self.linear_1(caption)))
+
+
+class JoyImageTransformer3DModel(nn.Module):
+    def __init__(
+        self,
+        patch_size: list = [1, 2, 2],
+        in_channels: int = 16,
+        out_channels: Optional[int] = None,
+        hidden_size: int = 3072,
+        num_attention_heads: int = 24,
+        text_dim: int = 4096,
+        mlp_width_ratio: float = 4.0,
+        num_layers: int = 20,
+        rope_dim_list: list = [16, 56, 56],
+        rope_type: str = "rope",
+        theta: int = 256,
+        image_model=None,
+        dtype=None,
+        device=None,
+        operations=None,
+    ):
+        super().__init__()
+        self.dtype = dtype
+        self.out_channels = out_channels or in_channels
+        self.patch_size = list(patch_size)
+        self.hidden_size = hidden_size
+        self.num_attention_heads = num_attention_heads
+        self.rope_dim_list = list(rope_dim_list)
+        self.rope_type = rope_type
+        self.theta = theta
+
+        if hidden_size % num_attention_heads != 0:
+            raise ValueError(
+                f"hidden_size ({hidden_size}) must be divisible by num_attention_heads ({num_attention_heads})"
+            )
+        attention_head_dim = hidden_size // num_attention_heads
+        if sum(self.rope_dim_list) != attention_head_dim:
+            raise ValueError(
+                f"sum(rope_dim_list) ({sum(self.rope_dim_list)}) must equal head_dim ({attention_head_dim})"
+            )
+
+        self.img_in = operations.Conv3d(
+            in_channels,
+            hidden_size,
+            kernel_size=tuple(self.patch_size),
+            stride=tuple(self.patch_size),
+            dtype=dtype,
+            device=device,
+        )
+
+        self.condition_embedder = JoyImageTimeTextImageEmbedding(
+            dim=hidden_size,
+            time_freq_dim=256,
+            time_proj_dim=hidden_size * 6,
+            text_embed_dim=text_dim,
+            dtype=dtype,
+            device=device,
+            operations=operations,
+        )
+
+        self.double_blocks = nn.ModuleList([
+            JoyImageTransformerBlock(
+                dim=hidden_size,
+                num_attention_heads=num_attention_heads,
+                attention_head_dim=attention_head_dim,
+                mlp_width_ratio=mlp_width_ratio,
+                dtype=dtype,
+                device=device,
+                operations=operations,
+            )
+            for _ in range(num_layers)
+        ])
+
+        self.norm_out = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+        self.proj_out = operations.Linear(
+            hidden_size,
+            self.out_channels * math.prod(self.patch_size),
+            bias=True,
+            dtype=dtype,
+            device=device,
+        )
+
+    def _get_rotary_pos_embed_for_range(
+        self,
+        start: Tuple[int, int, int],
+        stop: Tuple[int, int, int],
+        device=None,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        # 3D RoPE for the patch grid range [start, stop) over (t, h, w). Token order after
+        # reshape(-1) is (t, h, w), matching the img_in Conv3d flatten.
+        head_dim = self.hidden_size // self.num_attention_heads
+        rope_dim_list = self.rope_dim_list
+        if rope_dim_list is None:
+            rope_dim_list = [head_dim // 3 for _ in range(3)]
+        if sum(rope_dim_list) != head_dim:
+            raise ValueError("sum(rope_dim_list) should equal head_dim")
+
+        grids = [torch.arange(start[i], stop[i], dtype=torch.float32, device=device) for i in range(3)]
+        mesh = torch.stack(torch.meshgrid(*grids, indexing="ij"), dim=0)
+
+        cos_parts, sin_parts = [], []
+        for i, dim in enumerate(rope_dim_list):
+            pos = mesh[i].reshape(-1)
+            freqs = 1.0 / (self.theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device)[: (dim // 2)] / dim))
+            angles = torch.outer(pos, freqs)
+            cos_parts.append(angles.cos().repeat_interleave(2, dim=1))
+            sin_parts.append(angles.sin().repeat_interleave(2, dim=1))
+
+        return torch.cat(cos_parts, dim=1), torch.cat(sin_parts, dim=1)
+
+    def get_rotary_pos_embed_for_components(
+        self,
+        component_sizes,
+        device=None,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        # Per-component 3D RoPE. component_sizes is a list of (t, h, w) patch grid sizes in
+        # sequence order [target, ref0, ref1, ...]; h/w restart at 0 for each component while t
+        # continues from the running offset, giving every image its own temporal position band.
+        cos_parts, sin_parts = [], []
+        t_offset = 0
+        for (t, h, w) in component_sizes:
+            cos_emb, sin_emb = self._get_rotary_pos_embed_for_range(
+                start=(t_offset, 0, 0),
+                stop=(t_offset + t, h, w),
+                device=device,
+            )
+            cos_parts.append(cos_emb)
+            sin_parts.append(sin_emb)
+            t_offset += t
+        return torch.cat(cos_parts, dim=0), torch.cat(sin_parts, dim=0)
+
+    def unpatchify(self, x: torch.Tensor, t: int, h: int, w: int) -> torch.Tensor:
+        c = self.out_channels
+        pt, ph, pw = self.patch_size
+        if t * h * w != x.shape[1]:
+            raise ValueError(f"Expected t*h*w ({t * h * w}) to equal x.shape[1] ({x.shape[1]})")
+        x = x.reshape(x.shape[0], t, h, w, pt, ph, pw, c)
+        x = x.permute(0, 7, 1, 4, 2, 5, 3, 6)
+        return x.reshape(x.shape[0], c, t * pt, h * ph, w * pw)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        timestep: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        ref_latents=None,
+        transformer_options={},
+        **kwargs,
+    ) -> torch.Tensor:
+        return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+            self._forward,
+            self,
+            comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+        ).execute(hidden_states, timestep, encoder_hidden_states, ref_latents, transformer_options, **kwargs)
+
+    def _forward(
+        self,
+        hidden_states: torch.Tensor,
+        timestep: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        ref_latents=None,
+        transformer_options={},
+        **kwargs,
+    ) -> torch.Tensor:
+        # The target noise latent and each reference latent are independently patchified by img_in
+        # (Conv3d) and concatenated along the sequence dim, in the order [target, ref0, ref1, ...].
+        # RoPE is built per component so references may differ in resolution. Only the leading
+        # target segment (tt*th*tw tokens) is projected back out; reference tokens are dropped.
+        # A single reference is simply the len(ref_latents) == 1 case.
+        if hidden_states.ndim != 5:
+            raise ValueError(f"JoyImage transformer expects 5D (B,C,T,H,W) hidden_states; got shape {tuple(hidden_states.shape)}")
+
+        _, _, ot, oh, ow = hidden_states.shape
+        pt, ph, pw = self.patch_size
+        if ot % pt != 0 or oh % ph != 0 or ow % pw != 0:
+            raise ValueError(
+                f"JoyImage: target latent spatial/temporal shape {(ot, oh, ow)} must be divisible by patch_size {tuple(self.patch_size)}"
+            )
+        tt = ot // pt
+        th = oh // ph
+        tw = ow // pw
+
+        components = [hidden_states]
+        if ref_latents is not None:
+            for r in ref_latents:
+                if r.ndim != 5:
+                    raise ValueError(f"JoyImage: each reference latent must be 5D (B,C,T,H,W); got shape {tuple(r.shape)}")
+                components.append(r)
+
+        component_sizes = []
+        img_tokens = []
+        for comp in components:
+            _, _, ct, ch, cw = comp.shape
+            if ct % pt != 0 or ch % ph != 0 or cw % pw != 0:
+                raise ValueError(
+                    f"JoyImage: component shape {(ct, ch, cw)} must be divisible by patch_size {tuple(self.patch_size)}"
+                )
+            component_sizes.append((ct // pt, ch // ph, cw // pw))
+            tokens = self.img_in(comp).flatten(2).transpose(1, 2)  # (B, n_i, D)
+            img_tokens.append(tokens)
+
+        img = torch.cat(img_tokens, dim=1)
+
+        _, vec, txt = self.condition_embedder(timestep, encoder_hidden_states)
+        vec = vec.unflatten(1, (6, -1))
+
+        vis_cos, vis_sin = self.get_rotary_pos_embed_for_components(
+            component_sizes,
+            device=hidden_states.device,
+        )
+        vis_freqs = (vis_cos, vis_sin)
+        txt_freqs = None
+        image_rotary_emb = (vis_freqs, txt_freqs)
+
+        patches_replace = transformer_options.get("patches_replace", {})
+        blocks_replace = patches_replace.get("dit", {})
+        transformer_options["total_blocks"] = len(self.double_blocks)
+        transformer_options["block_type"] = "double"
+        for i, block in enumerate(self.double_blocks):
+            transformer_options["block_index"] = i
+            if ("double_block", i) in blocks_replace:
+                def block_wrap(args):
+                    out = {}
+                    out["img"], out["txt"] = block(
+                        hidden_states=args["img"],
+                        encoder_hidden_states=args["txt"],
+                        temb=args["vec"],
+                        image_rotary_emb=args["pe"],
+                        transformer_options=args.get("transformer_options"),
+                    )
+                    return out
+
+                out = blocks_replace[("double_block", i)]({"img": img,
+                                                           "txt": txt,
+                                                           "vec": vec,
+                                                           "pe": image_rotary_emb,
+                                                           "transformer_options": transformer_options},
+                                                          {"original_block": block_wrap})
+                txt = out["txt"]
+                img = out["img"]
+            else:
+                img, txt = block(
+                    hidden_states=img,
+                    encoder_hidden_states=txt,
+                    temb=vec,
+                    image_rotary_emb=image_rotary_emb,
+                    transformer_options=transformer_options,
+                )
+
+        img = self.proj_out(self.norm_out(img))
+        target_tokens = tt * th * tw
+        img = img[:, :target_tokens, :]
+        img = self.unpatchify(img, tt, th, tw)
+        return img

comfy/model_base.py

@@ -57,6 +57,7 @@ import comfy.ldm.ace.model
 import comfy.ldm.omnigen.omnigen2
 import comfy.ldm.boogu.model
 import comfy.ldm.qwen_image.model
+import comfy.ldm.joyimage.model
 import comfy.ldm.ideogram4.model
 import comfy.ldm.krea2.model
 import comfy.ldm.kandinsky5.model
@@ -2264,6 +2265,95 @@ class QwenImage(BaseModel):
             out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
         return out
 
+class JoyImage(BaseModel):
+    # The noise latent and every reference latent are concatenated as a token sequence inside the
+    # transformer. A single-reference edit is just the len(ref_latents) == 1 case. The required CFG
+    # guidance rescale is applied by the JoyImageGuidanceRescale node (comfy_extras/nodes_joyimage.py).
+    def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
+        super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.joyimage.model.JoyImageTransformer3DModel)
+        self.memory_usage_factor_conds = ("ref_latents",)
+
+    def extra_conds(self, **kwargs):
+        out = super().extra_conds(**kwargs)
+        cross_attn = kwargs.get("cross_attn", None)
+        if cross_attn is not None:
+            out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
+        ref_latents = kwargs.get("reference_latents", None)
+        if ref_latents is None or len(ref_latents) == 0:
+            raise ValueError(
+                "JoyImageEdit is an edit model: every conditioning (positive AND negative) must carry "
+                "reference_latents. Wire the same reference image(s) and vae into both the positive and "
+                "negative TextEncodeJoyImageEdit / TextEncodeJoyImageEditPlus nodes. Empty negative "
+                "prompts still need the image(s) and vae."
+            )
+        latents = []
+        for lat in ref_latents:
+            latents.append(self.process_latent_in(lat))
+        out['ref_latents'] = comfy.conds.CONDList(latents)
+        return out
+
+    def extra_conds_shapes(self, **kwargs):
+        out = {}
+        ref_latents = kwargs.get("reference_latents", None)
+        if ref_latents is not None:
+            out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
+        return out
+
+    def _apply_model(self, x, t, c_concat=None, c_crossattn=None, control=None, transformer_options={}, **kwargs):
+        # Pass the noise latent and the reference latents to the transformer, which patchifies each
+        # component and concatenates them along the sequence dim. References may be any resolution.
+        if c_concat is not None:
+            raise ValueError("JoyImage does not support c_concat / noise_concat conditioning")
+        transformer_options = transformer_options.copy()
+        sigma = t
+        xc = self.model_sampling.calculate_input(sigma, x)
+        context = c_crossattn
+        dtype = self.get_dtype_inference()
+        xc = xc.to(dtype)
+        device = xc.device
+        t_in = self.model_sampling.timestep(t).float()
+        if context is not None:
+            context = comfy.model_management.cast_to_device(context, device, dtype)
+
+        extra_conds = {}
+        for o in kwargs:
+            extra = kwargs[o]
+            if hasattr(extra, "dtype"):
+                extra = convert_tensor(extra, dtype, device)
+            elif isinstance(extra, list):
+                ex = []
+                for ext in extra:
+                    ex.append(convert_tensor(ext, dtype, device))
+                extra = ex
+            extra_conds[o] = extra
+
+        ref_latents = extra_conds.pop("ref_latents", None)
+        if ref_latents is None or len(ref_latents) == 0:
+            raise ValueError("JoyImageEdit forward requires ref_latents; got none.")
+
+        if xc.ndim != 5:
+            raise ValueError("JoyImageEdit: noise latent must be 5D (B,C,T,H,W); got shape {}.".format(tuple(xc.shape)))
+
+        refs = []
+        for r in ref_latents:
+            if r.ndim != 5:
+                raise ValueError(
+                    "JoyImageEdit: each reference latent must be 5D (B,C,T,H,W); got shape {}.".format(tuple(r.shape))
+                )
+            refs.append(r)
+
+        if control is not None:
+            raise ValueError("JoyImageEdit: control (ControlNet) is not supported by the transformer.")
+
+        # The transformer's forward signature is (hidden_states, timestep, encoder_hidden_states,
+        # ref_latents, transformer_options); it does not accept control/other extra_conds.
+        if extra_conds:
+            raise ValueError("JoyImageEdit: unexpected extra_conds keys {} reached the transformer.".format(list(extra_conds.keys())))
+
+        noise_pred = self.diffusion_model(xc, t_in, context, ref_latents=refs, transformer_options=transformer_options)
+
+        return self.model_sampling.calculate_denoised(sigma, noise_pred.float(), x)
+
 class Ideogram4(BaseModel):
     def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
         super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.ideogram4.model.Ideogram4Transformer2DModel)

comfy/model_detection.py

@@ -827,6 +827,27 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
             dit_config["default_ref_method"] = "negative_index"
         return dit_config
 
+    # JoyImageEdit: dual-stream double_blocks with img_attn_qkv, a condition_embedder
+    # time_embedder, and a 5D Conv3d img_in (kernel [1,2,2]).
+    if (
+        '{}double_blocks.0.attn.img_attn_qkv.weight'.format(key_prefix) in state_dict_keys
+        and '{}condition_embedder.time_embedder.linear_1.weight'.format(key_prefix) in state_dict_keys
+        and '{}img_in.weight'.format(key_prefix) in state_dict_keys
+        and len(state_dict['{}img_in.weight'.format(key_prefix)].shape) == 5
+    ):
+        img_in = state_dict['{}img_in.weight'.format(key_prefix)]
+        dit_config = {}
+        dit_config["image_model"] = "joyimage"
+        dit_config["in_channels"] = img_in.shape[1]
+        dit_config["hidden_size"] = img_in.shape[0]
+        dit_config["patch_size"] = list(img_in.shape[2:])
+        dit_config["num_layers"] = count_blocks(state_dict_keys, '{}double_blocks.'.format(key_prefix) + '{}.')
+        head_dim = state_dict['{}double_blocks.0.attn.img_attn_q_norm.weight'.format(key_prefix)].shape[0]
+        dit_config["num_attention_heads"] = dit_config["hidden_size"] // head_dim
+        # text_dim from the text-embedder input projection
+        dit_config["text_dim"] = state_dict['{}condition_embedder.text_embedder.linear_1.weight'.format(key_prefix)].shape[1]
+        return dit_config
+
     if '{}embed_image_indicator.weight'.format(key_prefix) in state_dict_keys:  # Ideogram 4
         dit_config = {}
         dit_config["image_model"] = "ideogram4"

comfy/sd.py

@@ -75,6 +75,7 @@ import comfy.text_encoders.gemma4
 import comfy.text_encoders.cogvideo
 import comfy.text_encoders.sa3
 import comfy.text_encoders.gpt_oss
+import comfy.text_encoders.joyimage
 
 import comfy.model_patcher
 import comfy.lora
@@ -1310,6 +1311,7 @@ class CLIPType(Enum):
     IDEOGRAM4 = 30
     BOOGU = 31
     KREA2 = 32
+    JOYIMAGE = 33
 
 
 
@@ -1639,6 +1641,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
                 clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
                 clip_target.clip = comfy.text_encoders.krea2.te(**llama_detect(clip_data))
                 clip_target.tokenizer = comfy.text_encoders.krea2.Krea2Tokenizer
+            elif clip_type == CLIPType.JOYIMAGE and te_model == TEModel.QWEN3VL_8B:  # JoyImageEdit: full Qwen3-VL-8B, edit-conditioning template + drop_idx.
+                clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
+                clip_target.clip = comfy.text_encoders.joyimage.te(**llama_detect(clip_data))
+                clip_target.tokenizer = comfy.text_encoders.joyimage.JoyImageTokenizer
             elif clip_type in (CLIPType.FLUX, CLIPType.FLUX2):  # Flux2 Klein reuses the Qwen3-VL LM (3-layer tap -> 12288); visual unused.
                 klein_model_type = "qwen3_8b" if te_model == TEModel.QWEN3VL_8B else "qwen3_4b"
                 clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type=klein_model_type)

comfy/supported_models.py

@@ -1877,6 +1877,45 @@ class QwenImage(supported_models_base.BASE):
         hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_7b.transformer.".format(pref))
         return supported_models_base.ClipTarget(comfy.text_encoders.qwen_image.QwenImageTokenizer, comfy.text_encoders.qwen_image.te(**hunyuan_detect))
 
+class JoyImage(supported_models_base.BASE):
+    unet_config = {
+        "image_model": "joyimage",
+    }
+
+    # multiplier=1000: the transformer's time embedding is trained on t in [0,1000].
+    # ModelSamplingDiscreteFlow.timestep(sigma)=sigma*multiplier yields that range; the
+    # multiplier cancels in the sigma table, so it only rescales the timestep value.
+    sampling_settings = {
+        "multiplier": 1000,
+        "shift": 1.5,
+    }
+
+    memory_usage_factor = 1.8
+
+    unet_extra_config = {
+        "theta": 10000,
+        "rope_dim_list": [16, 56, 56],
+    }
+
+    latent_format = latent_formats.Wan21  # AutoencoderKLWan: z_dim=16, scale_factor_spatial=8, scale_factor_temporal=4.
+
+    supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+    vae_key_prefix = ["vae."]
+    text_encoder_key_prefix = ["text_encoders."]
+
+    def get_model(self, state_dict, prefix="", device=None):
+        out = model_base.JoyImage(self, device=device)
+        return out
+
+    def clip_target(self, state_dict={}):
+        # Imported lazily so this module stays importable without the text-encoder deps loaded;
+        # the import is only resolved when a checkpoint is actually loaded.
+        import comfy.text_encoders.joyimage
+        pref = self.text_encoder_key_prefix[0]
+        qwen3vl_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl.transformer.".format(pref))
+        return supported_models_base.ClipTarget(comfy.text_encoders.joyimage.JoyImageTokenizer, comfy.text_encoders.joyimage.te(**qwen3vl_detect))
+
 class HunyuanImage21(HunyuanVideo):
     unet_config = {
         "image_model": "hunyuan_video",
@@ -2354,6 +2393,7 @@ models = [
     Omnigen2,
     Boogu,
     QwenImage,
+    JoyImage,
     Ideogram4,
     Krea2,
     Flux2,

comfy/text_encoders/joyimage.py

@@ -0,0 +1,203 @@
+"""JoyImageEdit text encoder: a stock Qwen3-VL-8B multimodal stack feeding the
+JoyImageEdit DiT, built on `comfy.text_encoders.qwen3vl` with the
+JoyImage-specific prompt templates, system-prompt strip, image preprocessing,
+and conditioning-path multimodal handling.
+"""
+
+import torch
+
+from comfy import sd1_clip
+import comfy.text_encoders.qwen_vl
+from comfy.text_encoders.qwen3vl import Qwen3VL, Qwen3VLTokenizer
+
+# Prompt templates for the text-only and image-conditioned modes. The image-conditioned template
+# wraps the user text with one `<|vision_start|><|image_pad|><|vision_end|>` block per reference
+# image (no separator between blocks); `{vision}` is filled with the N concatenated blocks and
+# `{prompt}` with the user text.
+JOYIMAGE_TEMPLATE_TEXT = (
+    "<|im_start|>system\n \\nDescribe the image by detailing the color, shape, size, texture, "
+    "quantity, text, spatial relationships of the objects and background:<|im_end|>\n"
+    "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
+)
+
+JOYIMAGE_TEMPLATE_IMAGE = (
+    "<|im_start|>system\n \\nDescribe the image by detailing the color, shape, size, texture, "
+    "quantity, text, spatial relationships of the objects and background:<|im_end|>\n"
+    "<|im_start|>user\n{vision}{prompt}<|im_end|>\n<|im_start|>assistant\n"
+)
+
+# A single vision block; N copies are concatenated to condition on N reference images.
+JOYIMAGE_VISION_BLOCK = "<|vision_start|><|image_pad|><|vision_end|>"
+
+# Number of leading template tokens (system prompt + the user block's opening
+# `<|im_start|>`) stripped from the encoded output by
+# JoyImageTEModel.encode_token_weights, so the kept sequence begins at the
+# `user` token.
+JOYIMAGE_DROP_IDX = 34
+
+# Special-token ids (vocab shared with Qwen2.5 / Qwen3, vocab_size 151936).
+IMAGE_PAD_TOKEN = 151655
+PAD_TOKEN = 151643
+
+
+class Qwen3VL8B_JoyImage(Qwen3VL):
+    """JoyImage Qwen3-VL-8B encoder.
+
+    Stock `qwen3vl_8b` config (text dims 4096 / 36L / 32H / 8 kv; interleaved
+    3D MRoPE rope_dims=[24,20,20], rope_theta=5e6; vision 1152/4304, depth 27,
+    patch_size 16, deepstack_visual_indexes=[8,16,24]).
+    """
+
+    model_type = "qwen3vl_8b"
+
+    def preprocess_embed(self, embed, device):
+        # Run the vision tower with JoyImage's bicubic+clamp preprocessing and
+        # return ``(merged, {"grid", "deepstack"})``.
+        if embed["type"] == "image":
+            image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(
+                embed["data"], min_pixels=65536, max_pixels=16777216, patch_size=16,
+                image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5],
+                interpolation="bicubic", clamp=True,
+            )
+            merged, deepstack = self.visual(image.to(device, dtype=torch.float32), grid)
+            return merged, {"grid": grid, "deepstack": deepstack}
+        return None, None
+
+    def forward(self, x, attention_mask=None, embeds=None, num_tokens=None,
+                intermediate_output=None, final_layer_norm_intermediate=True,
+                dtype=None, embeds_info=()):
+        # The conditioning path must build the 3D MRoPE position ids for the
+        # image-token block and inject the deepstack visual features.
+        # `build_image_inputs` returns the kwargs the decoder expects:
+        # (position_ids, visual_pos_masks, deepstack).
+        if embeds is not None:
+            position_ids, visual_pos_masks, deepstack = self.build_image_inputs(embeds, embeds_info)
+        else:
+            position_ids, visual_pos_masks, deepstack = None, None, None
+        return self.model(
+            x,
+            attention_mask=attention_mask,
+            embeds=embeds,
+            num_tokens=num_tokens,
+            intermediate_output=intermediate_output,
+            final_layer_norm_intermediate=final_layer_norm_intermediate,
+            dtype=dtype,
+            position_ids=position_ids,
+            deepstack_embeds=deepstack,
+            visual_pos_masks=visual_pos_masks,
+        )
+
+
+class JoyImageTokenizer(Qwen3VLTokenizer):
+    """JoyImageEdit tokenizer.
+
+    ``tokenize_with_weights(text, images=[...])`` selects the image-conditioned
+    template when one or more image tensors are passed, emitting one
+    ``<|vision_start|><|image_pad|><|vision_end|>`` block per image (N blocks
+    for N reference images), otherwise the text-only template. Each
+    ``<|image_pad|>`` token in the formatted prompt is replaced with an
+    embedding marker so `SDClipModel.process_tokens` routes each image through
+    `Qwen3VL8B_JoyImage.preprocess_embed`; ``drop_idx=34`` leading template
+    tokens are stripped downstream by `JoyImageTEModel.encode_token_weights`.
+    No ``<think>`` block is appended.
+    """
+
+    def __init__(self, embedding_directory=None, tokenizer_data={}):
+        super().__init__(
+            embedding_directory=embedding_directory, tokenizer_data=tokenizer_data,
+            model_type="qwen3vl_8b",
+        )
+        self.llama_template = JOYIMAGE_TEMPLATE_TEXT
+        self.llama_template_images = JOYIMAGE_TEMPLATE_IMAGE
+
+    def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None,
+                              images=[], **kwargs):
+        if text.startswith("<|im_start|>"):
+            llama_text = text
+        elif llama_template is not None:
+            llama_text = llama_template.format(text)
+        elif len(images) > 0:
+            # One vision block per reference image.
+            vision = JOYIMAGE_VISION_BLOCK * len(images)
+            llama_text = self.llama_template_images.format(vision=vision, prompt=text)
+        else:
+            llama_text = self.llama_template.format(text)
+
+        # Tokenize the already-rendered template via the grandparent
+        # (SD1Tokenizer); calling `super()` would re-apply the Qwen3VL template.
+        tokens = sd1_clip.SD1Tokenizer.tokenize_with_weights(
+            self, llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs,
+        )
+
+        key_name = next(iter(tokens))
+        embed_count = 0
+        qwen_tokens = tokens[key_name]
+        for r in qwen_tokens:
+            for i in range(len(r)):
+                if r[i][0] == IMAGE_PAD_TOKEN:
+                    if len(images) > embed_count:
+                        r[i] = ({"type": "image", "data": images[embed_count],
+                                 "original_type": "image"},) + r[i][1:]
+                        embed_count += 1
+        if embed_count != len(images):
+            raise ValueError(
+                f"JoyImageTokenizer: prompt had {embed_count} <|image_pad|> placeholders "
+                f"but {len(images)} image(s) were supplied. Either pre-format the prompt "
+                f"with `<|vision_start|><|image_pad|><|vision_end|>` per image or pass an "
+                f"image-free prompt."
+            )
+        return tokens
+
+
+class _JoyImageClipModel(sd1_clip.SDClipModel):
+    """Qwen3-VL multimodal encoder wrapper.
+
+    Conditions on the **pre-final-norm** output of the last decoder layer
+    (``layer="hidden", layer_idx=-1, layer_norm_hidden_state=False``). The
+    post-norm ``last_hidden_state`` differs by ~10x in scale and produces broken
+    DiT outputs, so these flags must not be changed.
+    """
+
+    def __init__(self, device="cpu", layer="hidden", layer_idx=-1, dtype=None,
+                 attention_mask=True, model_options={}):
+        super().__init__(
+            device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={},
+            dtype=dtype, special_tokens={"pad": PAD_TOKEN}, layer_norm_hidden_state=False,
+            model_class=Qwen3VL8B_JoyImage, enable_attention_masks=attention_mask,
+            return_attention_masks=attention_mask, model_options=model_options,
+        )
+
+
+class JoyImageTEModel(sd1_clip.SD1ClipModel):
+    def __init__(self, device="cpu", dtype=None, model_options={}):
+        super().__init__(
+            device=device, dtype=dtype, name="qwen3vl_8b",
+            clip_model=_JoyImageClipModel, model_options=model_options,
+        )
+
+    def encode_token_weights(self, token_weight_pairs):
+        out, pooled, extra = super().encode_token_weights(token_weight_pairs)
+        # Strip the JOYIMAGE_DROP_IDX-token system-prompt prefix from both the
+        # embedding sequence and the attention mask.
+        if out.shape[1] <= JOYIMAGE_DROP_IDX:
+            raise ValueError(
+                f"JoyImageTEModel: encoded sequence length {out.shape[1]} is shorter "
+                f"than drop_idx={JOYIMAGE_DROP_IDX}; the prompt did not include the "
+                f"template prefix."
+            )
+        out = out[:, JOYIMAGE_DROP_IDX:]
+        if "attention_mask" in extra:
+            extra["attention_mask"] = extra["attention_mask"][:, JOYIMAGE_DROP_IDX:]
+        return out, pooled, extra
+
+
+def te(dtype_llama=None, llama_quantization_metadata=None):
+    class JoyImageTEModel_(JoyImageTEModel):
+        def __init__(self, device="cpu", dtype=None, model_options={}):
+            if llama_quantization_metadata is not None:
+                model_options = model_options.copy()
+                model_options["quantization_metadata"] = llama_quantization_metadata
+            if dtype_llama is not None:
+                dtype = dtype_llama
+            super().__init__(device=device, dtype=dtype, model_options=model_options)
+    return JoyImageTEModel_

comfy/text_encoders/qwen_vl.py

@@ -15,6 +15,8 @@ def process_qwen2vl_images(
     merge_size: int = 2,
     image_mean: list = None,
     image_std: list = None,
+    interpolation: str = "bilinear",
+    clamp: bool = False,
 ):
     if image_mean is None:
         image_mean = [0.48145466, 0.4578275, 0.40821073]
@@ -47,9 +49,11 @@ def process_qwen2vl_images(
     img_resized = F.interpolate(
         img.unsqueeze(0),
         size=(h_bar, w_bar),
-        mode='bilinear',
+        mode=interpolation,
         align_corners=False
     ).squeeze(0)
+    if clamp:
+        img_resized = img_resized.clamp(0.0, 1.0)
 
     normalized = img_resized.clone()
     for c in range(3):

comfy_api/latest/_input_impl/video_types.py

@@ -281,11 +281,18 @@ class VideoFromFile(VideoInput):
         video_done = False
         audio_done = True
 
-        if len(container.streams.audio):
-            audio_stream = container.streams.audio[-1]
+        # Use the last decodable audio stream. Streams FFmpeg has no decoder for have no codec context,
+        # and decoding their packets crashes the process. (e.g. APAC spatial-audio track in iPhone)
+        audio_stream = next(
+            (s for s in reversed(container.streams.audio) if s.codec_context is not None),
+            None,
+        )
+        if audio_stream is not None:
             streams += [audio_stream]
             resampler = av.audio.resampler.AudioResampler(format='fltp')
             audio_done = False
+        elif len(container.streams.audio):
+            logging.warning("No decodable audio stream found in video; ignoring audio.")
 
         for packet in container.demux(*streams):
             if video_done and audio_done:
@@ -457,10 +464,13 @@ class VideoFromFile(VideoInput):
                         else:
                             output_container.metadata[key] = json.dumps(value)
 
-                # Add streams to the new container
+                # Add streams to the new container. Streams with no codec context cannot be used as an output template.
                 stream_map = {}
                 for stream in streams:
                     if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)):
+                        if stream.codec_context is None:
+                            logging.warning("Skipping %s stream %d with unsupported codec", stream.type, stream.index)
+                            continue
                         out_stream = output_container.add_stream_from_template(template=stream, opaque=True)
                         stream_map[stream] = out_stream
 

comfy_api_nodes/util/upload_helpers.py

@@ -158,7 +158,14 @@ async def upload_video_to_comfyapi(
 
     # Convert VideoInput to BytesIO using specified container/codec
     video_bytes_io = BytesIO()
-    video.save_to(video_bytes_io, format=container, codec=codec)
+    try:
+        video.save_to(video_bytes_io, format=container, codec=codec)
+    except Exception as e:
+        raise ValueError(
+            f"Could not convert the input video to {container.value.upper()} for upload; "
+            f"the file may be corrupted or use an unsupported codec. "
+            f"Try re-exporting it as MP4 (H.264). Original error: {e}"
+        ) from e
     video_bytes_io.seek(0)
 
     return await upload_file_to_comfyapi(cls, video_bytes_io, filename, upload_mime_type, wait_label)

comfy_extras/nodes_joyimage.py

@@ -0,0 +1,198 @@
+import node_helpers
+import comfy.utils
+import torch
+from typing_extensions import override
+from comfy_api.latest import ComfyExtension, io
+
+
+# fmt: off
+BUCKETS_1024 = [
+    (512, 1792), (512, 1856), (512, 1920), (512, 1984), (512, 2048),
+    (576, 1600), (576, 1664), (576, 1728), (576, 1792),
+    (640, 1472), (640, 1536), (640, 1600),
+    (704, 1344), (704, 1408), (704, 1472),
+    (768, 1216), (768, 1280), (768, 1344),
+    (832, 1152), (832, 1216),
+    (896, 1088), (896, 1152),
+    (960, 1024), (960, 1088),
+    (1024, 960), (1024, 1024),
+    (1088, 896), (1088, 960),
+    (1152, 832), (1152, 896),
+    (1216, 768), (1216, 832),
+    (1280, 768),
+    (1344, 704), (1344, 768),
+    (1408, 704),
+    (1472, 640), (1472, 704),
+    (1536, 640),
+    (1600, 576), (1600, 640),
+    (1664, 576),
+    (1728, 576),
+    (1792, 512), (1792, 576),
+    (1856, 512),
+    (1920, 512),
+    (1984, 512),
+    (2048, 512),
+]
+# fmt: on
+
+
+def _find_best_bucket(height: int, width: int) -> tuple[int, int]:
+    target_ratio = height / width
+    return min(BUCKETS_1024, key=lambda hw: abs(hw[0] / hw[1] - target_ratio))
+
+
+class TextEncodeJoyImageEdit(io.ComfyNode):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="TextEncodeJoyImageEdit",
+            category="advanced/conditioning",
+            inputs=[
+                io.Clip.Input("clip"),
+                io.String.Input("prompt", multiline=True, dynamic_prompts=True),
+                io.Vae.Input("vae"),
+                io.Image.Input("image"),
+            ],
+            outputs=[
+                io.Conditioning.Output(),
+                io.Image.Output(display_name="image"),
+            ],
+        )
+
+    @classmethod
+    def execute(cls, clip, prompt, vae, image) -> io.NodeOutput:
+        samples = image.movedim(-1, 1)
+        src_h, src_w = samples.shape[2], samples.shape[3]
+        bucket_h, bucket_w = _find_best_bucket(src_h, src_w)
+
+        resized = comfy.utils.common_upscale(samples, bucket_w, bucket_h, "bilinear", "center")
+        resized_image = resized.movedim(1, -1)[:, :, :, :3]
+
+        tokens = clip.tokenize(prompt, images=[resized_image])
+        conditioning = clip.encode_from_tokens_scheduled(tokens)
+
+        ref_latent = vae.encode(resized_image)
+        conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": [ref_latent]}, append=True)
+
+        # Return the bucketed image so VAEEncode can build a matching-size init latent;
+        # the bucket size isn't reproducible outside this node.
+        return io.NodeOutput(conditioning, resized_image)
+
+
+class TextEncodeJoyImageEditPlus(io.ComfyNode):
+    """JoyImageEdit multi-image (Plus) text-encode node.
+
+    Accepts 1-6 optional reference images. Each supplied image is
+    bucket-resized independently (same buckets/resize as the single-image
+    node), VAE-encoded, and appended in order to
+    ``conditioning["reference_latents"]`` (image1 → ref0, image2 → ref1, ...).
+    All resized images are passed to the VL tower in one call; the tokenizer
+    emits one ``<|vision_start|><|image_pad|><|vision_end|>`` block per image.
+    """
+
+    MAX_IMAGES = 6
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="TextEncodeJoyImageEditPlus",
+            category="advanced/conditioning",
+            inputs=[
+                io.Clip.Input("clip"),
+                io.String.Input("prompt", multiline=True, dynamic_prompts=True),
+                io.Vae.Input("vae"),
+                io.Image.Input("image1", optional=True),
+                io.Image.Input("image2", optional=True),
+                io.Image.Input("image3", optional=True),
+                io.Image.Input("image4", optional=True),
+                io.Image.Input("image5", optional=True),
+                io.Image.Input("image6", optional=True),
+            ],
+            outputs=[
+                io.Conditioning.Output(),
+                io.Image.Output(display_name="image"),
+            ],
+        )
+
+    @classmethod
+    def execute(cls, clip, prompt, vae, image1=None, image2=None, image3=None,
+                image4=None, image5=None, image6=None) -> io.NodeOutput:
+        images = [image1, image2, image3, image4, image5, image6]
+        supplied = [img for img in images if img is not None]
+        if len(supplied) == 0:
+            raise ValueError(
+                "TextEncodeJoyImageEditPlus requires at least one reference image."
+            )
+
+        resized_images = []
+        ref_latents = []
+        for image in supplied:
+            samples = image.movedim(-1, 1)
+            src_h, src_w = samples.shape[2], samples.shape[3]
+            bucket_h, bucket_w = _find_best_bucket(src_h, src_w)
+
+            resized = comfy.utils.common_upscale(samples, bucket_w, bucket_h, "bilinear", "center")
+            resized_image = resized.movedim(1, -1)[:, :, :, :3]
+            resized_images.append(resized_image)
+            ref_latents.append(vae.encode(resized_image))
+
+        tokens = clip.tokenize(prompt, images=resized_images)
+        conditioning = clip.encode_from_tokens_scheduled(tokens)
+        conditioning = node_helpers.conditioning_set_values(
+            conditioning, {"reference_latents": ref_latents}, append=True,
+        )
+
+        # The last reference sets the target resolution; return it for VAEEncode
+        # since the bucket size isn't reproducible outside this node.
+        return io.NodeOutput(conditioning, resized_images[-1])
+
+
+class JoyImageGuidanceRescale(io.ComfyNode):
+    """CFG combine + per-token L2 norm rescale required by JoyImageEdit.
+
+    Wire this onto the model before sampling: JoyImageEdit's diffusers pipeline
+    rescales the combined noise prediction back to the conditional branch's norm
+    (comb * ||cond|| / ||comb||), the same rescale CFGNorm's pre_cfg branch does.
+    """
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="JoyImageGuidanceRescale",
+            category="model/patch",
+            inputs=[
+                io.Model.Input("model"),
+            ],
+            outputs=[
+                io.Model.Output(),
+            ],
+        )
+
+    @classmethod
+    def execute(cls, model) -> io.NodeOutput:
+        def guidance_rescale(args):
+            cond = args["cond"]
+            uncond = args["uncond"]
+            cond_scale = args["cond_scale"]
+            comb = uncond + cond_scale * (cond - uncond)
+            cond_norm = torch.norm(cond, dim=1, keepdim=True)
+            comb_norm = torch.norm(comb, dim=1, keepdim=True)
+            return comb * (cond_norm / comb_norm.clamp_min(1e-6))
+
+        m = model.clone()
+        m.set_model_sampler_cfg_function(guidance_rescale)
+        return io.NodeOutput(m)
+
+
+class JoyImageExtension(ComfyExtension):
+    @override
+    async def get_node_list(self) -> list[type[io.ComfyNode]]:
+        return [
+            TextEncodeJoyImageEdit,
+            TextEncodeJoyImageEditPlus,
+            JoyImageGuidanceRescale,
+        ]
+
+
+async def comfy_entrypoint() -> JoyImageExtension:
+    return JoyImageExtension()

comfy_extras/nodes_text_overlay.py

@@ -0,0 +1,150 @@
+import numpy as np
+import torch
+from PIL import Image as PILImage, ImageColor, ImageDraw, ImageFont
+from typing_extensions import override
+
+from comfy_api.latest import ComfyExtension, IO
+
+
+class TextOverlay(IO.ComfyNode):
+    @classmethod
+    def define_schema(cls):
+        return IO.Schema(
+            node_id="TextOverlay",
+            display_name="Draw Text Overlay",
+            category="text",
+            description="Draw text overlay on an image or batch of images.",
+            search_aliases=["text", "label", "caption", "subtitle", "watermark", "title", "addlabel", "overlay"],
+            inputs=[
+                IO.Image.Input("images"),
+                IO.String.Input("text", multiline=True, default=""),
+                IO.Float.Input("font_size", default=5.0, min=0.5, max=50.0, step=0.5, tooltip="Font size as a percentage of the image height."),
+                IO.Color.Input("color", default="#ffffff", tooltip="Color of the text."),
+                IO.Combo.Input("position", options=["top", "bottom"], default="top"),
+                IO.Combo.Input("align", options=["left", "center", "right"], default="left"),
+                IO.Boolean.Input("outline", default=True, tooltip="Draw a black outline around the text."),
+            ],
+            outputs=[IO.Image.Output(display_name="images")],
+        )
+
+    @classmethod
+    def execute(cls, images, text, font_size, color, position, align, outline) -> IO.NodeOutput:
+        if text.strip() == "":
+            return IO.NodeOutput(images)
+
+        text = text.replace("\\n", "\n").replace("\\t", "\t")
+
+        text_rgba = cls.parse_color_to_rgba(color)
+        outline_rgba = (0, 0, 0, 255) if outline else (0, 0, 0, 0)
+
+        # Render the overlay once and composite it across all frames in the batch
+        height = images.shape[1]
+        width = images.shape[2]
+        overlay_rgb, overlay_alpha = cls.render_overlay_text(width, height, text, position, align, font_size, text_rgba, outline_rgba)
+        overlay_rgb = overlay_rgb.to(device=images.device, dtype=images.dtype)
+        overlay_alpha = overlay_alpha.to(device=images.device, dtype=images.dtype)
+
+        result = images * (1.0 - overlay_alpha) + overlay_rgb * overlay_alpha
+        return IO.NodeOutput(result)
+
+    @staticmethod
+    def parse_color_to_rgba(color_string):
+        parsed = ImageColor.getrgb(color_string)
+
+        if len(parsed) == 3:
+            return (*parsed, 255)
+
+        return parsed
+
+    @classmethod
+    def render_overlay_text(cls, width, height, text, position, align, font_size, text_rgba, outline_rgba):
+        line_spacing = 1.2
+        margin_percent = 1.0
+        min_font_percent = 2.0
+        min_font_pixels = 10
+        outline_thickness_factor = 0.04
+
+        # Draw onto a transparent layer so the result can be alpha-composited over any frame.
+        layer = PILImage.new("RGBA", (width, height), (0, 0, 0, 0))
+        draw = ImageDraw.Draw(layer)
+
+        margin = int(round(margin_percent / 100.0 * min(width, height)))
+        max_width = max(1, width - 2 * margin)
+        max_height = max(1, height - 2 * margin)
+
+        # Font scales with resolution, then shrinks to fit the height.
+        size = max(1, int(round(font_size / 100.0 * height)))
+        floor = min(size, max(min_font_pixels, int(round(min_font_percent / 100.0 * height))))
+
+        while True:
+            font = ImageFont.load_default(size=size)
+            stroke = max(1, int(round(size * outline_thickness_factor))) if outline_rgba[3] > 0 else 0
+            block = "\n".join(cls.wrap_text(text, font, max_width))
+            # convert line spacing to pixel spacing
+            single = draw.textbbox((0, 0), "Ay", font=font, stroke_width=stroke)
+            double = draw.multiline_textbbox((0, 0), "Ay\nAy", font=font, spacing=0, stroke_width=stroke)
+            natural_advance = (double[3] - double[1]) - (single[3] - single[1])
+            pixel_spacing = int(round(size * line_spacing - natural_advance))
+            box = draw.multiline_textbbox((0, 0), block, font=font, spacing=pixel_spacing, stroke_width=stroke)
+            block_height = box[3] - box[1]
+
+            if block_height <= max_height or size <= floor:
+                break
+
+            size = max(floor, int(size * 0.9))
+
+        anchor_h, x = {"left": ("l", margin), "center": ("m", width / 2), "right": ("r", width - margin)}[align]
+
+        # Offset y so the rendered text sits flush against the margin
+        if position == "bottom":
+            y = height - margin - box[3]
+        else:
+            y = margin - box[1]
+
+        draw.multiline_text((x, y), block, font=font, fill=text_rgba, anchor=anchor_h + "a",
+                            align=align, spacing=pixel_spacing, stroke_width=stroke, stroke_fill=outline_rgba)
+
+        overlay = np.array(layer).astype(np.float32) / 255.0
+        overlay_rgb = torch.from_numpy(overlay[:, :, :3])
+        overlay_alpha = torch.from_numpy(overlay[:, :, 3:4])
+        return overlay_rgb, overlay_alpha
+
+    @staticmethod
+    def wrap_text(text, font, max_width):
+        lines = []
+        for raw_line in text.split("\n"):
+            words = raw_line.split()
+            if not words:
+                lines.append("")
+                continue
+            current = ""
+            # Break the line into words and split words that are too long
+            for word in words:
+                while font.getlength(word) > max_width and len(word) > 1:
+                    cut = 1
+                    while cut < len(word) and font.getlength(word[:cut + 1]) <= max_width:
+                        cut += 1
+                    if current:
+                        lines.append(current)
+                        current = ""
+                    lines.append(word[:cut])
+                    word = word[cut:]
+                candidate = word if not current else current + " " + word
+                if not current or font.getlength(candidate) <= max_width:
+                    current = candidate
+                else:
+                    lines.append(current)
+                    current = word
+            if current:
+                lines.append(current)
+        return lines
+
+
+class TextOverlayExtension(ComfyExtension):
+    @override
+    async def get_node_list(self) -> list[type[IO.ComfyNode]]:
+        return [TextOverlay]
+
+
+async def comfy_entrypoint() -> TextOverlayExtension:
+    return TextOverlayExtension()

nodes.py

@@ -992,7 +992,7 @@ class CLIPLoader:
     @classmethod
     def INPUT_TYPES(s):
         return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),
-                              "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2"], ),
+                              "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage"], ),
                               },
                 "optional": {
                               "device": (["default", "cpu"], {"advanced": True}),
@@ -2460,6 +2460,7 @@ async def init_builtin_extra_nodes():
         "nodes_tcfg.py",
         "nodes_context_windows.py",
         "nodes_qwen.py",
+        "nodes_joyimage.py",
         "nodes_boogu.py",
         "nodes_chroma_radiance.py",
         "nodes_pid.py",
@@ -2478,6 +2479,7 @@ async def init_builtin_extra_nodes():
         "nodes_glsl.py",
         "nodes_lora_debug.py",
         "nodes_textgen.py",
+        "nodes_text_overlay.py",
         "nodes_color.py",
         "nodes_toolkit.py",
         "nodes_replacements.py",