diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index bbdfd4bc2..5c07ba55f 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -540,6 +540,9 @@ class Wan21(LatentFormat): latents_std = self.latents_std.to(latent.device, latent.dtype) return latent * latents_std / self.scale_factor + latents_mean +class LingBotVideo(Wan21): + pass + class Wan22(Wan21): latent_channels = 48 latent_dimensions = 3 diff --git a/comfy/ldm/lingbot_video/model.py b/comfy/ldm/lingbot_video/model.py new file mode 100644 index 000000000..16fda461a --- /dev/null +++ b/comfy/ldm/lingbot_video/model.py @@ -0,0 +1,530 @@ +import math +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from comfy.ldm.flux.math import apply_rope1, rope +from comfy.ldm.flux.layers import timestep_embedding +from comfy.ldm.modules.attention import optimized_attention + + +class LingBotVideoRMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6, device=None, dtype=None, operations=None): + super().__init__() + self.weight = nn.Parameter(torch.empty(dim, device=device, dtype=dtype)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states + + +class LingBotVideoRotaryEmbedding(nn.Module): + def __init__(self, axes_dims: Tuple[int, ...], axes_lens: Tuple[int, ...], theta: float): + super().__init__() + self.axes_dims = tuple(axes_dims) + self.theta = theta + + def forward(self, position_ids: torch.Tensor) -> torch.Tensor: + return torch.cat( + [rope(position_ids[None, :, i], self.axes_dims[i], self.theta) for i in range(len(self.axes_dims))], + dim=-3, + ).squeeze(0) + + +def make_joint_position_ids( + text_len: int, grid_t: int, grid_h: int, grid_w: int, device: torch.device, padded_text_len: Optional[int] = None +) -> torch.Tensor: + """3D positions in [video; text] order. Text t-axis is 1..text_len; video t-axis starts at text_len+1. + + Matches patchify_and_embed: cap start (1,0,0); vision start (cap_len+1,0,0); + freqs ordered with x first and cap second (same order as cat_interleave). + """ + tt = torch.arange(grid_t, device=device, dtype=torch.int32) + (text_len + 1) + hh = torch.arange(grid_h, device=device, dtype=torch.int32) + ww = torch.arange(grid_w, device=device, dtype=torch.int32) + grid = torch.stack(torch.meshgrid(tt, hh, ww, indexing="ij"), dim=-1).flatten(0, 2) + if padded_text_len is None: + padded_text_len = text_len + text_t = torch.arange(padded_text_len, device=device, dtype=torch.int32) + 1 + text_pos = torch.stack( + [text_t, torch.zeros_like(text_t), torch.zeros_like(text_t)], dim=-1 + ) + return torch.cat([grid, text_pos], dim=0) # (Nx + L, 3) + + +class LingBotVideoTimestepEmbedding(nn.Module): + def __init__(self, in_channels, time_embed_dim, bias=True, device=None, dtype=None, operations=None): + super().__init__() + self.linear_1 = operations.Linear(in_channels, time_embed_dim, bias=bias, device=device, dtype=dtype) + self.act = nn.SiLU() + self.linear_2 = operations.Linear(time_embed_dim, time_embed_dim, bias=bias, device=device, dtype=dtype) + + def forward(self, sample): + return self.linear_2(self.act(self.linear_1(sample))) + + +class LingBotVideoTextEmbedder(nn.Module): + """Matches CondProjection: RMSNorm(text_dim, eps=1e-6 fixed) -> Linear-SiLU-Linear.""" + + def __init__(self, text_dim: int, hidden_size: int, device=None, dtype=None, operations=None): + super().__init__() + self.norm = LingBotVideoRMSNorm(text_dim, eps=1e-6, device=device, dtype=dtype, operations=operations) + self.linear_1 = operations.Linear(text_dim, hidden_size, bias=True, device=device, dtype=dtype) + self.linear_2 = operations.Linear(hidden_size, hidden_size, bias=True, device=device, dtype=dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.norm(x) + return self.linear_2(F.silu(self.linear_1(x))) + + +class LingBotVideoAttention(nn.Module): + def __init__(self, hidden_size, num_heads, norm_eps, qkv_bias, out_bias, device=None, dtype=None, operations=None): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + self.to_q = operations.Linear(hidden_size, hidden_size, bias=qkv_bias, device=device, dtype=dtype) + self.to_k = operations.Linear(hidden_size, hidden_size, bias=qkv_bias, device=device, dtype=dtype) + self.to_v = operations.Linear(hidden_size, hidden_size, bias=qkv_bias, device=device, dtype=dtype) + self.norm_q = LingBotVideoRMSNorm(self.head_dim, norm_eps, device=device, dtype=dtype, operations=operations) + self.norm_k = LingBotVideoRMSNorm(self.head_dim, norm_eps, device=device, dtype=dtype, operations=operations) + self.to_out = operations.Linear(hidden_size, hidden_size, bias=out_bias, device=device, dtype=dtype) + + def forward( + self, + x, + rotary_emb, + attention_mask=None, + transformer_options={}, + ): + q = self.to_q(x).unflatten(2, (self.num_heads, self.head_dim)) + k = self.to_k(x).unflatten(2, (self.num_heads, self.head_dim)) + v = self.to_v(x).unflatten(2, (self.num_heads, self.head_dim)) + q = apply_rope1(self.norm_q(q), rotary_emb) + k = apply_rope1(self.norm_k(k), rotary_emb) + out = optimized_attention( + q.transpose(1, 2), + k.transpose(1, 2), + v.transpose(1, 2), + heads=self.num_heads, + mask=attention_mask, + skip_reshape=True, + transformer_options=transformer_options, + ) + return self.to_out(out) + + +class LingBotVideoMLP(nn.Module): + def __init__(self, hidden_size, intermediate_size, device=None, dtype=None, operations=None): + super().__init__() + self.gate_proj = operations.Linear(hidden_size, intermediate_size, bias=False, device=device, dtype=dtype) + self.up_proj = operations.Linear(hidden_size, intermediate_size, bias=False, device=device, dtype=dtype) + self.down_proj = operations.Linear(intermediate_size, hidden_size, bias=False, device=device, dtype=dtype) + + def forward(self, x): + return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x)) + + +class LingBotVideoRouter(nn.Module): + """Matches the TokenChoiceTopKRouter inference path (no capacity/jitter/load stats). + + The asymmetry must be preserved: selection uses the bias-added score, while gating + weights gather the bias-free score. + """ + + def __init__(self, hidden_size, num_experts, top_k, score_func, norm_topk_prob, + n_group, topk_group, route_scale, device=None, dtype=None, operations=None): + super().__init__() + self.num_experts = num_experts + self.top_k = top_k + self.score_func = score_func + self.norm_topk_prob = norm_topk_prob + self.n_group = n_group + self.topk_group = topk_group + self.route_scale = route_scale + self.weight = nn.Parameter(torch.empty(num_experts, hidden_size, device=device, dtype=dtype)) + self.register_buffer("e_score_correction_bias", torch.zeros(num_experts, device=device, dtype=dtype), persistent=True) + + def _group_limited_topk(self, scores_for_choice): + seq_len = scores_for_choice.shape[0] + experts_per_group = self.num_experts // self.n_group + grouped = scores_for_choice.view(seq_len, self.n_group, experts_per_group) + group_scores = grouped.topk(2, dim=-1)[0].sum(dim=-1) + group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1) + score_mask = ( + group_mask.unsqueeze(-1) + .expand(seq_len, self.n_group, experts_per_group) + .reshape(seq_len, -1) + ) + masked = scores_for_choice.masked_fill(~score_mask.bool(), float("-inf")) + return torch.topk(masked, k=self.top_k, dim=-1, sorted=False)[1] + + def forward(self, tokens: torch.Tensor): + logits = F.linear(tokens, self.weight) + if self.score_func == "softmax": + scores = F.softmax(logits, dim=-1) + else: + scores = logits.sigmoid() + scores_for_choice = scores + self.e_score_correction_bias.unsqueeze(0) + if self.n_group is not None and self.n_group > 1: + top_indices = self._group_limited_topk(scores_for_choice) + else: + top_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] + top_scores = scores.gather(1, top_indices) + if self.top_k > 1 and self.norm_topk_prob: + top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-20) + top_scores = top_scores * self.route_scale + return top_indices, top_scores.to(tokens.dtype), logits, scores, scores_for_choice + + +class LingBotVideoGroupedExperts(nn.Module): + """Weight layout matches GroupedExperts: w1 [E,I,H], w2 [E,H,I], w3 [E,I,H]. Eager per-expert compute.""" + + def __init__(self, num_experts, hidden_size, intermediate_size, device=None, dtype=None): + super().__init__() + self.num_experts = num_experts + self.w1 = nn.Parameter(torch.empty(num_experts, intermediate_size, hidden_size, device=device, dtype=dtype)) + self.w2 = nn.Parameter(torch.empty(num_experts, hidden_size, intermediate_size, device=device, dtype=dtype)) + self.w3 = nn.Parameter(torch.empty(num_experts, intermediate_size, hidden_size, device=device, dtype=dtype)) + + +class LingBotVideoSparseMoeBlock(nn.Module): + def __init__(self, hidden_size, intermediate_size, num_experts, top_k, + moe_intermediate_size, score_func, norm_topk_prob, n_group, topk_group, + routed_scaling_factor, n_shared_experts, device=None, dtype=None, operations=None): + super().__init__() + self.hidden_size = hidden_size + self.num_experts = num_experts + self.router = LingBotVideoRouter( + hidden_size, num_experts, top_k, score_func, norm_topk_prob, + n_group, topk_group, routed_scaling_factor, device=device, dtype=dtype, operations=operations, + ) + self.experts = LingBotVideoGroupedExperts(num_experts, hidden_size, moe_intermediate_size, device=device, dtype=dtype) + self.shared_experts = None + if n_shared_experts is not None and n_shared_experts > 0: + self.shared_experts = LingBotVideoMLP( + hidden_size, moe_intermediate_size * n_shared_experts, device=device, dtype=dtype, operations=operations + ) + + def _run_expert(self, expert_idx: int, tokens: torch.Tensor) -> torch.Tensor: + h = F.silu(tokens @ self.experts.w1[expert_idx].transpose(-2, -1)) + h = h * (tokens @ self.experts.w3[expert_idx].transpose(-2, -1)) + return h @ self.experts.w2[expert_idx].transpose(-2, -1) + + def _run_selected_experts( + self, + tokens: torch.Tensor, + top_scores: torch.Tensor, + top_indices: torch.Tensor, + ) -> torch.Tensor: + out = tokens.new_zeros(tokens.shape) + for expert_idx in range(self.num_experts): + selected = top_indices == expert_idx + if not bool(selected.any()): + continue + token_indices, choice_indices = torch.where(selected) + expert_tokens = tokens[token_indices] + expert_output = self._run_expert(expert_idx, expert_tokens) + expert_output = expert_output * top_scores[token_indices, choice_indices].unsqueeze(-1) + out.index_add_(0, token_indices, expert_output) + return out + + def forward(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.Tensor] = None): + # hidden_states: (B, S, H); padding_mask: (B*S,) with 1=valid (only needed when B>1) + B = hidden_states.shape[0] + tokens = hidden_states.view(-1, self.hidden_size) + top_indices, top_scores, logits, scores, scores_for_choice = self.router(tokens) + del logits, scores, scores_for_choice + if padding_mask is not None: + pm = padding_mask.unsqueeze(-1).to(top_scores.dtype) + top_scores = top_scores * pm + top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-9) + top_scores = top_scores * self.router.route_scale + + out = self._run_selected_experts(tokens, top_scores, top_indices) + + out = out.view(B, -1, self.hidden_size) + if self.shared_experts is not None: + shared_output = self.shared_experts(hidden_states) + out = out + shared_output + return out + + +class LingBotVideoBlock(nn.Module): + def __init__( + self, + hidden_size, + num_attention_heads, + intermediate_size, + norm_eps, + qkv_bias, + out_bias, + num_experts, + num_experts_per_tok, + moe_intermediate_size, + decoder_sparse_step, + mlp_only_layers, + n_shared_experts, + score_func, + norm_topk_prob, + n_group, + topk_group, + routed_scaling_factor, + layer_idx: int, + device=None, + dtype=None, + operations=None, + ): + super().__init__() + self.layer_idx = layer_idx + h = hidden_size + self.scale_shift_table = nn.Parameter(torch.empty(1, 6 * h, device=device, dtype=dtype)) + self.norm1 = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations) + self.attn = LingBotVideoAttention( + h, num_attention_heads, norm_eps, qkv_bias, out_bias, device=device, dtype=dtype, operations=operations + ) + self.norm_post_attn = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations) + self.norm2 = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations) + # Sparsity decision matches MoEBlock: mlp_only_layers + decoder_sparse_step + num_experts + if layer_idx not in mlp_only_layers and ( + num_experts > 0 and (layer_idx + 1) % decoder_sparse_step == 0 + ): + self.ffn = LingBotVideoSparseMoeBlock( + h, intermediate_size, num_experts, num_experts_per_tok, + moe_intermediate_size, score_func, norm_topk_prob, + n_group, topk_group, routed_scaling_factor, + n_shared_experts, device=device, dtype=dtype, operations=operations, + ) + else: + self.ffn = LingBotVideoMLP(h, intermediate_size, device=device, dtype=dtype, operations=operations) + self.norm_post_ffn = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations) + + def forward( + self, + x, + temb6, + rotary_emb, + attention_mask=None, + moe_padding_mask=None, + transformer_options={}, + ): + expected_tokens = x.shape[0] * x.shape[1] + if temb6.ndim != 2 or temb6.shape[0] != expected_tokens: + raise ValueError( + "LingBotVideoBlock expects token-level temb6 with shape " + f"(B*S, 6D); got {tuple(temb6.shape)} for hidden states {tuple(x.shape)}." + ) + mod = temb6.view(x.shape[0], x.shape[1], -1) + self.scale_shift_table.unsqueeze(0) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1) + gate_msa, gate_mlp = gate_msa.tanh(), gate_mlp.tanh() + scale_msa, scale_mlp = 1.0 + scale_msa, 1.0 + scale_mlp + + attn_in = self.norm1(x) * scale_msa + shift_msa + attn_out = self.attn( + attn_in, + rotary_emb, + attention_mask, + transformer_options=transformer_options, + ) + x = x + (gate_msa * self.norm_post_attn(attn_out)).to(x.dtype) + + ffn_in = self.norm2(x) * scale_mlp + shift_mlp + if isinstance(self.ffn, LingBotVideoSparseMoeBlock): + ffn_out = self.ffn(ffn_in, padding_mask=moe_padding_mask) + else: + ffn_out = self.ffn(ffn_in) + ffn_normed = self.norm_post_ffn(ffn_out) + x = x + (gate_mlp * ffn_normed).to(x.dtype) + return x + + +class LingBotVideo(nn.Module): + _no_split_modules = ["LingBotVideoBlock"] + + def __init__( + self, + image_model=None, + patch_size: Tuple[int, int, int] = (1, 2, 2), + in_channels: int = 16, + out_channels: int = 16, + hidden_size: int = 2048, + num_attention_heads: int = 16, + depth: int = 24, + intermediate_size: int = 6144, + text_dim: int = 2560, + freq_dim: int = 256, + norm_eps: float = 1e-6, + rope_theta: float = 256.0, + axes_dims: Tuple[int, int, int] = (32, 48, 48), + axes_lens: Tuple[int, int, int] = (8192, 1024, 1024), + qkv_bias: bool = False, + out_bias: bool = True, + patch_embed_bias: bool = True, + timestep_mlp_bias: bool = True, + num_experts: int = 0, + num_experts_per_tok: int = 8, + moe_intermediate_size: int = 512, + decoder_sparse_step: int = 1, + mlp_only_layers: Tuple[int, ...] = (), + n_shared_experts: Optional[int] = None, + score_func: str = "sigmoid", + norm_topk_prob: bool = True, + n_group: Optional[int] = None, + topk_group: Optional[int] = None, + routed_scaling_factor: float = 1.0, + dtype=None, + device=None, + operations=None, + ): + super().__init__() + self.dtype = dtype + self.patch_size = tuple(patch_size) + self.out_channels = out_channels + head_dim = hidden_size // num_attention_heads + assert head_dim == sum(axes_dims), f"head_dim {head_dim} != sum(axes_dims) {sum(axes_dims)}" + mlp_only_layers = tuple(mlp_only_layers) + + self.patch_embedder = operations.Linear( + in_channels * math.prod(patch_size), hidden_size, bias=patch_embed_bias, device=device, dtype=dtype + ) + self.freq_dim = freq_dim + self.time_embedder = LingBotVideoTimestepEmbedding( + freq_dim, hidden_size, bias=timestep_mlp_bias, device=device, dtype=dtype, operations=operations + ) + self.time_modulation = nn.Sequential( + nn.SiLU(), + operations.Linear(hidden_size, 6 * hidden_size, device=device, dtype=dtype), + ) + self.text_embedder = LingBotVideoTextEmbedder(text_dim, hidden_size, device=device, dtype=dtype, operations=operations) + self.rope = LingBotVideoRotaryEmbedding(axes_dims, axes_lens, rope_theta) + self.blocks = nn.ModuleList( + [ + LingBotVideoBlock( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + norm_eps=norm_eps, + qkv_bias=qkv_bias, + out_bias=out_bias, + num_experts=num_experts, + num_experts_per_tok=num_experts_per_tok, + moe_intermediate_size=moe_intermediate_size, + decoder_sparse_step=decoder_sparse_step, + mlp_only_layers=mlp_only_layers, + n_shared_experts=n_shared_experts, + score_func=score_func, + norm_topk_prob=norm_topk_prob, + n_group=n_group, + topk_group=topk_group, + routed_scaling_factor=routed_scaling_factor, + layer_idx=i, + device=device, + dtype=dtype, + operations=operations, + ) + for i in range(depth) + ] + ) + self.norm_out = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=norm_eps, device=device, dtype=dtype) + self.norm_out_modulation = nn.Sequential( + nn.SiLU(), + operations.Linear(hidden_size, 2 * hidden_size, device=device, dtype=dtype), + ) + self.proj_out = operations.Linear(hidden_size, math.prod(patch_size) * out_channels, device=device, dtype=dtype) + + def forward( + self, + hidden_states: torch.Tensor, # (B, C, T, H, W) + timestep: torch.Tensor, # (B,) ∈ [0, 1000](= sigma*1000) + context: torch.Tensor = None, # (B, L, text_dim) + encoder_attention_mask: Optional[torch.Tensor] = None, # (B, L) 1=valid + attention_mask: Optional[torch.Tensor] = None, + transformer_options={}, + **kwargs, + ): + encoder_hidden_states = context + if encoder_hidden_states is None: + raise ValueError("LingBotVideo requires text conditioning.") + if encoder_attention_mask is None: + encoder_attention_mask = attention_mask + B, C, T, H, W = hidden_states.shape + pF, pH, pW = self.patch_size + gt, gh, gw = T // pF, H // pH, W // pW + n_video = gt * gh * gw + L = encoder_hidden_states.shape[1] + device = hidden_states.device + if encoder_attention_mask is not None: + text_lens = encoder_attention_mask.sum(dim=-1).long() + else: + text_lens = torch.full((B,), L, dtype=torch.long, device=device) + text_lens_list = [int(v) for v in text_lens.detach().cpu().tolist()] + + # patchify: token order (f h w), feature order (pf ph pw c) -- matches patchify_and_embed + patch_tokens = hidden_states.reshape(B, C, gt, pF, gh, pH, gw, pW) + patch_tokens = patch_tokens.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape( + B, + n_video, + pF * pH * pW * C, + ) + x = self.patch_embedder(patch_tokens) + text = self.text_embedder(encoder_hidden_states) + joint = torch.cat([x, text], dim=1) # [video; text] + joint_seq_len = joint.shape[1] + + # Per-sample RoPE: video t-axis start = real text length of this sample + 1 + rotary_parts = [ + self.rope(make_joint_position_ids(text_lens_list[i], gt, gh, gw, device, L)) + for i in range(B) + ] + rotary = torch.stack(rotary_parts, dim=0).unsqueeze(2) # (B, S, 1, head_dim/2, 2, 2) + + attention_mask = None + moe_padding_mask = None + has_padding = encoder_attention_mask is not None and bool((text_lens < L).any()) + if has_padding: + key_mask = torch.cat( + [torch.ones(B, n_video, dtype=torch.bool, device=device), + encoder_attention_mask.bool()], + dim=1, + ) + attention_mask = key_mask[:, None, None, :] # (B,1,1,S) → SDPA broadcast + moe_padding_mask = key_mask.reshape(-1) # (B*S,) + + timestep_proj = timestep_embedding(timestep.to(hidden_states.dtype), self.freq_dim, time_factor=1.0) + t_emb = self.time_embedder(timestep_proj) # (B, D) + temb_input = t_emb.unsqueeze(1).expand(B, joint_seq_len, -1) # (B, S, D) + temb6 = self.time_modulation(temb_input.reshape(B * joint_seq_len, -1)) + temb6 = temb6.reshape(B, joint_seq_len, -1) # (B, S, 6D) + + temb6 = temb6.reshape(temb6.shape[0] * temb6.shape[1], -1) + + for block in self.blocks: + joint = block( + joint, + temb6, + rotary, + attention_mask, + moe_padding_mask, + transformer_options=transformer_options, + ) + + final_mod = self.norm_out_modulation(temb_input.reshape(joint.shape[0] * joint.shape[1], -1)) + shift, scale = final_mod.reshape(joint.shape[0], joint.shape[1], -1).chunk(2, dim=-1) + final_hidden = self.norm_out(joint) * (1.0 + scale) + shift + projected = self.proj_out(final_hidden) + x = projected[:, :n_video] + + # unpatchify (matches the rearrange in postprocess) + Cout = self.out_channels + x = x.reshape(B, gt, gh, gw, pF, pH, pW, Cout) + x = x.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(B, Cout, T, H, W) + + return x + + +LingBotVideoTransformer3DModel = LingBotVideo diff --git a/comfy/model_base.py b/comfy/model_base.py index dcfa555dc..0d8444dc2 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -63,6 +63,7 @@ import comfy.ldm.kandinsky5.model import comfy.ldm.anima.model import comfy.ldm.ace.ace_step15 import comfy.ldm.cogvideo.model +import comfy.ldm.lingbot_video.model import comfy.ldm.rt_detr.rtdetr_v4 import comfy.ldm.ernie.model import comfy.ldm.sam3.detector @@ -1376,6 +1377,20 @@ class HunyuanVideoSkyreelsI2V(HunyuanVideo): def scale_latent_inpaint(self, latent_image, **kwargs): return super().scale_latent_inpaint(latent_image=latent_image, **kwargs) +class LingBotVideo(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lingbot_video.model.LingBotVideo) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + cross_attn = kwargs.get("cross_attn", None) + if cross_attn is not None: + out["c_crossattn"] = comfy.conds.CONDRegular(cross_attn) + return out + + def scale_latent_inpaint(self, latent_image, **kwargs): + return latent_image + class CosmosVideo(BaseModel): def __init__(self, model_config, model_type=ModelType.EDM, image_to_video=False, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.cosmos.model.GeneralDIT) diff --git a/comfy/model_detection.py b/comfy/model_detection.py index e53d848c9..c195d1cf4 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -232,6 +232,54 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): dit_config["meanflow_sum"] = False return dit_config + if '{}patch_embedder.weight'.format(key_prefix) in state_dict_keys and '{}text_embedder.norm.weight'.format(key_prefix) in state_dict_keys and '{}blocks.0.attn.to_q.weight'.format(key_prefix) in state_dict_keys: # LingBot Video + dit_config = {} + patch_size = (1, 2, 2) + patch_prod = math.prod(patch_size) + patch_embed = state_dict['{}patch_embedder.weight'.format(key_prefix)] + proj_out = state_dict['{}proj_out.weight'.format(key_prefix)] + hidden_size = patch_embed.shape[0] + dit_config["image_model"] = "lingbot_video" + dit_config["patch_size"] = patch_size + dit_config["in_channels"] = 16 + dit_config["out_channels"] = proj_out.shape[0] // patch_prod + dit_config["hidden_size"] = hidden_size + dit_config["num_attention_heads"] = hidden_size // 128 + dit_config["depth"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.') + dit_config["text_dim"] = state_dict['{}text_embedder.norm.weight'.format(key_prefix)].shape[0] + dit_config["freq_dim"] = 256 + dit_config["qkv_bias"] = '{}blocks.0.attn.to_q.bias'.format(key_prefix) in state_dict_keys + dit_config["out_bias"] = '{}blocks.0.attn.to_out.bias'.format(key_prefix) in state_dict_keys + dit_config["patch_embed_bias"] = '{}patch_embedder.bias'.format(key_prefix) in state_dict_keys + dit_config["timestep_mlp_bias"] = '{}time_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys + + moe_key = '{}blocks.0.ffn.experts.w1'.format(key_prefix) + if moe_key in state_dict_keys: + experts = state_dict[moe_key] + dit_config["num_experts"] = experts.shape[0] + dit_config["moe_intermediate_size"] = experts.shape[1] + if experts.shape[0] == 128: + dit_config["n_group"] = 4 + dit_config["topk_group"] = 2 + dit_config["routed_scaling_factor"] = 2.5 + mlp_only_layers = [] + for i in range(dit_config["depth"]): + if '{}blocks.{}.ffn.gate_proj.weight'.format(key_prefix, i) in state_dict_keys: + mlp_only_layers.append(i) + dit_config["mlp_only_layers"] = tuple(mlp_only_layers) + if len(mlp_only_layers) > 0: + dit_config["intermediate_size"] = state_dict['{}blocks.{}.ffn.gate_proj.weight'.format(key_prefix, mlp_only_layers[0])].shape[0] + if '{}blocks.0.ffn.shared_experts.gate_proj.weight'.format(key_prefix) in state_dict_keys: + shared = state_dict['{}blocks.0.ffn.shared_experts.gate_proj.weight'.format(key_prefix)] + dit_config["n_shared_experts"] = shared.shape[0] // experts.shape[1] + else: + dit_config["num_experts"] = 0 + dit_config["intermediate_size"] = state_dict['{}blocks.0.ffn.gate_proj.weight'.format(key_prefix)].shape[0] + + if metadata is not None and "config" in metadata: + dit_config.update(json.loads(metadata["config"]).get("transformer", {})) + return dit_config + if any_suffix_in(state_dict_keys, key_prefix, 'double_blocks.0.img_attn.norm.key_norm.', ["weight", "scale"]) and ('{}img_in.weight'.format(key_prefix) in state_dict_keys or any_suffix_in(state_dict_keys, key_prefix, 'distilled_guidance_layer.norms.0.', ["weight", "scale"])): #Flux, Chroma or Chroma Radiance (has no img_in.weight) dit_config = {} if '{}double_stream_modulation_img.lin.weight'.format(key_prefix) in state_dict_keys: diff --git a/comfy/sd.py b/comfy/sd.py index 071a3102a..3eb35b242 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -69,6 +69,7 @@ import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image import comfy.text_encoders.qwen35 import comfy.text_encoders.qwen3vl +import comfy.text_encoders.lingbot_video import comfy.text_encoders.boogu import comfy.text_encoders.ernie import comfy.text_encoders.gemma4 @@ -1313,6 +1314,7 @@ class CLIPType(Enum): IDEOGRAM4 = 30 BOOGU = 31 KREA2 = 32 + LINGBOT_VIDEO = 33 @@ -1646,6 +1648,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip klein_model_type = "qwen3_8b" if te_model == TEModel.QWEN3VL_8B else "qwen3_4b" clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type=klein_model_type) clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8B if te_model == TEModel.QWEN3VL_8B else comfy.text_encoders.flux.KleinTokenizer + elif clip_type == CLIPType.LINGBOT_VIDEO and te_model == TEModel.QWEN3VL_4B: + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + clip_target.clip = comfy.text_encoders.lingbot_video.te(**llama_detect(clip_data), model_type="qwen3vl_4b") + clip_target.tokenizer = comfy.text_encoders.lingbot_video.tokenizer(model_type="qwen3vl_4b") else: clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) qwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model] diff --git a/comfy/supported_models.py b/comfy/supported_models.py index afb66e6f3..b8c0ea33b 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -27,6 +27,8 @@ import comfy.text_encoders.z_image import comfy.text_encoders.ideogram4 import comfy.text_encoders.boogu import comfy.text_encoders.krea2 +import comfy.text_encoders.qwen3vl +import comfy.text_encoders.lingbot_video import comfy.text_encoders.anima import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image @@ -1023,6 +1025,40 @@ class HunyuanVideoSkyreelsI2V(HunyuanVideo): out = model_base.HunyuanVideoSkyreelsI2V(self, device=device) return out +class LingBotVideo(supported_models_base.BASE): + unet_config = { + "image_model": "lingbot_video", + } + + sampling_settings = { + "shift": 3.0, + } + + unet_extra_config = {} + latent_format = latent_formats.LingBotVideo + + memory_usage_factor = 1.8 + + supported_inference_dtypes = [torch.bfloat16, torch.float32] + + vae_key_prefix = ["vae."] + text_encoder_key_prefix = ["text_encoders."] + + def __init__(self, unet_config): + super().__init__(unet_config) + self.memory_usage_factor = self.memory_usage_factor * (unet_config.get("hidden_size", 2048) / 2048) + + def get_model(self, state_dict, prefix="", device=None): + return model_base.LingBotVideo(self, device=device) + + def clip_target(self, state_dict={}): + pref = self.text_encoder_key_prefix[0] + qwen3vl_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref)) + if len(qwen3vl_detect) > 0: + qwen3vl_detect["model_type"] = "qwen3vl_4b" + return supported_models_base.ClipTarget(comfy.text_encoders.lingbot_video.tokenizer(model_type="qwen3vl_4b"), comfy.text_encoders.lingbot_video.te(**qwen3vl_detect)) + return None + class CosmosT2V(supported_models_base.BASE): unet_config = { "image_model": "cosmos", @@ -2317,6 +2353,7 @@ models = [ HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, + LingBotVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, diff --git a/comfy/text_encoders/lingbot_video.py b/comfy/text_encoders/lingbot_video.py new file mode 100644 index 000000000..f326c5d3d --- /dev/null +++ b/comfy/text_encoders/lingbot_video.py @@ -0,0 +1,149 @@ +import comfy.text_encoders.qwen3vl +from comfy import sd1_clip + + +PAD_TOKEN = 151643 +CROP_MARKER = {"type": "lingbot_video_crop_start"} + +PROMPT_TEMPLATE = ( + "<|im_start|>system\nGiven a user input that may include a text prompt alone, " + "a text prompt with an image reference, or a text prompt with a video reference " + "or a video reference alone, generate an \"Enhanced prompt\" that provides detailed " + "visual descriptions suitable for video generation. Evaluate the level of detail " + "in the user's input: if it is simple, enrich it by adding specifics about colors, " + "shapes, sizes, textures, lighting, motion dynamics, camera movement, temporal " + "progression, and spatial relationships to create vivid, concrete, and temporally " + "coherent scenes to create vivid and concrete scenes. Please generate only the " + "enhanced description for the prompt below and avoid including any additional " + "commentary or evaluations:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" + "<|im_start|>assistant\n" +) +IMAGE_PROMPT_TEMPLATE = "<|vision_start|><|image_pad|><|vision_end|>" + + +def _marker_tuple(example=None): + if example is not None and len(example) > 2: + return (CROP_MARKER, 1.0, None) + return (CROP_MARKER, 1.0) + + +class LingBotVideoTokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}, model_type="qwen3vl_4b"): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type=model_type) + self._lingbot_crop_start = None + + def lingbot_crop_start(self): + if self._lingbot_crop_start is not None: + return self._lingbot_crop_start + prefix = PROMPT_TEMPLATE.split("{}")[0] + tokens = super().tokenize_with_weights(prefix, thinking=True) + key = next(iter(tokens)) + count = 0 + for token in tokens[key][0]: + token_id = token[0] + if isinstance(token_id, int) and token_id == PAD_TOKEN: + continue + count += 1 + self._lingbot_crop_start = count + return count + + def tokenize_with_weights(self, text, return_word_ids=False, images=[], prevent_empty_text=False, thinking=True, **kwargs): + image = kwargs.get("image", None) + if image is not None and len(images) == 0: + images = [image[i:i + 1] for i in range(image.shape[0])] + + prompt_text = text + if len(images) > 0 and not prompt_text.startswith("<|vision_start|>"): + prompt_text = IMAGE_PROMPT_TEMPLATE + prompt_text + + tokens = super().tokenize_with_weights( + prompt_text, + return_word_ids=return_word_ids, + llama_template=PROMPT_TEMPLATE, + images=images, + prevent_empty_text=prevent_empty_text, + thinking=thinking, + **kwargs, + ) + crop_start = self.lingbot_crop_start() + for key in tokens: + for row in tokens[key]: + example = row[0] if len(row) > 0 else None + row.insert(crop_start, _marker_tuple(example)) + return tokens + + +class LingBotVideoClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}, model_type="qwen3vl_4b"): + super().__init__( + device=device, + layer="last", + layer_idx=None, + dtype=dtype, + attention_mask=attention_mask, + model_options=model_options, + model_type=model_type, + ) + self.return_attention_masks = False + + @staticmethod + def _strip_crop_markers(tokens): + clean_tokens = [] + crop_starts = [] + for row in tokens: + clean_row = [] + crop_start = None + for token in row: + token_value = token[0] if isinstance(token, tuple) else token + if isinstance(token_value, dict) and token_value.get("type") == CROP_MARKER["type"]: + crop_start = len(clean_row) + continue + clean_row.append(token) + clean_tokens.append(clean_row) + crop_starts.append(crop_start) + return clean_tokens, crop_starts + + def forward(self, tokens): + clean_tokens, crop_starts = self._strip_crop_markers(tokens) + out = super().forward(clean_tokens) + crop_starts = [c for c in crop_starts if c is not None] + if len(crop_starts) == 0: + return out + + crop_start = min(crop_starts) + z, pooled_output = out[:2] + z = z[:, crop_start:] + + return z, pooled_output + + +class LingBotVideoTEModel(comfy.text_encoders.qwen3vl.Qwen3VLTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen3vl_4b"): + clip_model = lambda **kw: LingBotVideoClipModel(**kw, model_type=model_type) + sd1_clip.SD1ClipModel.__init__( + self, + device=device, + dtype=dtype, + name=model_type, + clip_model=clip_model, + model_options=model_options, + ) + + +def tokenizer(model_type="qwen3vl_4b"): + class LingBotVideoTokenizer_(LingBotVideoTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type=model_type) + return LingBotVideoTokenizer_ + + +def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen3vl_4b"): + class LingBotVideoTEModel_(LingBotVideoTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type) + return LingBotVideoTEModel_ diff --git a/comfy/text_encoders/qwen3vl.py b/comfy/text_encoders/qwen3vl.py index 2082c42e7..7a329d2d6 100644 --- a/comfy/text_encoders/qwen3vl.py +++ b/comfy/text_encoders/qwen3vl.py @@ -90,6 +90,27 @@ class Qwen3VL(BaseLlama, BaseQwen3, BaseGenerate, torch.nn.Module): deepstack = [torch.cat([deepstack[i], ds[i]], dim=0) for i in range(len(ds))] return position_ids, visual_pos_masks, deepstack + def forward(self, input_ids, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[], **kwargs): + position_ids = kwargs.pop("position_ids", None) + visual_pos_masks = kwargs.pop("visual_pos_masks", None) + deepstack_embeds = kwargs.pop("deepstack_embeds", None) + if embeds is not None and position_ids is None: + position_ids, visual_pos_masks, deepstack_embeds = self.build_image_inputs(embeds, embeds_info) + return self.model( + input_ids, + attention_mask=attention_mask, + embeds=embeds, + num_tokens=num_tokens, + intermediate_output=intermediate_output, + final_layer_norm_intermediate=final_layer_norm_intermediate, + dtype=dtype, + position_ids=position_ids, + embeds_info=embeds_info, + visual_pos_masks=visual_pos_masks, + deepstack_embeds=deepstack_embeds, + **kwargs, + ) + def _make_qwen3vl_model(model_type): class Qwen3VL_(Qwen3VL): diff --git a/comfy_extras/nodes_lingbot_video.py b/comfy_extras/nodes_lingbot_video.py new file mode 100644 index 000000000..2bffe308f --- /dev/null +++ b/comfy_extras/nodes_lingbot_video.py @@ -0,0 +1,167 @@ +import math +import nodes +import numpy as np +import torch +from PIL import Image + +import comfy.latent_formats +import comfy.model_management +import comfy.utils +from comfy_api.latest import ComfyExtension, io +from typing_extensions import override + + +IMAGE_MIN_TOKEN_NUM = 4 +IMAGE_MAX_TOKEN_NUM = 16384 +MAX_RATIO = 200 +SPATIAL_MERGE_SIZE = 2 +VISION_PATCH_SIZE = 16 + + +def _crop_image(image, width, height): + image = image[:1].movedim(-1, 1) + image = comfy.utils.common_upscale(image, width, height, "bilinear", "center") + return image.movedim(1, -1)[:, :, :, :3] + + +def _round_by_factor(number, factor): + return round(number / factor) * factor + + +def _ceil_by_factor(number, factor): + return math.ceil(number / factor) * factor + + +def _floor_by_factor(number, factor): + return math.floor(number / factor) * factor + + +def _smart_resize(height, width, factor, min_pixels=None, max_pixels=None): + max_pixels = max_pixels if max_pixels is not None else IMAGE_MAX_TOKEN_NUM * factor ** 2 + min_pixels = min_pixels if min_pixels is not None else IMAGE_MIN_TOKEN_NUM * factor ** 2 + if max_pixels < min_pixels: + raise ValueError("max_pixels must be greater than or equal to min_pixels.") + if max(height, width) / min(height, width) > MAX_RATIO: + raise ValueError(f"LingBotVideo image aspect ratio must be smaller than {MAX_RATIO}.") + + resized_height = max(factor, _round_by_factor(height, factor)) + resized_width = max(factor, _round_by_factor(width, factor)) + if resized_height * resized_width > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + resized_height = _floor_by_factor(height / beta, factor) + resized_width = _floor_by_factor(width / beta, factor) + elif resized_height * resized_width < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + resized_height = _ceil_by_factor(height * beta, factor) + resized_width = _ceil_by_factor(width * beta, factor) + return resized_height, resized_width + + +def _vlm_image(image): + factor = VISION_PATCH_SIZE * SPATIAL_MERGE_SIZE + height, width = image.shape[1:3] + resized_height, resized_width = _smart_resize(height, width, factor) + array = image[0].detach().cpu().clamp(0, 1).mul(255).byte().numpy() + pil_image = Image.fromarray(array, mode="RGB").resize((resized_width, resized_height)) + array = np.asarray(pil_image).astype(np.float32) / 255.0 + return torch.from_numpy(array).unsqueeze(0) + + +class TextEncodeLingBotVideoI2V(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TextEncodeLingBotVideoI2V", + category="model/conditioning/lingbot_video", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.String.Input("negative_prompt", multiline=True, dynamic_prompts=True, default=""), + io.Int.Input("width", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Image.Input("image", optional=True), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, clip, prompt, negative_prompt, width, height, image=None) -> io.NodeOutput: + if height % 16 != 0 or width % 16 != 0: + raise ValueError(f"LingBotVideo width and height must be multiples of 16, got {width}x{height}.") + if image is None: + images = [] + else: + image = _crop_image(image, width, height) + images = [_vlm_image(image)] + positive = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=images)) + negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt, images=images)) + return io.NodeOutput(positive, negative) + + +class LingBotImageToVideo(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LingBotImageToVideo", + category="model/conditioning/lingbot_video", + inputs=[ + io.Conditioning.Input("positive"), + io.Conditioning.Input("negative"), + io.Vae.Input("vae"), + io.Int.Input("width", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), + io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4), + io.Int.Input("batch_size", default=1, min=1, max=4096), + io.Image.Input("start_image", optional=True), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent"), + ], + ) + + @classmethod + def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None) -> io.NodeOutput: + if length != 1 and (length - 1) % 4 != 0: + raise ValueError(f"LingBotVideo length must be 1 or 4n+1, got {length}.") + if height % 16 != 0 or width % 16 != 0: + raise ValueError(f"LingBotVideo width and height must be multiples of 16, got {width}x{height}.") + latent_frames = ((length - 1) // 4) + 1 + latent = torch.zeros( + [batch_size, 16, latent_frames, height // 8, width // 8], + device=comfy.model_management.intermediate_device(), + ) + out_latent = {"samples": latent} + + if start_image is not None: + start_image = _crop_image(start_image, width, height) + cond_latent = comfy.latent_formats.LingBotVideo().process_in(vae.encode(start_image)) + cond_latent = comfy.utils.resize_to_batch_size(cond_latent, batch_size) + cond_t = min(cond_latent.shape[2], latent.shape[2]) + latent[:, :, :cond_t] = cond_latent[:, :, :cond_t].to(device=latent.device, dtype=latent.dtype) + noise_mask = torch.ones( + (batch_size, 1, latent.shape[2], latent.shape[3], latent.shape[4]), + device=latent.device, + dtype=latent.dtype, + ) + noise_mask[:, :, :cond_t] = 0.0 + out_latent["noise_mask"] = noise_mask + + return io.NodeOutput(positive, negative, out_latent) + + +class LingBotVideoExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + TextEncodeLingBotVideoI2V, + LingBotImageToVideo, + ] + + +async def comfy_entrypoint() -> LingBotVideoExtension: + return LingBotVideoExtension() diff --git a/nodes.py b/nodes.py index e126576fe..9a9a5e03c 100644 --- a/nodes.py +++ b/nodes.py @@ -992,7 +992,7 @@ class CLIPLoader: @classmethod def INPUT_TYPES(s): return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ), - "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2"], ), + "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "lingbot_video"], ), }, "optional": { "device": (["default", "cpu"], {"advanced": True}), @@ -2460,6 +2460,7 @@ async def init_builtin_extra_nodes(): "nodes_tcfg.py", "nodes_context_windows.py", "nodes_qwen.py", + "nodes_lingbot_video.py", "nodes_boogu.py", "nodes_chroma_radiance.py", "nodes_pid.py",