Lingbot WIP.

Probably needs cleanup.

comfy/latent_formats.py

@@ -540,6 +540,9 @@ class Wan21(LatentFormat):
         latents_std = self.latents_std.to(latent.device, latent.dtype)
         return latent * latents_std / self.scale_factor + latents_mean
 
+class LingBotVideo(Wan21):
+    pass
+
 class Wan22(Wan21):
     latent_channels = 48
     latent_dimensions = 3

comfy/ldm/lingbot_video/model.py

@@ -0,0 +1,530 @@
+import math
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from comfy.ldm.flux.math import apply_rope1, rope
+from comfy.ldm.flux.layers import timestep_embedding
+from comfy.ldm.modules.attention import optimized_attention
+
+
+class LingBotVideoRMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6, device=None, dtype=None, operations=None):
+        super().__init__()
+        self.weight = nn.Parameter(torch.empty(dim, device=device, dtype=dtype))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states
+
+
+class LingBotVideoRotaryEmbedding(nn.Module):
+    def __init__(self, axes_dims: Tuple[int, ...], axes_lens: Tuple[int, ...], theta: float):
+        super().__init__()
+        self.axes_dims = tuple(axes_dims)
+        self.theta = theta
+
+    def forward(self, position_ids: torch.Tensor) -> torch.Tensor:
+        return torch.cat(
+            [rope(position_ids[None, :, i], self.axes_dims[i], self.theta) for i in range(len(self.axes_dims))],
+            dim=-3,
+        ).squeeze(0)
+
+
+def make_joint_position_ids(
+    text_len: int, grid_t: int, grid_h: int, grid_w: int, device: torch.device, padded_text_len: Optional[int] = None
+) -> torch.Tensor:
+    """3D positions in [video; text] order. Text t-axis is 1..text_len; video t-axis starts at text_len+1.
+
+    Matches patchify_and_embed: cap start (1,0,0); vision start (cap_len+1,0,0);
+    freqs ordered with x first and cap second (same order as cat_interleave).
+    """
+    tt = torch.arange(grid_t, device=device, dtype=torch.int32) + (text_len + 1)
+    hh = torch.arange(grid_h, device=device, dtype=torch.int32)
+    ww = torch.arange(grid_w, device=device, dtype=torch.int32)
+    grid = torch.stack(torch.meshgrid(tt, hh, ww, indexing="ij"), dim=-1).flatten(0, 2)
+    if padded_text_len is None:
+        padded_text_len = text_len
+    text_t = torch.arange(padded_text_len, device=device, dtype=torch.int32) + 1
+    text_pos = torch.stack(
+        [text_t, torch.zeros_like(text_t), torch.zeros_like(text_t)], dim=-1
+    )
+    return torch.cat([grid, text_pos], dim=0)  # (Nx + L, 3)
+
+
+class LingBotVideoTimestepEmbedding(nn.Module):
+    def __init__(self, in_channels, time_embed_dim, bias=True, device=None, dtype=None, operations=None):
+        super().__init__()
+        self.linear_1 = operations.Linear(in_channels, time_embed_dim, bias=bias, device=device, dtype=dtype)
+        self.act = nn.SiLU()
+        self.linear_2 = operations.Linear(time_embed_dim, time_embed_dim, bias=bias, device=device, dtype=dtype)
+
+    def forward(self, sample):
+        return self.linear_2(self.act(self.linear_1(sample)))
+
+
+class LingBotVideoTextEmbedder(nn.Module):
+    """Matches CondProjection: RMSNorm(text_dim, eps=1e-6 fixed) -> Linear-SiLU-Linear."""
+
+    def __init__(self, text_dim: int, hidden_size: int, device=None, dtype=None, operations=None):
+        super().__init__()
+        self.norm = LingBotVideoRMSNorm(text_dim, eps=1e-6, device=device, dtype=dtype, operations=operations)
+        self.linear_1 = operations.Linear(text_dim, hidden_size, bias=True, device=device, dtype=dtype)
+        self.linear_2 = operations.Linear(hidden_size, hidden_size, bias=True, device=device, dtype=dtype)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.norm(x)
+        return self.linear_2(F.silu(self.linear_1(x)))
+
+
+class LingBotVideoAttention(nn.Module):
+    def __init__(self, hidden_size, num_heads, norm_eps, qkv_bias, out_bias, device=None, dtype=None, operations=None):
+        super().__init__()
+        self.num_heads = num_heads
+        self.head_dim = hidden_size // num_heads
+        self.to_q = operations.Linear(hidden_size, hidden_size, bias=qkv_bias, device=device, dtype=dtype)
+        self.to_k = operations.Linear(hidden_size, hidden_size, bias=qkv_bias, device=device, dtype=dtype)
+        self.to_v = operations.Linear(hidden_size, hidden_size, bias=qkv_bias, device=device, dtype=dtype)
+        self.norm_q = LingBotVideoRMSNorm(self.head_dim, norm_eps, device=device, dtype=dtype, operations=operations)
+        self.norm_k = LingBotVideoRMSNorm(self.head_dim, norm_eps, device=device, dtype=dtype, operations=operations)
+        self.to_out = operations.Linear(hidden_size, hidden_size, bias=out_bias, device=device, dtype=dtype)
+
+    def forward(
+        self,
+        x,
+        rotary_emb,
+        attention_mask=None,
+        transformer_options={},
+    ):
+        q = self.to_q(x).unflatten(2, (self.num_heads, self.head_dim))
+        k = self.to_k(x).unflatten(2, (self.num_heads, self.head_dim))
+        v = self.to_v(x).unflatten(2, (self.num_heads, self.head_dim))
+        q = apply_rope1(self.norm_q(q), rotary_emb)
+        k = apply_rope1(self.norm_k(k), rotary_emb)
+        out = optimized_attention(
+            q.transpose(1, 2),
+            k.transpose(1, 2),
+            v.transpose(1, 2),
+            heads=self.num_heads,
+            mask=attention_mask,
+            skip_reshape=True,
+            transformer_options=transformer_options,
+        )
+        return self.to_out(out)
+
+
+class LingBotVideoMLP(nn.Module):
+    def __init__(self, hidden_size, intermediate_size, device=None, dtype=None, operations=None):
+        super().__init__()
+        self.gate_proj = operations.Linear(hidden_size, intermediate_size, bias=False, device=device, dtype=dtype)
+        self.up_proj = operations.Linear(hidden_size, intermediate_size, bias=False, device=device, dtype=dtype)
+        self.down_proj = operations.Linear(intermediate_size, hidden_size, bias=False, device=device, dtype=dtype)
+
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class LingBotVideoRouter(nn.Module):
+    """Matches the TokenChoiceTopKRouter inference path (no capacity/jitter/load stats).
+
+    The asymmetry must be preserved: selection uses the bias-added score, while gating
+    weights gather the bias-free score.
+    """
+
+    def __init__(self, hidden_size, num_experts, top_k, score_func, norm_topk_prob,
+                 n_group, topk_group, route_scale, device=None, dtype=None, operations=None):
+        super().__init__()
+        self.num_experts = num_experts
+        self.top_k = top_k
+        self.score_func = score_func
+        self.norm_topk_prob = norm_topk_prob
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.route_scale = route_scale
+        self.weight = nn.Parameter(torch.empty(num_experts, hidden_size, device=device, dtype=dtype))
+        self.register_buffer("e_score_correction_bias", torch.zeros(num_experts, device=device, dtype=dtype), persistent=True)
+
+    def _group_limited_topk(self, scores_for_choice):
+        seq_len = scores_for_choice.shape[0]
+        experts_per_group = self.num_experts // self.n_group
+        grouped = scores_for_choice.view(seq_len, self.n_group, experts_per_group)
+        group_scores = grouped.topk(2, dim=-1)[0].sum(dim=-1)
+        group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
+        group_mask = torch.zeros_like(group_scores)
+        group_mask.scatter_(1, group_idx, 1)
+        score_mask = (
+            group_mask.unsqueeze(-1)
+            .expand(seq_len, self.n_group, experts_per_group)
+            .reshape(seq_len, -1)
+        )
+        masked = scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))
+        return torch.topk(masked, k=self.top_k, dim=-1, sorted=False)[1]
+
+    def forward(self, tokens: torch.Tensor):
+        logits = F.linear(tokens, self.weight)
+        if self.score_func == "softmax":
+            scores = F.softmax(logits, dim=-1)
+        else:
+            scores = logits.sigmoid()
+        scores_for_choice = scores + self.e_score_correction_bias.unsqueeze(0)
+        if self.n_group is not None and self.n_group > 1:
+            top_indices = self._group_limited_topk(scores_for_choice)
+        else:
+            top_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1]
+        top_scores = scores.gather(1, top_indices)
+        if self.top_k > 1 and self.norm_topk_prob:
+            top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-20)
+        top_scores = top_scores * self.route_scale
+        return top_indices, top_scores.to(tokens.dtype), logits, scores, scores_for_choice
+
+
+class LingBotVideoGroupedExperts(nn.Module):
+    """Weight layout matches GroupedExperts: w1 [E,I,H], w2 [E,H,I], w3 [E,I,H]. Eager per-expert compute."""
+
+    def __init__(self, num_experts, hidden_size, intermediate_size, device=None, dtype=None):
+        super().__init__()
+        self.num_experts = num_experts
+        self.w1 = nn.Parameter(torch.empty(num_experts, intermediate_size, hidden_size, device=device, dtype=dtype))
+        self.w2 = nn.Parameter(torch.empty(num_experts, hidden_size, intermediate_size, device=device, dtype=dtype))
+        self.w3 = nn.Parameter(torch.empty(num_experts, intermediate_size, hidden_size, device=device, dtype=dtype))
+
+
+class LingBotVideoSparseMoeBlock(nn.Module):
+    def __init__(self, hidden_size, intermediate_size, num_experts, top_k,
+                 moe_intermediate_size, score_func, norm_topk_prob, n_group, topk_group,
+                 routed_scaling_factor, n_shared_experts, device=None, dtype=None, operations=None):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.num_experts = num_experts
+        self.router = LingBotVideoRouter(
+            hidden_size, num_experts, top_k, score_func, norm_topk_prob,
+            n_group, topk_group, routed_scaling_factor, device=device, dtype=dtype, operations=operations,
+        )
+        self.experts = LingBotVideoGroupedExperts(num_experts, hidden_size, moe_intermediate_size, device=device, dtype=dtype)
+        self.shared_experts = None
+        if n_shared_experts is not None and n_shared_experts > 0:
+            self.shared_experts = LingBotVideoMLP(
+                hidden_size, moe_intermediate_size * n_shared_experts, device=device, dtype=dtype, operations=operations
+            )
+
+    def _run_expert(self, expert_idx: int, tokens: torch.Tensor) -> torch.Tensor:
+        h = F.silu(tokens @ self.experts.w1[expert_idx].transpose(-2, -1))
+        h = h * (tokens @ self.experts.w3[expert_idx].transpose(-2, -1))
+        return h @ self.experts.w2[expert_idx].transpose(-2, -1)
+
+    def _run_selected_experts(
+        self,
+        tokens: torch.Tensor,
+        top_scores: torch.Tensor,
+        top_indices: torch.Tensor,
+    ) -> torch.Tensor:
+        out = tokens.new_zeros(tokens.shape)
+        for expert_idx in range(self.num_experts):
+            selected = top_indices == expert_idx
+            if not bool(selected.any()):
+                continue
+            token_indices, choice_indices = torch.where(selected)
+            expert_tokens = tokens[token_indices]
+            expert_output = self._run_expert(expert_idx, expert_tokens)
+            expert_output = expert_output * top_scores[token_indices, choice_indices].unsqueeze(-1)
+            out.index_add_(0, token_indices, expert_output)
+        return out
+
+    def forward(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.Tensor] = None):
+        # hidden_states: (B, S, H); padding_mask: (B*S,) with 1=valid (only needed when B>1)
+        B = hidden_states.shape[0]
+        tokens = hidden_states.view(-1, self.hidden_size)
+        top_indices, top_scores, logits, scores, scores_for_choice = self.router(tokens)
+        del logits, scores, scores_for_choice
+        if padding_mask is not None:
+            pm = padding_mask.unsqueeze(-1).to(top_scores.dtype)
+            top_scores = top_scores * pm
+            top_scores = top_scores / (top_scores.sum(dim=-1, keepdim=True) + 1e-9)
+            top_scores = top_scores * self.router.route_scale
+
+        out = self._run_selected_experts(tokens, top_scores, top_indices)
+
+        out = out.view(B, -1, self.hidden_size)
+        if self.shared_experts is not None:
+            shared_output = self.shared_experts(hidden_states)
+            out = out + shared_output
+        return out
+
+
+class LingBotVideoBlock(nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        num_attention_heads,
+        intermediate_size,
+        norm_eps,
+        qkv_bias,
+        out_bias,
+        num_experts,
+        num_experts_per_tok,
+        moe_intermediate_size,
+        decoder_sparse_step,
+        mlp_only_layers,
+        n_shared_experts,
+        score_func,
+        norm_topk_prob,
+        n_group,
+        topk_group,
+        routed_scaling_factor,
+        layer_idx: int,
+        device=None,
+        dtype=None,
+        operations=None,
+    ):
+        super().__init__()
+        self.layer_idx = layer_idx
+        h = hidden_size
+        self.scale_shift_table = nn.Parameter(torch.empty(1, 6 * h, device=device, dtype=dtype))
+        self.norm1 = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations)
+        self.attn = LingBotVideoAttention(
+            h, num_attention_heads, norm_eps, qkv_bias, out_bias, device=device, dtype=dtype, operations=operations
+        )
+        self.norm_post_attn = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations)
+        self.norm2 = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations)
+        # Sparsity decision matches MoEBlock: mlp_only_layers + decoder_sparse_step + num_experts
+        if layer_idx not in mlp_only_layers and (
+            num_experts > 0 and (layer_idx + 1) % decoder_sparse_step == 0
+        ):
+            self.ffn = LingBotVideoSparseMoeBlock(
+                h, intermediate_size, num_experts, num_experts_per_tok,
+                moe_intermediate_size, score_func, norm_topk_prob,
+                n_group, topk_group, routed_scaling_factor,
+                n_shared_experts, device=device, dtype=dtype, operations=operations,
+            )
+        else:
+            self.ffn = LingBotVideoMLP(h, intermediate_size, device=device, dtype=dtype, operations=operations)
+        self.norm_post_ffn = LingBotVideoRMSNorm(h, norm_eps, device=device, dtype=dtype, operations=operations)
+
+    def forward(
+        self,
+        x,
+        temb6,
+        rotary_emb,
+        attention_mask=None,
+        moe_padding_mask=None,
+        transformer_options={},
+    ):
+        expected_tokens = x.shape[0] * x.shape[1]
+        if temb6.ndim != 2 or temb6.shape[0] != expected_tokens:
+            raise ValueError(
+                "LingBotVideoBlock expects token-level temb6 with shape "
+                f"(B*S, 6D); got {tuple(temb6.shape)} for hidden states {tuple(x.shape)}."
+            )
+        mod = temb6.view(x.shape[0], x.shape[1], -1) + self.scale_shift_table.unsqueeze(0)
+        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1)
+        gate_msa, gate_mlp = gate_msa.tanh(), gate_mlp.tanh()
+        scale_msa, scale_mlp = 1.0 + scale_msa, 1.0 + scale_mlp
+
+        attn_in = self.norm1(x) * scale_msa + shift_msa
+        attn_out = self.attn(
+            attn_in,
+            rotary_emb,
+            attention_mask,
+            transformer_options=transformer_options,
+        )
+        x = x + (gate_msa * self.norm_post_attn(attn_out)).to(x.dtype)
+
+        ffn_in = self.norm2(x) * scale_mlp + shift_mlp
+        if isinstance(self.ffn, LingBotVideoSparseMoeBlock):
+            ffn_out = self.ffn(ffn_in, padding_mask=moe_padding_mask)
+        else:
+            ffn_out = self.ffn(ffn_in)
+        ffn_normed = self.norm_post_ffn(ffn_out)
+        x = x + (gate_mlp * ffn_normed).to(x.dtype)
+        return x
+
+
+class LingBotVideo(nn.Module):
+    _no_split_modules = ["LingBotVideoBlock"]
+
+    def __init__(
+        self,
+        image_model=None,
+        patch_size: Tuple[int, int, int] = (1, 2, 2),
+        in_channels: int = 16,
+        out_channels: int = 16,
+        hidden_size: int = 2048,
+        num_attention_heads: int = 16,
+        depth: int = 24,
+        intermediate_size: int = 6144,
+        text_dim: int = 2560,
+        freq_dim: int = 256,
+        norm_eps: float = 1e-6,
+        rope_theta: float = 256.0,
+        axes_dims: Tuple[int, int, int] = (32, 48, 48),
+        axes_lens: Tuple[int, int, int] = (8192, 1024, 1024),
+        qkv_bias: bool = False,
+        out_bias: bool = True,
+        patch_embed_bias: bool = True,
+        timestep_mlp_bias: bool = True,
+        num_experts: int = 0,
+        num_experts_per_tok: int = 8,
+        moe_intermediate_size: int = 512,
+        decoder_sparse_step: int = 1,
+        mlp_only_layers: Tuple[int, ...] = (),
+        n_shared_experts: Optional[int] = None,
+        score_func: str = "sigmoid",
+        norm_topk_prob: bool = True,
+        n_group: Optional[int] = None,
+        topk_group: Optional[int] = None,
+        routed_scaling_factor: float = 1.0,
+        dtype=None,
+        device=None,
+        operations=None,
+    ):
+        super().__init__()
+        self.dtype = dtype
+        self.patch_size = tuple(patch_size)
+        self.out_channels = out_channels
+        head_dim = hidden_size // num_attention_heads
+        assert head_dim == sum(axes_dims), f"head_dim {head_dim} != sum(axes_dims) {sum(axes_dims)}"
+        mlp_only_layers = tuple(mlp_only_layers)
+
+        self.patch_embedder = operations.Linear(
+            in_channels * math.prod(patch_size), hidden_size, bias=patch_embed_bias, device=device, dtype=dtype
+        )
+        self.freq_dim = freq_dim
+        self.time_embedder = LingBotVideoTimestepEmbedding(
+            freq_dim, hidden_size, bias=timestep_mlp_bias, device=device, dtype=dtype, operations=operations
+        )
+        self.time_modulation = nn.Sequential(
+            nn.SiLU(),
+            operations.Linear(hidden_size, 6 * hidden_size, device=device, dtype=dtype),
+        )
+        self.text_embedder = LingBotVideoTextEmbedder(text_dim, hidden_size, device=device, dtype=dtype, operations=operations)
+        self.rope = LingBotVideoRotaryEmbedding(axes_dims, axes_lens, rope_theta)
+        self.blocks = nn.ModuleList(
+            [
+                LingBotVideoBlock(
+                    hidden_size=hidden_size,
+                    num_attention_heads=num_attention_heads,
+                    intermediate_size=intermediate_size,
+                    norm_eps=norm_eps,
+                    qkv_bias=qkv_bias,
+                    out_bias=out_bias,
+                    num_experts=num_experts,
+                    num_experts_per_tok=num_experts_per_tok,
+                    moe_intermediate_size=moe_intermediate_size,
+                    decoder_sparse_step=decoder_sparse_step,
+                    mlp_only_layers=mlp_only_layers,
+                    n_shared_experts=n_shared_experts,
+                    score_func=score_func,
+                    norm_topk_prob=norm_topk_prob,
+                    n_group=n_group,
+                    topk_group=topk_group,
+                    routed_scaling_factor=routed_scaling_factor,
+                    layer_idx=i,
+                    device=device,
+                    dtype=dtype,
+                    operations=operations,
+                )
+                for i in range(depth)
+            ]
+        )
+        self.norm_out = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=norm_eps, device=device, dtype=dtype)
+        self.norm_out_modulation = nn.Sequential(
+            nn.SiLU(),
+            operations.Linear(hidden_size, 2 * hidden_size, device=device, dtype=dtype),
+        )
+        self.proj_out = operations.Linear(hidden_size, math.prod(patch_size) * out_channels, device=device, dtype=dtype)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,             # (B, C, T, H, W)
+        timestep: torch.Tensor,                  # (B,) ∈ [0, 1000](= sigma*1000)
+        context: torch.Tensor = None,            # (B, L, text_dim)
+        encoder_attention_mask: Optional[torch.Tensor] = None,  # (B, L) 1=valid
+        attention_mask: Optional[torch.Tensor] = None,
+        transformer_options={},
+        **kwargs,
+    ):
+        encoder_hidden_states = context
+        if encoder_hidden_states is None:
+            raise ValueError("LingBotVideo requires text conditioning.")
+        if encoder_attention_mask is None:
+            encoder_attention_mask = attention_mask
+        B, C, T, H, W = hidden_states.shape
+        pF, pH, pW = self.patch_size
+        gt, gh, gw = T // pF, H // pH, W // pW
+        n_video = gt * gh * gw
+        L = encoder_hidden_states.shape[1]
+        device = hidden_states.device
+        if encoder_attention_mask is not None:
+            text_lens = encoder_attention_mask.sum(dim=-1).long()
+        else:
+            text_lens = torch.full((B,), L, dtype=torch.long, device=device)
+        text_lens_list = [int(v) for v in text_lens.detach().cpu().tolist()]
+
+        # patchify: token order (f h w), feature order (pf ph pw c) -- matches patchify_and_embed
+        patch_tokens = hidden_states.reshape(B, C, gt, pF, gh, pH, gw, pW)
+        patch_tokens = patch_tokens.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(
+            B,
+            n_video,
+            pF * pH * pW * C,
+        )
+        x = self.patch_embedder(patch_tokens)
+        text = self.text_embedder(encoder_hidden_states)
+        joint = torch.cat([x, text], dim=1)  # [video; text]
+        joint_seq_len = joint.shape[1]
+
+        # Per-sample RoPE: video t-axis start = real text length of this sample + 1
+        rotary_parts = [
+            self.rope(make_joint_position_ids(text_lens_list[i], gt, gh, gw, device, L))
+            for i in range(B)
+        ]
+        rotary = torch.stack(rotary_parts, dim=0).unsqueeze(2)  # (B, S, 1, head_dim/2, 2, 2)
+
+        attention_mask = None
+        moe_padding_mask = None
+        has_padding = encoder_attention_mask is not None and bool((text_lens < L).any())
+        if has_padding:
+            key_mask = torch.cat(
+                [torch.ones(B, n_video, dtype=torch.bool, device=device),
+                 encoder_attention_mask.bool()],
+                dim=1,
+            )
+            attention_mask = key_mask[:, None, None, :]      # (B,1,1,S) → SDPA broadcast
+            moe_padding_mask = key_mask.reshape(-1)  # (B*S,)
+
+        timestep_proj = timestep_embedding(timestep.to(hidden_states.dtype), self.freq_dim, time_factor=1.0)
+        t_emb = self.time_embedder(timestep_proj)                            # (B, D)
+        temb_input = t_emb.unsqueeze(1).expand(B, joint_seq_len, -1)       # (B, S, D)
+        temb6 = self.time_modulation(temb_input.reshape(B * joint_seq_len, -1))
+        temb6 = temb6.reshape(B, joint_seq_len, -1)                        # (B, S, 6D)
+
+        temb6 = temb6.reshape(temb6.shape[0] * temb6.shape[1], -1)
+
+        for block in self.blocks:
+            joint = block(
+                joint,
+                temb6,
+                rotary,
+                attention_mask,
+                moe_padding_mask,
+                transformer_options=transformer_options,
+            )
+
+        final_mod = self.norm_out_modulation(temb_input.reshape(joint.shape[0] * joint.shape[1], -1))
+        shift, scale = final_mod.reshape(joint.shape[0], joint.shape[1], -1).chunk(2, dim=-1)
+        final_hidden = self.norm_out(joint) * (1.0 + scale) + shift
+        projected = self.proj_out(final_hidden)
+        x = projected[:, :n_video]
+
+        # unpatchify (matches the rearrange in postprocess)
+        Cout = self.out_channels
+        x = x.reshape(B, gt, gh, gw, pF, pH, pW, Cout)
+        x = x.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(B, Cout, T, H, W)
+
+        return x
+
+
+LingBotVideoTransformer3DModel = LingBotVideo

comfy/model_base.py

@@ -63,6 +63,7 @@ import comfy.ldm.kandinsky5.model
 import comfy.ldm.anima.model
 import comfy.ldm.ace.ace_step15
 import comfy.ldm.cogvideo.model
+import comfy.ldm.lingbot_video.model
 import comfy.ldm.rt_detr.rtdetr_v4
 import comfy.ldm.ernie.model
 import comfy.ldm.sam3.detector
@@ -1376,6 +1377,20 @@ class HunyuanVideoSkyreelsI2V(HunyuanVideo):
     def scale_latent_inpaint(self, latent_image, **kwargs):
         return super().scale_latent_inpaint(latent_image=latent_image, **kwargs)
 
+class LingBotVideo(BaseModel):
+    def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
+        super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lingbot_video.model.LingBotVideo)
+
+    def extra_conds(self, **kwargs):
+        out = super().extra_conds(**kwargs)
+        cross_attn = kwargs.get("cross_attn", None)
+        if cross_attn is not None:
+            out["c_crossattn"] = comfy.conds.CONDRegular(cross_attn)
+        return out
+
+    def scale_latent_inpaint(self, latent_image, **kwargs):
+        return latent_image
+
 class CosmosVideo(BaseModel):
     def __init__(self, model_config, model_type=ModelType.EDM, image_to_video=False, device=None):
         super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.cosmos.model.GeneralDIT)

comfy/model_detection.py

@@ -232,6 +232,54 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
             dit_config["meanflow_sum"] = False
         return dit_config
 
+    if '{}patch_embedder.weight'.format(key_prefix) in state_dict_keys and '{}text_embedder.norm.weight'.format(key_prefix) in state_dict_keys and '{}blocks.0.attn.to_q.weight'.format(key_prefix) in state_dict_keys: # LingBot Video
+        dit_config = {}
+        patch_size = (1, 2, 2)
+        patch_prod = math.prod(patch_size)
+        patch_embed = state_dict['{}patch_embedder.weight'.format(key_prefix)]
+        proj_out = state_dict['{}proj_out.weight'.format(key_prefix)]
+        hidden_size = patch_embed.shape[0]
+        dit_config["image_model"] = "lingbot_video"
+        dit_config["patch_size"] = patch_size
+        dit_config["in_channels"] = 16
+        dit_config["out_channels"] = proj_out.shape[0] // patch_prod
+        dit_config["hidden_size"] = hidden_size
+        dit_config["num_attention_heads"] = hidden_size // 128
+        dit_config["depth"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.')
+        dit_config["text_dim"] = state_dict['{}text_embedder.norm.weight'.format(key_prefix)].shape[0]
+        dit_config["freq_dim"] = 256
+        dit_config["qkv_bias"] = '{}blocks.0.attn.to_q.bias'.format(key_prefix) in state_dict_keys
+        dit_config["out_bias"] = '{}blocks.0.attn.to_out.bias'.format(key_prefix) in state_dict_keys
+        dit_config["patch_embed_bias"] = '{}patch_embedder.bias'.format(key_prefix) in state_dict_keys
+        dit_config["timestep_mlp_bias"] = '{}time_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys
+
+        moe_key = '{}blocks.0.ffn.experts.w1'.format(key_prefix)
+        if moe_key in state_dict_keys:
+            experts = state_dict[moe_key]
+            dit_config["num_experts"] = experts.shape[0]
+            dit_config["moe_intermediate_size"] = experts.shape[1]
+            if experts.shape[0] == 128:
+                dit_config["n_group"] = 4
+                dit_config["topk_group"] = 2
+                dit_config["routed_scaling_factor"] = 2.5
+            mlp_only_layers = []
+            for i in range(dit_config["depth"]):
+                if '{}blocks.{}.ffn.gate_proj.weight'.format(key_prefix, i) in state_dict_keys:
+                    mlp_only_layers.append(i)
+            dit_config["mlp_only_layers"] = tuple(mlp_only_layers)
+            if len(mlp_only_layers) > 0:
+                dit_config["intermediate_size"] = state_dict['{}blocks.{}.ffn.gate_proj.weight'.format(key_prefix, mlp_only_layers[0])].shape[0]
+            if '{}blocks.0.ffn.shared_experts.gate_proj.weight'.format(key_prefix) in state_dict_keys:
+                shared = state_dict['{}blocks.0.ffn.shared_experts.gate_proj.weight'.format(key_prefix)]
+                dit_config["n_shared_experts"] = shared.shape[0] // experts.shape[1]
+        else:
+            dit_config["num_experts"] = 0
+            dit_config["intermediate_size"] = state_dict['{}blocks.0.ffn.gate_proj.weight'.format(key_prefix)].shape[0]
+
+        if metadata is not None and "config" in metadata:
+            dit_config.update(json.loads(metadata["config"]).get("transformer", {}))
+        return dit_config
+
     if any_suffix_in(state_dict_keys, key_prefix, 'double_blocks.0.img_attn.norm.key_norm.', ["weight", "scale"]) and ('{}img_in.weight'.format(key_prefix) in state_dict_keys or any_suffix_in(state_dict_keys, key_prefix, 'distilled_guidance_layer.norms.0.', ["weight", "scale"])): #Flux, Chroma or Chroma Radiance (has no img_in.weight)
         dit_config = {}
         if '{}double_stream_modulation_img.lin.weight'.format(key_prefix) in state_dict_keys:

comfy/sd.py

@@ -69,6 +69,7 @@ import comfy.text_encoders.ace15
 import comfy.text_encoders.longcat_image
 import comfy.text_encoders.qwen35
 import comfy.text_encoders.qwen3vl
+import comfy.text_encoders.lingbot_video
 import comfy.text_encoders.boogu
 import comfy.text_encoders.ernie
 import comfy.text_encoders.gemma4
@@ -1313,6 +1314,7 @@ class CLIPType(Enum):
     IDEOGRAM4 = 30
     BOOGU = 31
     KREA2 = 32
+    LINGBOT_VIDEO = 33
 
 
 
@@ -1646,6 +1648,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
                 klein_model_type = "qwen3_8b" if te_model == TEModel.QWEN3VL_8B else "qwen3_4b"
                 clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type=klein_model_type)
                 clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8B if te_model == TEModel.QWEN3VL_8B else comfy.text_encoders.flux.KleinTokenizer
+            elif clip_type == CLIPType.LINGBOT_VIDEO and te_model == TEModel.QWEN3VL_4B:
+                clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
+                clip_target.clip = comfy.text_encoders.lingbot_video.te(**llama_detect(clip_data), model_type="qwen3vl_4b")
+                clip_target.tokenizer = comfy.text_encoders.lingbot_video.tokenizer(model_type="qwen3vl_4b")
             else:
                 clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
                 qwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model]

comfy/supported_models.py

@@ -27,6 +27,8 @@ import comfy.text_encoders.z_image
 import comfy.text_encoders.ideogram4
 import comfy.text_encoders.boogu
 import comfy.text_encoders.krea2
+import comfy.text_encoders.qwen3vl
+import comfy.text_encoders.lingbot_video
 import comfy.text_encoders.anima
 import comfy.text_encoders.ace15
 import comfy.text_encoders.longcat_image
@@ -1023,6 +1025,40 @@ class HunyuanVideoSkyreelsI2V(HunyuanVideo):
         out = model_base.HunyuanVideoSkyreelsI2V(self, device=device)
         return out
 
+class LingBotVideo(supported_models_base.BASE):
+    unet_config = {
+        "image_model": "lingbot_video",
+    }
+
+    sampling_settings = {
+        "shift": 3.0,
+    }
+
+    unet_extra_config = {}
+    latent_format = latent_formats.LingBotVideo
+
+    memory_usage_factor = 1.8
+
+    supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+    vae_key_prefix = ["vae."]
+    text_encoder_key_prefix = ["text_encoders."]
+
+    def __init__(self, unet_config):
+        super().__init__(unet_config)
+        self.memory_usage_factor = self.memory_usage_factor * (unet_config.get("hidden_size", 2048) / 2048)
+
+    def get_model(self, state_dict, prefix="", device=None):
+        return model_base.LingBotVideo(self, device=device)
+
+    def clip_target(self, state_dict={}):
+        pref = self.text_encoder_key_prefix[0]
+        qwen3vl_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref))
+        if len(qwen3vl_detect) > 0:
+            qwen3vl_detect["model_type"] = "qwen3vl_4b"
+            return supported_models_base.ClipTarget(comfy.text_encoders.lingbot_video.tokenizer(model_type="qwen3vl_4b"), comfy.text_encoders.lingbot_video.te(**qwen3vl_detect))
+        return None
+
 class CosmosT2V(supported_models_base.BASE):
     unet_config = {
         "image_model": "cosmos",
@@ -2317,6 +2353,7 @@ models = [
     HunyuanVideoSkyreelsI2V,
     HunyuanVideoI2V,
     HunyuanVideo,
+    LingBotVideo,
     CosmosT2V,
     CosmosI2V,
     CosmosT2IPredict2,

comfy/text_encoders/lingbot_video.py

@@ -0,0 +1,149 @@
+import comfy.text_encoders.qwen3vl
+from comfy import sd1_clip
+
+
+PAD_TOKEN = 151643
+CROP_MARKER = {"type": "lingbot_video_crop_start"}
+
+PROMPT_TEMPLATE = (
+    "<|im_start|>system\nGiven a user input that may include a text prompt alone, "
+    "a text prompt with an image reference, or a text prompt with a video reference "
+    "or a video reference alone, generate an \"Enhanced prompt\" that provides detailed "
+    "visual descriptions suitable for video generation. Evaluate the level of detail "
+    "in the user's input: if it is simple, enrich it by adding specifics about colors, "
+    "shapes, sizes, textures, lighting, motion dynamics, camera movement, temporal "
+    "progression, and spatial relationships to create vivid, concrete, and temporally "
+    "coherent scenes to create vivid and concrete scenes. Please generate only the "
+    "enhanced description for the prompt below and avoid including any additional "
+    "commentary or evaluations:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n"
+    "<|im_start|>assistant\n"
+)
+IMAGE_PROMPT_TEMPLATE = "<|vision_start|><|image_pad|><|vision_end|>"
+
+
+def _marker_tuple(example=None):
+    if example is not None and len(example) > 2:
+        return (CROP_MARKER, 1.0, None)
+    return (CROP_MARKER, 1.0)
+
+
+class LingBotVideoTokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer):
+    def __init__(self, embedding_directory=None, tokenizer_data={}, model_type="qwen3vl_4b"):
+        super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type=model_type)
+        self._lingbot_crop_start = None
+
+    def lingbot_crop_start(self):
+        if self._lingbot_crop_start is not None:
+            return self._lingbot_crop_start
+        prefix = PROMPT_TEMPLATE.split("{}")[0]
+        tokens = super().tokenize_with_weights(prefix, thinking=True)
+        key = next(iter(tokens))
+        count = 0
+        for token in tokens[key][0]:
+            token_id = token[0]
+            if isinstance(token_id, int) and token_id == PAD_TOKEN:
+                continue
+            count += 1
+        self._lingbot_crop_start = count
+        return count
+
+    def tokenize_with_weights(self, text, return_word_ids=False, images=[], prevent_empty_text=False, thinking=True, **kwargs):
+        image = kwargs.get("image", None)
+        if image is not None and len(images) == 0:
+            images = [image[i:i + 1] for i in range(image.shape[0])]
+
+        prompt_text = text
+        if len(images) > 0 and not prompt_text.startswith("<|vision_start|>"):
+            prompt_text = IMAGE_PROMPT_TEMPLATE + prompt_text
+
+        tokens = super().tokenize_with_weights(
+            prompt_text,
+            return_word_ids=return_word_ids,
+            llama_template=PROMPT_TEMPLATE,
+            images=images,
+            prevent_empty_text=prevent_empty_text,
+            thinking=thinking,
+            **kwargs,
+        )
+        crop_start = self.lingbot_crop_start()
+        for key in tokens:
+            for row in tokens[key]:
+                example = row[0] if len(row) > 0 else None
+                row.insert(crop_start, _marker_tuple(example))
+        return tokens
+
+
+class LingBotVideoClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel):
+    def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}, model_type="qwen3vl_4b"):
+        super().__init__(
+            device=device,
+            layer="last",
+            layer_idx=None,
+            dtype=dtype,
+            attention_mask=attention_mask,
+            model_options=model_options,
+            model_type=model_type,
+        )
+        self.return_attention_masks = False
+
+    @staticmethod
+    def _strip_crop_markers(tokens):
+        clean_tokens = []
+        crop_starts = []
+        for row in tokens:
+            clean_row = []
+            crop_start = None
+            for token in row:
+                token_value = token[0] if isinstance(token, tuple) else token
+                if isinstance(token_value, dict) and token_value.get("type") == CROP_MARKER["type"]:
+                    crop_start = len(clean_row)
+                    continue
+                clean_row.append(token)
+            clean_tokens.append(clean_row)
+            crop_starts.append(crop_start)
+        return clean_tokens, crop_starts
+
+    def forward(self, tokens):
+        clean_tokens, crop_starts = self._strip_crop_markers(tokens)
+        out = super().forward(clean_tokens)
+        crop_starts = [c for c in crop_starts if c is not None]
+        if len(crop_starts) == 0:
+            return out
+
+        crop_start = min(crop_starts)
+        z, pooled_output = out[:2]
+        z = z[:, crop_start:]
+
+        return z, pooled_output
+
+
+class LingBotVideoTEModel(comfy.text_encoders.qwen3vl.Qwen3VLTEModel):
+    def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen3vl_4b"):
+        clip_model = lambda **kw: LingBotVideoClipModel(**kw, model_type=model_type)
+        sd1_clip.SD1ClipModel.__init__(
+            self,
+            device=device,
+            dtype=dtype,
+            name=model_type,
+            clip_model=clip_model,
+            model_options=model_options,
+        )
+
+
+def tokenizer(model_type="qwen3vl_4b"):
+    class LingBotVideoTokenizer_(LingBotVideoTokenizer):
+        def __init__(self, embedding_directory=None, tokenizer_data={}):
+            super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type=model_type)
+    return LingBotVideoTokenizer_
+
+
+def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen3vl_4b"):
+    class LingBotVideoTEModel_(LingBotVideoTEModel):
+        def __init__(self, device="cpu", dtype=None, model_options={}):
+            if dtype_llama is not None:
+                dtype = dtype_llama
+            if llama_quantization_metadata is not None:
+                model_options = model_options.copy()
+                model_options["quantization_metadata"] = llama_quantization_metadata
+            super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type)
+    return LingBotVideoTEModel_

comfy/text_encoders/qwen3vl.py

@@ -90,6 +90,27 @@ class Qwen3VL(BaseLlama, BaseQwen3, BaseGenerate, torch.nn.Module):
                 deepstack = [torch.cat([deepstack[i], ds[i]], dim=0) for i in range(len(ds))]
         return position_ids, visual_pos_masks, deepstack
 
+    def forward(self, input_ids, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[], **kwargs):
+        position_ids = kwargs.pop("position_ids", None)
+        visual_pos_masks = kwargs.pop("visual_pos_masks", None)
+        deepstack_embeds = kwargs.pop("deepstack_embeds", None)
+        if embeds is not None and position_ids is None:
+            position_ids, visual_pos_masks, deepstack_embeds = self.build_image_inputs(embeds, embeds_info)
+        return self.model(
+            input_ids,
+            attention_mask=attention_mask,
+            embeds=embeds,
+            num_tokens=num_tokens,
+            intermediate_output=intermediate_output,
+            final_layer_norm_intermediate=final_layer_norm_intermediate,
+            dtype=dtype,
+            position_ids=position_ids,
+            embeds_info=embeds_info,
+            visual_pos_masks=visual_pos_masks,
+            deepstack_embeds=deepstack_embeds,
+            **kwargs,
+        )
+
 
 def _make_qwen3vl_model(model_type):
     class Qwen3VL_(Qwen3VL):

comfy_extras/nodes_lingbot_video.py

@@ -0,0 +1,167 @@
+import math
+import nodes
+import numpy as np
+import torch
+from PIL import Image
+
+import comfy.latent_formats
+import comfy.model_management
+import comfy.utils
+from comfy_api.latest import ComfyExtension, io
+from typing_extensions import override
+
+
+IMAGE_MIN_TOKEN_NUM = 4
+IMAGE_MAX_TOKEN_NUM = 16384
+MAX_RATIO = 200
+SPATIAL_MERGE_SIZE = 2
+VISION_PATCH_SIZE = 16
+
+
+def _crop_image(image, width, height):
+    image = image[:1].movedim(-1, 1)
+    image = comfy.utils.common_upscale(image, width, height, "bilinear", "center")
+    return image.movedim(1, -1)[:, :, :, :3]
+
+
+def _round_by_factor(number, factor):
+    return round(number / factor) * factor
+
+
+def _ceil_by_factor(number, factor):
+    return math.ceil(number / factor) * factor
+
+
+def _floor_by_factor(number, factor):
+    return math.floor(number / factor) * factor
+
+
+def _smart_resize(height, width, factor, min_pixels=None, max_pixels=None):
+    max_pixels = max_pixels if max_pixels is not None else IMAGE_MAX_TOKEN_NUM * factor ** 2
+    min_pixels = min_pixels if min_pixels is not None else IMAGE_MIN_TOKEN_NUM * factor ** 2
+    if max_pixels < min_pixels:
+        raise ValueError("max_pixels must be greater than or equal to min_pixels.")
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(f"LingBotVideo image aspect ratio must be smaller than {MAX_RATIO}.")
+
+    resized_height = max(factor, _round_by_factor(height, factor))
+    resized_width = max(factor, _round_by_factor(width, factor))
+    if resized_height * resized_width > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        resized_height = _floor_by_factor(height / beta, factor)
+        resized_width = _floor_by_factor(width / beta, factor)
+    elif resized_height * resized_width < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        resized_height = _ceil_by_factor(height * beta, factor)
+        resized_width = _ceil_by_factor(width * beta, factor)
+    return resized_height, resized_width
+
+
+def _vlm_image(image):
+    factor = VISION_PATCH_SIZE * SPATIAL_MERGE_SIZE
+    height, width = image.shape[1:3]
+    resized_height, resized_width = _smart_resize(height, width, factor)
+    array = image[0].detach().cpu().clamp(0, 1).mul(255).byte().numpy()
+    pil_image = Image.fromarray(array, mode="RGB").resize((resized_width, resized_height))
+    array = np.asarray(pil_image).astype(np.float32) / 255.0
+    return torch.from_numpy(array).unsqueeze(0)
+
+
+class TextEncodeLingBotVideoI2V(io.ComfyNode):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="TextEncodeLingBotVideoI2V",
+            category="model/conditioning/lingbot_video",
+            inputs=[
+                io.Clip.Input("clip"),
+                io.String.Input("prompt", multiline=True, dynamic_prompts=True),
+                io.String.Input("negative_prompt", multiline=True, dynamic_prompts=True, default=""),
+                io.Int.Input("width", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+                io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+                io.Image.Input("image", optional=True),
+            ],
+            outputs=[
+                io.Conditioning.Output(display_name="positive"),
+                io.Conditioning.Output(display_name="negative"),
+            ],
+        )
+
+    @classmethod
+    def execute(cls, clip, prompt, negative_prompt, width, height, image=None) -> io.NodeOutput:
+        if height % 16 != 0 or width % 16 != 0:
+            raise ValueError(f"LingBotVideo width and height must be multiples of 16, got {width}x{height}.")
+        if image is None:
+            images = []
+        else:
+            image = _crop_image(image, width, height)
+            images = [_vlm_image(image)]
+        positive = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=images))
+        negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt, images=images))
+        return io.NodeOutput(positive, negative)
+
+
+class LingBotImageToVideo(io.ComfyNode):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="LingBotImageToVideo",
+            category="model/conditioning/lingbot_video",
+            inputs=[
+                io.Conditioning.Input("positive"),
+                io.Conditioning.Input("negative"),
+                io.Vae.Input("vae"),
+                io.Int.Input("width", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+                io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+                io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+                io.Int.Input("batch_size", default=1, min=1, max=4096),
+                io.Image.Input("start_image", optional=True),
+            ],
+            outputs=[
+                io.Conditioning.Output(display_name="positive"),
+                io.Conditioning.Output(display_name="negative"),
+                io.Latent.Output(display_name="latent"),
+            ],
+        )
+
+    @classmethod
+    def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None) -> io.NodeOutput:
+        if length != 1 and (length - 1) % 4 != 0:
+            raise ValueError(f"LingBotVideo length must be 1 or 4n+1, got {length}.")
+        if height % 16 != 0 or width % 16 != 0:
+            raise ValueError(f"LingBotVideo width and height must be multiples of 16, got {width}x{height}.")
+        latent_frames = ((length - 1) // 4) + 1
+        latent = torch.zeros(
+            [batch_size, 16, latent_frames, height // 8, width // 8],
+            device=comfy.model_management.intermediate_device(),
+        )
+        out_latent = {"samples": latent}
+
+        if start_image is not None:
+            start_image = _crop_image(start_image, width, height)
+            cond_latent = comfy.latent_formats.LingBotVideo().process_in(vae.encode(start_image))
+            cond_latent = comfy.utils.resize_to_batch_size(cond_latent, batch_size)
+            cond_t = min(cond_latent.shape[2], latent.shape[2])
+            latent[:, :, :cond_t] = cond_latent[:, :, :cond_t].to(device=latent.device, dtype=latent.dtype)
+            noise_mask = torch.ones(
+                (batch_size, 1, latent.shape[2], latent.shape[3], latent.shape[4]),
+                device=latent.device,
+                dtype=latent.dtype,
+            )
+            noise_mask[:, :, :cond_t] = 0.0
+            out_latent["noise_mask"] = noise_mask
+
+        return io.NodeOutput(positive, negative, out_latent)
+
+
+class LingBotVideoExtension(ComfyExtension):
+    @override
+    async def get_node_list(self) -> list[type[io.ComfyNode]]:
+        return [
+            TextEncodeLingBotVideoI2V,
+            LingBotImageToVideo,
+        ]
+
+
+async def comfy_entrypoint() -> LingBotVideoExtension:
+    return LingBotVideoExtension()

nodes.py

@@ -992,7 +992,7 @@ class CLIPLoader:
     @classmethod
     def INPUT_TYPES(s):
         return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),
-                              "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2"], ),
+                              "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "lingbot_video"], ),
                               },
                 "optional": {
                               "device": (["default", "cpu"], {"advanced": True}),
@@ -2460,6 +2460,7 @@ async def init_builtin_extra_nodes():
         "nodes_tcfg.py",
         "nodes_context_windows.py",
         "nodes_qwen.py",
+        "nodes_lingbot_video.py",
         "nodes_boogu.py",
         "nodes_chroma_radiance.py",
         "nodes_pid.py",