This commit is contained in:
Jukka Seppänen 2026-07-05 15:41:56 +08:00 committed by GitHub
commit 04e297fb9c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 12405 additions and 80 deletions

View File

@ -10,6 +10,7 @@ import comfy.utils
import comfy.clip_model
import comfy.image_encoders.dino2
import comfy.image_encoders.dino3
from comfy.image_encoders.naf import NAF
class Output:
def __getitem__(self, key):
@ -53,6 +54,7 @@ class ClipVisionModel():
self.model.eval()
self.patcher = comfy.model_patcher.CoreModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device)
self.naf = None
def load_sd(self, sd):
return self.model.load_state_dict(sd, strict=False, assign=self.patcher.is_dynamic())
@ -141,6 +143,8 @@ def load_clipvision_from_sd(sd, prefix="", convert_keys=False):
json_config = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "image_encoders"), "dino2_large.json")
elif 'layer.0.mlp.gate_proj.weight' in sd and 'layer.31.norm1.weight' in sd: # Dinov3 ViT-H/16+ (SwiGLU gated MLP, 32 layers)
json_config = comfy.image_encoders.dino3.DINOV3_VITH_CONFIG
elif 'layer.9.attention.o_proj.bias' in sd: # dinov3 large (24 layers); generic o_proj.bias key, so must come after the ViT-H check
json_config = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "image_encoders"), "dino3_large.json")
else:
return None
@ -153,6 +157,14 @@ def load_clipvision_from_sd(sd, prefix="", convert_keys=False):
for k in keys:
if k not in u:
sd.pop(k)
# NAF feature upsampler bundled into the DINOv3 file under the `naf.` prefix.
naf_keys = [k for k in sd if k.startswith("naf.")]
if naf_keys:
naf_sd = {k[len("naf."):]: sd.pop(k) for k in naf_keys}
naf = NAF().eval()
naf.load_state_dict(naf_sd, strict=False)
naf.to(comfy.model_management.text_encoder_dtype(clip.load_device))
clip.naf = comfy.model_patcher.CoreModelPatcher(naf, load_device=clip.load_device, offload_device=comfy.model_management.text_encoder_offload_device())
return clip
def load(ckpt_path):

View File

@ -0,0 +1,23 @@
{
"model_type": "dinov3",
"hidden_size": 1024,
"image_size": 224,
"initializer_range": 0.02,
"intermediate_size": 4096,
"key_bias": false,
"layer_norm_eps": 1e-05,
"mlp_bias": true,
"num_attention_heads": 16,
"num_channels": 3,
"num_hidden_layers": 24,
"num_register_tokens": 4,
"patch_size": 16,
"pos_embed_rescale": 2.0,
"proj_bias": true,
"query_bias": true,
"rope_theta": 100.0,
"use_gated_mlp": false,
"value_bias": true,
"image_mean": [0.485, 0.456, 0.406],
"image_std": [0.229, 0.224, 0.225]
}

283
comfy/image_encoders/naf.py Normal file
View File

@ -0,0 +1,283 @@
"""NAF (Neighborhood Attention Filtering) feature upsampler.
Vendored from valeoai/NAF (Apache-2.0):
https://github.com/valeoai/NAF src/model/naf.py + src/layers/{convolutions,attentions,rope}.py
Used by Pixal3D's shape/texture conditioning to produce
the 2x-upsampled half of the 2048-channel proj feature map.
"""
import math
from typing import Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
# Pure-torch neighborhood attention (replaces natten.na2d / na2d_qk + na2d_av).
def upsample_lr_slice(src_lr: torch.Tensor, lr_dh: int, lr_dw: int,
hr_h_range: Tuple[int, int], hr_w_range: Tuple[int, int]) -> torch.Tensor:
"""Slice a LR-layout tensor [B, h_lr, w_lr, n, C], permute to BCHW, and
nearest-exact upsample only the region covering [hr_h_range, hr_w_range].
Returns BCHW at hr_h_end-hr_h_start x hr_w_end-hr_w_start (no padding for
out-of-bounds regions)."""
B = src_lr.shape[0]
n = src_lr.shape[-2]
C = src_lr.shape[-1]
h_hr_start, h_hr_end = hr_h_range
w_hr_start, w_hr_end = hr_w_range
# LR positions covering [h_hr_start, h_hr_end). Nearest-exact maps HR p → p // D.
lr_h_start = h_hr_start // lr_dh
lr_h_end = (h_hr_end - 1) // lr_dh + 1
lr_w_start = w_hr_start // lr_dw
lr_w_end = (w_hr_end - 1) // lr_dw + 1
lr_slice = src_lr[:, lr_h_start:lr_h_end, lr_w_start:lr_w_end]
lh, lw = lr_slice.shape[1], lr_slice.shape[2]
lr_bcd = lr_slice.permute(0, 3, 4, 1, 2).reshape(B * n, C, lh, lw).contiguous()
up = F.interpolate(lr_bcd, scale_factor=(lr_dh, lr_dw), mode="nearest-exact")
offset_h = h_hr_start - lr_h_start * lr_dh
offset_w = w_hr_start - lr_w_start * lr_dw
return up[:, :, offset_h:offset_h + (h_hr_end - h_hr_start),
offset_w:offset_w + (w_hr_end - w_hr_start)]
def na2d_pure(
q: torch.Tensor, # [B, H, W, n_heads, d_qk] at HR.
k_lr: torch.Tensor, # [B, h_lr, w_lr, n_heads, d_qk] at LR
v_lr: torch.Tensor, # [B, h_lr, w_lr, n_heads, d_v] at LR
kernel_size: Tuple[int, int], # (Kh, Kw) attention window.
dilation: Tuple[int, int], # (Dh, Dw) stride within the unrolled K/V grid; also the LR→HR upsample factor.
scale: float, # 1 / sqrt(d_qk) scaling for the Q·K scores.
tile: int = 128, # Spatial tile size (output positions per tile)
v_chunk: int = 64, # Sub-divide d_v into chunks of this size when computing attn·V. None disables chunking.
output: torch.Tensor = None, # Pre-allocated [B, n_heads, d_v, H, W] buffer (may be on CPU).
) -> torch.Tensor: # [B, n_heads, d_v, H, W] (caller views as BCHW).
"""Neighborhood attention in pure torch via F.unfold + per-tile slicing.
K and V are passed at LR resolution and upsampled (nearest-exact) per-tile only
for the slice the unfold needs. Avoids the [B, n*d, H, W] HR allocations for K
(512 MB) and V (2 GB) at tex_1024 fp16. Spatial tiling bounds the per-tile
F.unfold blob; `v_chunk` further slices d_v so attn·V is computed in C-sized
chunks (attn is reused, computed once from Q/K).
"""
B, H, W, n, d_qk = q.shape
d_v = v_lr.shape[-1]
Kh, Kw = kernel_size
Dh, Dw = dilation
pad_h, pad_w = (Kh // 2) * Dh, (Kw // 2) * Dw
out = output if output is not None else torch.empty((B, n, d_v, H, W), device=q.device, dtype=q.dtype)
th = min(tile, H) if tile else H
tw = min(tile, W) if tile else W
chunk = v_chunk if (v_chunk and v_chunk < d_v) else d_v
for h0 in range(0, H, th):
for w0 in range(0, W, tw):
h1, w1 = min(h0 + th, H), min(w0 + tw, W)
t_h, t_w = h1 - h0, w1 - w0
# Padded HR region the unfold needs (kernel span = (K-1)*D + 1).
h_src_start = max(0, h0 - pad_h)
h_src_end = min(H, h1 + pad_h)
w_src_start = max(0, w0 - pad_w)
w_src_end = min(W, w1 + pad_w)
pad_top = max(0, pad_h - h0)
pad_bot = max(0, (h1 + pad_h) - H)
pad_lft = max(0, pad_w - w0)
pad_rgt = max(0, (w1 + pad_w) - W)
# Upsample only the tile region from k_lr / v_lr.
k_tile = upsample_lr_slice(k_lr, Dh, Dw,
(h_src_start, h_src_end),
(w_src_start, w_src_end))
v_tile = upsample_lr_slice(v_lr, Dh, Dw,
(h_src_start, h_src_end),
(w_src_start, w_src_end))
if pad_top or pad_bot or pad_lft or pad_rgt:
k_tile = F.pad(k_tile, [pad_lft, pad_rgt, pad_top, pad_bot])
v_tile = F.pad(v_tile, [pad_lft, pad_rgt, pad_top, pad_bot])
# Q·K → attention weights (small: KK=81 per output position).
KK = Kh * Kw
k_w = F.unfold(k_tile, kernel_size=(Kh, Kw), dilation=(Dh, Dw), padding=0)
k_w = k_w.view(B, n, d_qk, KK, t_h * t_w).permute(0, 1, 4, 3, 2) # [B, n, t, KK, d_qk]
# q is [B, H, W, n, d_qk]; per-tile slice + permute -> [B, n, t_h*t_w, 1, d_qk].
q_tile = q[:, h0:h1, w0:w1].permute(0, 3, 1, 2, 4).reshape(B, n, t_h * t_w, 1, d_qk)
scores = torch.matmul(q_tile, k_w.transpose(-1, -2)) * scale
attn = scores.softmax(dim=-1)
del k_w, scores, q_tile, k_tile
# attn · V, chunked over d_v.
for c0 in range(0, d_v, chunk):
c1 = min(c0 + chunk, d_v)
v_w = F.unfold(v_tile[:, c0:c1], kernel_size=(Kh, Kw),dilation=(Dh, Dw), padding=0) # [B*n, (c1-c0)*KK, t]
v_w = v_w.view(B, n, c1 - c0, KK, t_h * t_w).permute(0, 1, 4, 3, 2)
out_chunk = torch.matmul(attn, v_w).squeeze(-2) # [B, n, t, c1-c0]
out_chunk = out_chunk.view(B, n, t_h, t_w, c1 - c0).permute(0, 1, 4, 2, 3)
out[:, :, c0:c1, h0:h1, w0:w1] = out_chunk
del v_w, out_chunk
del attn, v_tile
return out # [B, n, d_v, H, W] — sole caller (CrossAttention) views it as BCHW directly.
class CrossAttention(nn.Module):
"""Window-restricted cross-attention. No learnable parameters; the model's
capacity lives entirely in the ImageEncoder convs."""
def __init__(self, dim: int, num_heads: int, kernel_size: Tuple[int, int] = (9, 9)):
super().__init__()
assert dim % num_heads == 0, "dim must be divisible by num_heads"
self.num_heads = num_heads
self.kernel_size = kernel_size
self.scale = (dim // num_heads) ** -0.5
@staticmethod
def _split_heads_lr(x: torch.Tensor, num_heads: int) -> torch.Tensor:
"""[B, n*d, h, w] -> [B, h, w, n, d] at the input resolution (no upsample)."""
B, C, H, W = x.shape
return x.view(B, num_heads, C // num_heads, H, W).permute(0, 3, 4, 1, 2).contiguous()
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
output_device=None) -> torch.Tensor:
hq, wq = q.shape[-2:]
hk, wk = k.shape[-2:]
dilation = (hq // hk, wq // wk)
B, C, _, _ = q.shape
q = q.view(B, self.num_heads, C // self.num_heads, hq, wq).permute(0, 3, 4, 1, 2).contiguous()
k_lr = self._split_heads_lr(k, self.num_heads).to(q.dtype)
v_lr = self._split_heads_lr(v, self.num_heads).to(q.dtype)
out_buf = None
if output_device is not None:
n = self.num_heads
d_v = v.shape[1] // n
out_buf = torch.empty(B, n, d_v, hq, wq, device=output_device, dtype=q.dtype)
out = na2d_pure(q, k_lr, v_lr, self.kernel_size, dilation, self.scale, output=out_buf)
return out.view(B, -1, hq, wq)
# RoPE positional embedding
def rope_rotate_half(x: torch.Tensor) -> torch.Tensor:
x1, x2 = x.chunk(2, dim=-1)
return torch.cat([-x2, x1], dim=-1)
class RoPE(nn.Module):
def __init__(self, embed_dim: int, num_heads: int, base: float = 100.0):
super().__init__()
assert embed_dim % (4 * num_heads) == 0
self.num_heads = num_heads
self.D_head = embed_dim // num_heads
self.base = base
self.register_buffer("periods", torch.empty(self.D_head // 4), persistent=True) # loaded from the checkpoint
def _cos_sin(self, H: int, W: int, dtype: torch.dtype):
"""cos/sin depend only on (H, W, dtype) and the checkpoint-fixed periods; recomputed per forward."""
device = self.periods.device
coords_h = torch.arange(0.5, H, device=device, dtype=torch.float32) / H
coords_w = torch.arange(0.5, W, device=device, dtype=torch.float32) / W
coords = torch.stack(torch.meshgrid(coords_h, coords_w, indexing="ij"), dim=-1) # [H, W, 2]
coords = coords.flatten(0, 1) * 2.0 - 1.0 # [HW, 2]
angles = 2 * math.pi * coords[:, :, None] / self.periods.to(coords.dtype)[None, None, :] # [HW, 2, D//4]
angles = angles.flatten(1, 2).tile(2) # [HW, D]
cos = torch.cos(angles).to(dtype)
sin = torch.sin(angles).to(dtype)
return cos, sin
def forward(self, x: torch.Tensor) -> torch.Tensor:
# x: [B, n*D_head, H, W]
B, C, H, W = x.shape
n = self.num_heads
D = C // n
x = x.view(B, n, D, H, W).permute(0, 1, 3, 4, 2).reshape(B, n, H * W, D)
cos, sin = self._cos_sin(H, W, x.dtype)
x = (x * cos) + (rope_rotate_half(x) * sin)
x = x.view(B, n, H, W, D).permute(0, 1, 4, 2, 3).reshape(B, n * D, H, W)
return x
# Image encoder
class EncBlock(nn.Module):
def __init__(self, channels: int, kernel_size: int, num_groups: int = 8):
super().__init__()
self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=channels)
self.conv1 = nn.Conv2d(channels, channels, kernel_size=kernel_size,
padding=kernel_size // 2, padding_mode="reflect", bias=True)
self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=channels)
self.conv2 = nn.Conv2d(channels, channels, kernel_size=kernel_size,
padding=kernel_size // 2, padding_mode="reflect", bias=True)
self.activation_fn = nn.SiLU()
def forward(self, x):
x = self.norm1(x)
x = self.activation_fn(x)
x = self.conv1(x)
x = self.norm2(x)
x = self.activation_fn(x)
x = self.conv2(x)
return x # no skip connection
def _encoder(in_dim: int, hidden_dim: int, kernel_size: int = 1, ks_res: int = 1, num_layers: int = 2) -> nn.Sequential:
return nn.Sequential(
nn.Conv2d(in_dim, hidden_dim, kernel_size=kernel_size, padding=kernel_size // 2, padding_mode="reflect", bias=True),
*[EncBlock(hidden_dim, kernel_size=ks_res) for _ in range(num_layers)],
)
class ImageEncoder(nn.Module):
"""Two parallel conv stacks (1x1 + 3x3) producing dim/2 channels each, then concat,
spatial average-pool to target size, RoPE-embed positions."""
def __init__(self, in_channels: int = 3, out_channels: int = 256,
heads_rope: int = 4, rope_base: float = 100.0, img_layers: int = 2):
super().__init__()
half = out_channels // 2
self.encoder = _encoder(in_channels, half, kernel_size=1, ks_res=1, num_layers=img_layers)
self.sem_encoder = _encoder(in_channels, half, kernel_size=3, ks_res=3, num_layers=img_layers)
self.rope = RoPE(embed_dim=out_channels, num_heads=heads_rope, base=rope_base)
def forward(self, x: torch.Tensor, output_size: Tuple[int, int]) -> torch.Tensor:
# Avoid running the conv stacks on >4× the target resolution.
out_h, out_w = output_size
if x.shape[-2] > 4 * out_h or x.shape[-1] > 4 * out_w:
x = F.interpolate(x, size=(min(x.shape[-2], 4 * out_h),
min(x.shape[-1], 4 * out_w)),
mode="bilinear", align_corners=False)
x = torch.cat([self.encoder(x), self.sem_encoder(x)], dim=1)
x = F.adaptive_avg_pool2d(x, output_size=output_size)
x = self.rope(x)
return x
class NAF(nn.Module):
"""NAF feature upsampler."""
def __init__(
self, dim: int = 256, # internal channel dimension of the ImageEncoder
heads_attn: int = 4, # attention heads in the windowed cross-attn
heads_rope: int = 4, # heads for RoPE position encoding (must divide dim)
kernel_size: int = 9, # square kernel for the neighborhood attention window
rope_base: float = 100.0, # base for RoPE frequency periods
img_layers: int = 2 # number of EncBlocks in each conv stack
):
super().__init__()
self.image_encoder = ImageEncoder(in_channels=3, out_channels=dim, heads_rope=heads_rope, rope_base=rope_base, img_layers=img_layers)
self.upsampler = CrossAttention(dim=dim, num_heads=heads_attn, kernel_size=(kernel_size, kernel_size))
def forward(
self,
image: torch.Tensor, # [B, 3, H_img, W_img] in [0, 1].
features: torch.Tensor, # [B, C, H_feat, W_feat] low-resolution features (any C).
output_size: Tuple[int, int], # (H_out, W_out) target spatial resolution for the upsampled features.
output_device=None,
) -> torch.Tensor: # [B, C, H_out, W_out] upsampled features.
"""Upsample low-res feature map to output_size, guided by the image."""
q = self.image_encoder(image, output_size=output_size)
k = F.adaptive_avg_pool2d(q, output_size=features.shape[-2:])
return self.upsampler(q, k, features, output_device=output_device)

View File

@ -249,6 +249,53 @@ class TripoSplat(LatentFormat):
def process_out(self, latent):
return latent
class Trellis2(LatentFormat):
latent_channels = 32
class Trellis2SLAT(Trellis2):
# Sparse structured latent: per-token feats [N, 32]. process_out denormalizes
# the decoded feats (latent * std + mean); subclasses carry each space's stats.
latents_mean = None
latents_std = None
def process_in(self, latent):
mean = self.latents_mean.to(latent.device, latent.dtype)
std = self.latents_std.to(latent.device, latent.dtype)
return (latent - mean) / std
def process_out(self, latent):
mean = self.latents_mean.to(latent.device, latent.dtype)
std = self.latents_std.to(latent.device, latent.dtype)
return latent * std + mean
class Trellis2ShapeSLAT(Trellis2SLAT):
latents_mean = torch.tensor([
0.781296, 0.018091, -0.495192, -0.558457, 1.060530, 0.093252, 1.518149, -0.933218,
-0.732996, 2.604095, -0.118341, -2.143904, 0.495076, -2.179512, -2.130751, -0.996944,
0.261421, -2.217463, 1.260067, -0.150213, 3.790713, 1.481266, -1.046058, -1.523667,
-0.059621, 2.220780, 1.621212, 0.877230, 0.567247, -3.175944, -3.186688, 1.578665
])[None]
latents_std = torch.tensor([
5.972266, 4.706852, 5.445010, 5.209927, 5.320220, 4.547237, 5.020802, 5.444004,
5.226681, 5.683095, 4.831436, 5.286469, 5.652043, 5.367606, 5.525084, 4.730578,
4.805265, 5.124013, 5.530808, 5.619001, 5.103930, 5.417670, 5.269677, 5.547194,
5.634698, 5.235274, 6.110351, 5.511298, 6.237273, 4.879207, 5.347008, 5.405691
])[None]
class Trellis2TexSLAT(Trellis2SLAT):
latents_mean = torch.tensor([
3.501659, 2.212398, 2.226094, 0.251093, -0.026248, -0.687364, 0.439898, -0.928075,
0.029398, -0.339596, -0.869527, 1.038479, -0.972385, 0.126042, -1.129303, 0.455149,
-1.209521, 2.069067, 0.544735, 2.569128, -0.323407, 2.293000, -1.925608, -1.217717,
1.213905, 0.971588, -0.023631, 0.106750, 2.021786, 0.250524, -0.662387, -0.768862
])[None]
latents_std = torch.tensor([
2.665652, 2.743913, 2.765121, 2.595319, 3.037293, 2.291316, 2.144656, 2.911822,
2.969419, 2.501689, 2.154811, 3.163343, 2.621215, 2.381943, 3.186697, 3.021588,
2.295916, 3.234985, 3.233086, 2.260140, 2.874801, 2.810596, 3.292720, 2.674999,
2.680878, 2.372054, 2.451546, 2.353556, 2.995195, 2.379849, 2.786195, 2.775190
])[None]
class Mochi(LatentFormat):
latent_channels = 12
latent_dimensions = 3

View File

@ -0,0 +1,154 @@
from typing import Optional, Tuple
import torch
import comfy.model_management
def compute_kernel_offsets(Kw, Kh, Kd, Dw, Dh, Dd, device):
"""Kernel spatial offsets in the same order as the CUDA/Triton kernels."""
offsets = []
for vx in range(Kw):
for vy in range(Kh):
for vz in range(Kd):
offsets.append((vx * Dw, vy * Dh, vz * Dd))
return torch.tensor(offsets, device=device, dtype=torch.int32)
class TorchHashMap:
"""Sorted-array hashmap backed by torch.searchsorted."""
def __init__(self, keys: torch.Tensor, values: torch.Tensor):
self.sorted_keys, order = torch.sort(keys.to(torch.long))
self.sorted_vals = values.to(torch.long)[order]
self._n = self.sorted_keys.numel()
# Chunk size for lookup_flat, caps each transient to ~CHUNK rows.
_LOOKUP_CHUNK = 1 << 23 # 8M rows ≈ 64 MB per int64 temp
def lookup_flat(self, flat_keys: torch.Tensor) -> torch.Tensor:
N = flat_keys.shape[0]
out = torch.full((N,), -1, device=flat_keys.device, dtype=torch.int32)
if self._n == 0 or N == 0:
return out
for s in range(0, N, self._LOOKUP_CHUNK):
e = min(s + self._LOOKUP_CHUNK, N)
flat_chunk = flat_keys[s:e].to(torch.long)
idx = torch.searchsorted(self.sorted_keys, flat_chunk)
in_range = idx < self._n
idx.clamp_(max=self._n - 1) # reuse idx as the "safe" index
found = in_range & (self.sorted_keys[idx] == flat_chunk)
if found.any():
found_idx = found.nonzero(as_tuple=True)[0]
out[s + found_idx] = self.sorted_vals[idx[found_idx]].to(torch.int32)
return out
def build_submanifold_neighbor_map(
hashmap,
coords: torch.Tensor,
W, H, D,
Kw, Kh, Kd,
Dw, Dh, Dd,
):
# neighbor[i, v] = index of the voxel at voxel i's coord + kernel-offset v, or -1.
# Chunked over voxels so the [chunk, V, 3] candidate transient stays bounded.
device = coords.device
M = coords.shape[0]
offsets = compute_kernel_offsets(Kw, Kh, Kd, Dw, Dh, Dd, device).long() # [V, 3]
V = offsets.shape[0]
center = torch.tensor([(Kw // 2) * Dw, (Kh // 2) * Dh, (Kd // 2) * Dd], device=device)
WHD, HD = W * H * D, H * D
neighbor = torch.empty((M, V), dtype=torch.int32, device=device)
# ~V*40 bytes/voxel of transient (int64 cand + flat + masks); cap at ~0.5 GB.
chunk = max(1, min(M, int(0.5 * (1024 ** 3) / (V * 40))))
for s in range(0, M, chunk):
e = min(s + chunk, M)
b = coords[s:e, 0].long()
cand = coords[s:e, 1:4].long()[:, None, :] + offsets[None, :, :] - center # [c, V, 3]
x, y, z = cand[..., 0], cand[..., 1], cand[..., 2]
in_bounds = (x >= 0) & (x < W) & (y >= 0) & (y < H) & (z >= 0) & (z < D) # [c, V]
flat = b[:, None] * WHD + x * HD + y * D + z # [c, V]
flat = torch.where(in_bounds, flat, torch.full_like(flat, -1)) # OOB -> guaranteed miss
neighbor[s:e] = hashmap.lookup_flat(flat.reshape(-1)).view(e - s, V)
return neighbor
def get_recommended_chunk_mem(
device=None,
safety_fraction: float = 0.2,
min_gb: float = 0.25,
max_gb: float = 2.0,
):
"""Pick a chunk-memory budget (in GB) for sparse conv batching."""
free_gb = comfy.model_management.get_free_memory(device) / (1024 ** 3)
return max(min_gb, min(free_gb * safety_fraction, max_gb))
def sparse_submanifold_conv3d(
feats: torch.Tensor,
coords: torch.Tensor,
shape: tuple,
weight: torch.Tensor,
bias: Optional[torch.Tensor],
neighbor_cache: Optional[torch.Tensor],
dilation: tuple,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
if feats.shape[0] == 0:
Co = weight.shape[0]
return torch.empty((0, Co), device=feats.device, dtype=feats.dtype), None
W, H, D = shape
Co, Kw, Kh, Kd, Ci = weight.shape
V = Kw * Kh * Kd
device = feats.device
if neighbor_cache is None:
b_stride = W * H * D
x_stride = H * D
y_stride = D
z_stride = 1
flat_keys = (coords[:, 0].long() * b_stride +
coords[:, 1].long() * x_stride +
coords[:, 2].long() * y_stride +
coords[:, 3].long() * z_stride)
vals = torch.arange(coords.shape[0], dtype=torch.int32, device=device)
hashmap = TorchHashMap(flat_keys, vals)
neighbor = build_submanifold_neighbor_map(
hashmap, coords, W, H, D, Kw, Kh, Kd,
dilation[0], dilation[1], dilation[2]
)
else:
neighbor = neighbor_cache
N_pts = feats.shape[0]
weight_T = weight.view(Co, V * Ci).T
output = torch.empty(N_pts, Co, device=device, dtype=feats.dtype)
# Zero row at index N_pts; missing neighbors (-1) gather it -> no separate masking.
feats_padded = torch.cat([feats, feats.new_zeros(1, Ci)], dim=0)
# Chunk over voxels to bound the (chunk, V, Ci) gather.
max_chunk_mem_gb = get_recommended_chunk_mem(device)
mem_per_row = V * Ci * feats.element_size()
max_chunk_mem = max_chunk_mem_gb * (1024 ** 3)
chunk_size = max(1, int(max_chunk_mem / mem_per_row))
chunk_size = min(chunk_size, N_pts)
for start in range(0, N_pts, chunk_size):
end = min(start + chunk_size, N_pts)
actual_chunk = end - start
chunk_idx = torch.where(neighbor[start:end] < 0, N_pts, neighbor[start:end]) # -1 -> zero row
gathered = feats_padded[chunk_idx] # (chunk, V, Ci)
gathered_flat = gathered.view(actual_chunk, V * Ci)
torch.matmul(gathered_flat, weight_T, out=output[start:end]) # (chunk, V*Ci) @ (V*Ci, Co)
if bias is not None:
output += bias.unsqueeze(0).to(output.dtype)
return output, neighbor

1150
comfy/ldm/trellis2/model.py Normal file

File diff suppressed because it is too large Load Diff

1145
comfy/ldm/trellis2/vae.py Normal file

File diff suppressed because it is too large Load Diff

View File

@ -61,6 +61,7 @@ import comfy.ldm.ideogram4.model
import comfy.ldm.krea2.model
import comfy.ldm.kandinsky5.model
import comfy.ldm.anima.model
import comfy.ldm.trellis2.model
import comfy.ldm.ace.ace_step15
import comfy.ldm.cogvideo.model
import comfy.ldm.rt_detr.rtdetr_v4
@ -1853,6 +1854,27 @@ class WAN22(WAN21):
def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):
return latent_image
class Trellis2(BaseModel):
def __init__(self, model_config, model_type=ModelType.FLOW, device=None, unet_model=comfy.ldm.trellis2.model.Trellis2):
super().__init__(model_config, model_type, device, unet_model)
def extra_conds(self, **kwargs):
out = super().extra_conds(**kwargs)
embeds = kwargs.get("embeds")
out["embeds"] = comfy.conds.CONDRegular(embeds)
# CONDConstant: shared across pos/neg
for k in ("trellis2_coords", "trellis2_coord_counts",
"trellis2_generation_mode", "trellis2_shape_slat",
"trellis2_proj_feats", "trellis2_model_frame"):
v = kwargs.get(k)
if v is not None:
out[k] = comfy.conds.CONDConstant(v)
# Pixal3D's per-stage feature maps + camera params travel as a dict
proj_feat_pack = kwargs.get("proj_feat_pack")
if proj_feat_pack is not None:
out["proj_feat_pack"] = comfy.conds.CONDConstant(proj_feat_pack)
return out
class WAN21_FlowRVS(WAN21):
def __init__(self, model_config, model_type=ModelType.IMG_TO_IMG_FLOW, image_to_video=False, device=None):
model_config.unet_config["model_type"] = "t2v"

View File

@ -113,6 +113,25 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
unet_config['block_repeat'] = [[1, 1, 1, 1], [2, 2, 2, 2]]
return unet_config
shape_key = '{}img2shape.t_embedder.mlp.0.weight'.format(key_prefix)
tex_key = '{}shape2txt.t_embedder.mlp.0.weight'.format(key_prefix)
if shape_key in state_dict_keys or tex_key in state_dict_keys: # trellis2 / pixal3d
has_shape = shape_key in state_dict_keys
has_tex = tex_key in state_dict_keys
unet_config = {
"image_model": "trellis2",
"resolution": 32 if (metadata is not None and "is_512" in metadata) else 64,
"init_txt_model": has_tex,
"txt_only": has_tex and not has_shape,
}
# Per-submodel projection head (Pixal3D adds `proj_linear`; Trellis2 doesn't).
for sub, name in (("img2shape", "shape"), ("shape2txt", "texture"), ("structure_model", "structure")):
key = '{}{}.blocks.0.cross_attn.proj_linear.weight'.format(key_prefix, sub)
if key in state_dict_keys:
unet_config["image_attn_mode_{}".format(name)] = "proj"
unet_config["proj_in_channels_{}".format(name)] = int(state_dict[key].shape[1])
return unet_config
if '{}transformer.rotary_pos_emb.inv_freq'.format(key_prefix) in state_dict_keys: #stable audio dit
unet_config = {}
unet_config["audio_model"] = "dit1.0"

View File

@ -14,6 +14,7 @@ import comfy.ldm.lightricks.vae.causal_video_autoencoder
import comfy.ldm.lightricks.vae.audio_vae
import comfy.ldm.cosmos.vae
import comfy.ldm.wan.vae
import comfy.ldm.trellis2.vae
import comfy.ldm.wan.vae2_2
import comfy.ldm.hunyuan3d.vae
import comfy.ldm.triposplat.vae
@ -543,6 +544,16 @@ class VAE:
self.first_stage_model = StageC_coder()
self.downscale_ratio = 32
self.latent_channels = 16
elif "shape_dec.blocks.1.16.to_subdiv.weight" in sd: # trellis2 shape vae (struct_dec + shape_dec)
self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]
self.memory_used_decode = lambda shape, dtype: (2500 * shape[2] * shape[3]) * model_management.dtype_size(dtype)
self.memory_used_encode = lambda shape, dtype: (2500 * shape[2] * shape[3]) * model_management.dtype_size(dtype)
self.first_stage_model = comfy.ldm.trellis2.vae.ShapeVae()
elif "txt_dec.blocks.3.4.conv2.weight" in sd: # trellis2 texture vae
self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]
self.memory_used_decode = lambda shape, dtype: (2500 * shape[2] * shape[3]) * model_management.dtype_size(dtype)
self.memory_used_encode = lambda shape, dtype: (2500 * shape[2] * shape[3]) * model_management.dtype_size(dtype)
self.first_stage_model = comfy.ldm.trellis2.vae.TextureVae()
elif "decoder.conv_in.weight" in sd:
if sd['decoder.conv_in.weight'].shape[1] == 64:
ddconfig = {"block_out_channels": [128, 256, 512, 512, 1024, 1024], "in_channels": 3, "out_channels": 3, "num_res_blocks": 2, "ffactor_spatial": 32, "downsample_match_channel": True, "upsample_match_channel": True}
@ -1101,6 +1112,15 @@ class VAE:
pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1)
return pixel_samples
def prepare_decode(self, sample_shape, memory_required=None):
"""For VAEs whose real decode entry point bypasses decode()"""
if memory_required is None:
memory_required = self.memory_used_decode(sample_shape, self.vae_dtype)
memory_required = max(1, int(memory_required))
model_management.load_models_gpu([self.patcher], memory_required=memory_required, force_full_load=self.disable_offload)
free_memory = self.patcher.get_free_memory(self.device)
return max(1, int(free_memory / memory_required))
def decode_tiled(self, samples, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):
self.throw_exception_if_invalid()
memory_used = self.memory_used_decode(samples.shape, self.vae_dtype) #TODO: calculate mem required for tile

View File

@ -1432,6 +1432,31 @@ class WAN22_T2V(WAN21_T2V):
out = model_base.WAN22(self, image_to_video=True, device=device)
return out
class Trellis2(supported_models_base.BASE):
unet_config = {
"image_model": "trellis2"
}
unet_extra_config = {"num_heads": 12}
sampling_settings = {
"shift": 3.0,
}
memory_usage_factor = 3.5
latent_format = latent_formats.Trellis2
vae_key_prefix = ["vae."]
clip_vision_prefix = "conditioner.main_image_encoder.model."
# this is only needed for the texture model
supported_inference_dtypes = [torch.bfloat16, torch.float32]
def get_model(self, state_dict, prefix="", device=None):
return model_base.Trellis2(self, device=device)
def clip_target(self, state_dict={}):
return None
class WAN21_FlowRVS(WAN21_T2V):
unet_config = {
"image_model": "wan2.1",
@ -2369,5 +2394,6 @@ models = [
CogVideoX_I2V,
CogVideoX_T2V,
SVD_img2vid,
Trellis2,
DepthAnything3,
]

View File

@ -7,9 +7,10 @@ import torch
class VOXEL:
def __init__(self, data: torch.Tensor):
def __init__(self, data: torch.Tensor, voxel_colors=None, resolution=None):
self.data = data
self.voxel_colors = voxel_colors
self.resolution = resolution # each 3d model has its own resolution
class SPLAT:
"""A batch of 3D Gaussian splats in render-ready (activated, world-space) form.
@ -34,9 +35,16 @@ class MESH:
uvs: torch.Tensor | None = None,
vertex_colors: torch.Tensor | None = None,
texture: torch.Tensor | None = None,
metallic_roughness: torch.Tensor | None = None,
vertex_counts: torch.Tensor | None = None,
face_counts: torch.Tensor | None = None,
unlit: bool = False):
unlit: bool = False,
normals: torch.Tensor | None = None,
tangents: torch.Tensor | None = None,
normal_map: torch.Tensor | None = None,
occlusion_in_mr: bool = False,
material: dict | None = None,
emissive: torch.Tensor | None = None):
assert (vertex_counts is None) == (face_counts is None), \
"vertex_counts and face_counts must be provided together (both or neither)"
@ -44,13 +52,25 @@ class MESH:
self.faces = faces # faces: (B, M, 3)
self.uvs = uvs # uvs: (B, N, 2)
self.vertex_colors = vertex_colors # vertex_colors: (B, N, 3 or 4)
self.texture = texture # texture: (B, H, W, 3)
# Optional per-vertex normals: (B, N, 3). When None, SaveGLB computes smooth
# area-weighted normals so viewers don't fall back to flat (per-face) shading.
self.normals = normals
self.texture = texture # texture (baseColor): (B, H, W, 3)
# glTF metallicRoughness texture: (B, H, W, 3), R unused, G=roughness, B=metallic
self.metallic_roughness = metallic_roughness
# When vertices/faces are zero-padded to a common N/M across the batch (variable-size mesh batch),
# these hold the real per-item lengths (B,). None means rows are uniform and no slicing is needed.
self.vertex_counts = vertex_counts
self.face_counts = face_counts
# Render flat / emissive (no scene lighting) when saved, e.g. for gaussian-splat-derived meshes.
self.unlit = unlit
# Extra maps / material overrides attached by bake, normal/AO, and SetMeshMaterial nodes;
# consumed by SaveGLB. Declared here (with defaults) so consumers read them directly.
self.tangents = tangents # (B, N, 4) per-vertex tangents for normal mapping
self.normal_map = normal_map # tangent-space normal map: (B, H, W, 3)
self.occlusion_in_mr = occlusion_in_mr # True = R channel of metallic_roughness holds AO (ORM)
self.material = material # SetMeshMaterial scalar/factor overrides
self.emissive = emissive # emissive map: (B, H, W, 3)
class File3D:

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,162 @@
"""Mesh container, edge/face adjacency, manifold cleanup."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Dict, List
import numpy as np
import torch
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import connected_components
from torch import Tensor
# ---- Per-face / per-vertex geometry ----
def face_normals(vertices: Tensor, faces: Tensor) -> Tensor:
"""[F,3] unit face normals (degenerate faces -> zero)."""
v0 = vertices[faces[:, 0]]
v1 = vertices[faces[:, 1]]
v2 = vertices[faces[:, 2]]
n = torch.linalg.cross(v1 - v0, v2 - v0)
return n / n.norm(dim=1, keepdim=True).clamp_min(1e-20)
def face_areas(vertices: Tensor, faces: Tensor) -> Tensor:
"""[F] triangle areas."""
v0 = vertices[faces[:, 0]]
v1 = vertices[faces[:, 1]]
v2 = vertices[faces[:, 2]]
return 0.5 * torch.linalg.cross(v1 - v0, v2 - v0).norm(dim=1)
def face_centroids(vertices: Tensor, faces: Tensor) -> Tensor:
"""[F,3] triangle centroids."""
return vertices[faces].mean(dim=1)
def face_edge_lengths(vertices: Tensor, faces: Tensor) -> Tensor:
"""[F,3] edge lengths; column e = |v[faces[:,e]] - v[faces[:,(e+1)%3]]|."""
va = vertices[faces]
vb = vertices[faces.roll(shifts=-1, dims=1)]
return (vb - va).norm(dim=-1).to(torch.float32)
def chart_3d_areas(face_area: Tensor, face_chart: Tensor, n_charts: int) -> Tensor:
"""[n_charts] sum of face areas per chart."""
out = torch.zeros(n_charts, dtype=face_area.dtype, device=face_area.device)
out.scatter_add_(0, face_chart, face_area)
return out
@dataclass
class MeshData:
"""Cleaned mesh with adjacency; face_face[f, i] = face sharing edge (faces[f,i], faces[f,(i+1)%3]) or -1 if boundary."""
vertices: Tensor # [V, 3] float
faces: Tensor # [F, 3] long
face_face: Tensor # [F, 3] long, neighbor face id or -1
face_normal: Tensor # [F, 3] float
face_area: Tensor # [F] float
face_centroid: Tensor # [F, 3] float
component: Tensor # [F] long, connected-component id
n_components: int
def build_mesh(vertices: Tensor, faces: Tensor) -> MeshData:
"""Build adjacency; non-manifold edges (>2 incident faces) get no neighbor and act as boundary."""
if vertices.dtype != torch.float32:
vertices = vertices.to(torch.float32)
if faces.dtype != torch.long:
faces = faces.to(torch.long)
device = faces.device
V = vertices.shape[0]
F = faces.shape[0]
# Per directed face-edge; flat layout p = f*3+i.
a = faces.flatten()
b = faces.roll(shifts=-1, dims=1).flatten()
lo = torch.minimum(a, b)
hi = torch.maximum(a, b)
edge_key = lo * (V + 1) + hi
# Pair manifold (count==2) face-edges; others get no neighbor.
_, inverse, counts = torch.unique(edge_key, return_inverse=True, return_counts=True)
edge_count = counts[inverse]
manifold_mask = edge_count == 2
sort_idx = torch.argsort(edge_key, stable=True)
sorted_manifold = manifold_mask[sort_idx]
pair_positions = sort_idx[sorted_manifold]
pair_a = pair_positions[0::2]
pair_b = pair_positions[1::2]
face_id_flat = torch.arange(F, device=device).repeat_interleave(3)
face_face_flat = torch.full((3 * F,), -1, dtype=torch.long, device=device)
face_face_flat[pair_a] = face_id_flat[pair_b]
face_face_flat[pair_b] = face_id_flat[pair_a]
face_face = face_face_flat.view(F, 3)
face_face_np = face_face.cpu().numpy()
rows_mask = face_face_np >= 0
if rows_mask.any():
rows = np.broadcast_to(np.arange(F)[:, None], (F, 3))[rows_mask]
cols = face_face_np[rows_mask]
adj = csr_matrix(
(np.ones(rows.size, dtype=np.int8), (rows, cols)),
shape=(F, F),
)
else:
adj = csr_matrix((F, F), dtype=np.int8)
n_components, labels = connected_components(adj, directed=False)
face_normal = face_normals(vertices, faces)
face_area = face_areas(vertices, faces)
face_centroid = face_centroids(vertices, faces)
return MeshData(
vertices=vertices,
faces=faces,
face_face=face_face,
face_normal=face_normal,
face_area=face_area,
face_centroid=face_centroid,
component=torch.from_numpy(labels.astype(np.int64)).to(device),
n_components=int(n_components),
)
def chart_boundary_loops(
faces_subset: Tensor, face_face_subset: Tensor
) -> List[List[int]]:
"""Return ordered boundary vertex loops for a chart submesh (face_face_subset[f,i]==-1 marks a boundary edge)."""
F = faces_subset.shape[0]
faces_np = faces_subset.cpu().numpy()
ff = face_face_subset.cpu().numpy()
next_v: Dict[int, int] = {}
for f in range(F):
for i in range(3):
if ff[f, i] == -1:
a = int(faces_np[f, i])
b = int(faces_np[f, (i + 1) % 3])
next_v[a] = b
loops: List[List[int]] = []
visited = set()
for start in list(next_v.keys()):
if start in visited:
continue
loop = [start]
visited.add(start)
cur = next_v.get(start)
while cur is not None and cur != start:
if cur in visited:
break
loop.append(cur)
visited.add(cur)
cur = next_v.get(cur)
if len(loop) >= 3:
loops.append(loop)
return loops

View File

@ -0,0 +1,861 @@
"""Atlas packing via bitmap rasterize-and-place."""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Tuple
import numpy as np
import torch
from torch import Tensor
from torch.nn.functional import max_pool1d
import comfy.model_management
# Numba is optional, but ~5x faster than torch on these operations, potential TODO: comfy-kitchen cuda/triton kernels as even faster alternative
try:
from numba import njit as _njit, prange as _prange, get_num_threads as _nb_threads
_HAVE_NUMBA_PACK = True
except ImportError:
_HAVE_NUMBA_PACK = False
_prange = range
def _nb_threads(): return 1
def _njit(*args, **kwargs):
def deco(fn): return fn
return deco if not args else args[0]
# Cap on deterministic sweep density: tiny charts on a large atlas would otherwise enumerate every texel column.
_SWEEP_CAP = 1024
@dataclass
class ChartPlacement:
chart_id: int
offset: Tuple[float, float] # in texels
scale: float # texels per UV unit
rotation: float = 0.0 # radians
swap_xy: bool = False # extra 90° bitmap rotation chosen at place time
chart_h: float = 0.0 # unswapped bitmap height in texels (rotation pivot)
@_njit(cache=True, boundscheck=False, parallel=True)
def _prepare_dims_jit(uvs, uv_off, a3, auv, tpu, padding, theta, scale, bw, bh, rot_uv):
"""Pass 1: per-chart best rotation, texel scale, rotated/scaled UVs, padded bitmap dims."""
n = uv_off.shape[0] - 1
half_pi = math.pi * 0.5
for c in _prange(n):
v0, v1 = uv_off[c], uv_off[c + 1]
best_area = 1e30
best_t = 0.0
for k in range(36):
th = half_pi * k / 36.0
co = math.cos(th)
si = math.sin(th)
xmin = 1e30
xmax = -1e30
ymin = 1e30
ymax = -1e30
for i in range(v0, v1):
xr = uvs[i, 0] * co - uvs[i, 1] * si
yr = uvs[i, 0] * si + uvs[i, 1] * co
if xr < xmin:
xmin = xr
if xr > xmax:
xmax = xr
if yr < ymin:
ymin = yr
if yr > ymax:
ymax = yr
area = (xmax - xmin) * (ymax - ymin)
if area < best_area:
best_area = area
best_t = th
theta[c] = best_t
co = math.cos(best_t)
si = math.sin(best_t)
xmin = 1e30
xmax = -1e30
ymin = 1e30
ymax = -1e30
for i in range(v0, v1):
xr = uvs[i, 0] * co - uvs[i, 1] * si
yr = uvs[i, 0] * si + uvs[i, 1] * co
rot_uv[i, 0] = xr
rot_uv[i, 1] = yr
if xr < xmin:
xmin = xr
if xr > xmax:
xmax = xr
if yr < ymin:
ymin = yr
if yr > ymax:
ymax = yr
if v1 == v0:
xmin = 0.0
xmax = 0.0
ymin = 0.0
ymax = 0.0
s = math.sqrt(max(a3[c], 1e-12) / max(auv[c], 1e-12)) * tpu
nominal = math.sqrt(max(a3[c], 1e-12)) * tpu
max_bbox = max(8.0, 4.0 * nominal)
bbox_max = max(max(xmax - xmin, ymax - ymin), 1e-12)
if s * bbox_max > max_bbox:
s = max_bbox / bbox_max
scale[c] = s
wmax = 0.0
hmax = 0.0
for i in range(v0, v1):
rot_uv[i, 0] = (rot_uv[i, 0] - xmin) * s
rot_uv[i, 1] = (rot_uv[i, 1] - ymin) * s
if rot_uv[i, 0] > wmax:
wmax = rot_uv[i, 0]
if rot_uv[i, 1] > hmax:
hmax = rot_uv[i, 1]
bw[c] = int(math.ceil(wmax)) + padding + 1
bh[c] = int(math.ceil(hmax)) + padding + 1
@_njit(cache=True, boundscheck=False, parallel=True)
def _raster_all_jit(rot_uv, uv_off, faces, f_off, bw, bh, boff, buf, padding,
tw, th_out, perim):
"""Pass 2: rasterize + dilate each chart into the flat buffer; records trimmed dims
(origin kept) and the perimeter used for placement ordering."""
n = uv_off.shape[0] - 1
eps = 1e-7
for c in _prange(n):
f0, f1 = f_off[c], f_off[c + 1]
v0 = uv_off[c]
V = uv_off[c + 1] - v0
w = bw[c]
h = bh[c]
o = boff[c]
for fi in range(f0, f1):
i0 = faces[fi, 0] + v0
i1 = faces[fi, 1] + v0
i2 = faces[fi, 2] + v0
x0 = rot_uv[i0, 0]
y0 = rot_uv[i0, 1]
x1 = rot_uv[i1, 0]
y1 = rot_uv[i1, 1]
x2 = rot_uv[i2, 0]
y2 = rot_uv[i2, 1]
xmin_f = min(x0, min(x1, x2))
xmax_f = max(x0, max(x1, x2))
ymin_f = min(y0, min(y1, y2))
ymax_f = max(y0, max(y1, y2))
xmin = max(int(math.floor(xmin_f)), 0)
xmax = min(int(math.ceil(xmax_f)), w - 1)
ymin = max(int(math.floor(ymin_f)), 0)
ymax = min(int(math.ceil(ymax_f)), h - 1)
if xmax < xmin or ymax < ymin:
continue
denom = (y1 - y2) * (x0 - x2) + (x2 - x1) * (y0 - y2)
if abs(denom) < 1e-20:
continue
inv_denom = 1.0 / denom
for py in range(ymin, ymax + 1):
yc = py + 0.5
for px in range(xmin, xmax + 1):
xc = px + 0.5
aa = ((y1 - y2) * (xc - x2) + (x2 - x1) * (yc - y2)) * inv_denom
bb = ((y2 - y0) * (xc - x2) + (x0 - x2) * (yc - y2)) * inv_denom
cc = 1.0 - aa - bb
if aa >= -eps and bb >= -eps and cc >= -eps:
buf[o + py * w + px] = True
# Manhattan dilation by `padding` steps (ping-pong on a scratch copy)
if padding > 0 and f1 > f0:
tmp = np.empty(h * w, dtype=np.bool_)
for _ in range(padding):
for j in range(h * w):
tmp[j] = buf[o + j]
for py in range(h):
for px in range(w):
if tmp[py * w + px]:
continue
hit = False
if py > 0 and tmp[(py - 1) * w + px]:
hit = True
elif py < h - 1 and tmp[(py + 1) * w + px]:
hit = True
elif px > 0 and tmp[py * w + px - 1]:
hit = True
elif px < w - 1 and tmp[py * w + px + 1]:
hit = True
if hit:
buf[o + py * w + px] = True
# trimmed dims (keep origin; 1x1 empty bitmap when nothing was rasterized)
rmax = -1
cmax = -1
for py in range(h):
for px in range(w):
if buf[o + py * w + px]:
if py > rmax:
rmax = py
if px > cmax:
cmax = px
if rmax < 0:
for j in range(h * w):
buf[o + j] = False
tw[c] = 1
th_out[c] = 1
else:
tw[c] = cmax + 1
th_out[c] = rmax + 1
# unique-edge perimeter via sorted int64 keys
Fc = f1 - f0
if Fc > 0 and V > 0:
keys = np.empty(Fc * 3, dtype=np.int64)
for fi in range(f0, f1):
for j in range(3):
a = faces[fi, j]
b = faces[fi, (j + 1) % 3]
if a < b:
keys[(fi - f0) * 3 + j] = a * V + b
else:
keys[(fi - f0) * 3 + j] = b * V + a
keys = np.sort(keys)
p = 0.0
for i in range(keys.shape[0]):
if i > 0 and keys[i] == keys[i - 1]:
continue
a = keys[i] // V + v0
b = keys[i] % V + v0
dx = rot_uv[a, 0] - rot_uv[b, 0]
dy = rot_uv[a, 1] - rot_uv[b, 1]
p += math.sqrt(dx * dx + dy * dy)
perim[c] = p
@_njit(cache=True, boundscheck=False, parallel=True)
def _place_all_jit(buf, boff, stride_w, tw, th, order, start, stop,
atlas, skyline, pool, attempts, sweep_cap, margin,
n_threads, cur_wh, out_x, out_y, out_sw):
"""Place charts order[start:stop]; returns the first index NOT processed (== stop when
done, earlier when the atlas must grow the caller resizes and resumes). The candidate
scan is striped with a (score, index) min-reduction: deterministic for any thread count,
and no thread intrinsics (dynamic globals would defeat cache=True)."""
aw = atlas.shape[1]
ah = atlas.shape[0]
cur_w = cur_wh[0]
cur_h = cur_wh[1]
n_pool = pool.shape[0]
big = np.int64(1) << 62
nt = n_threads
t_score = np.empty(nt, dtype=np.int64)
t_k = np.empty(nt, dtype=np.int64)
t_x = np.empty(nt, dtype=np.int64)
t_y = np.empty(nt, dtype=np.int64)
t_sw = np.empty(nt, dtype=np.int64)
for oi in range(start, stop):
ci = order[oi]
if cur_h + margin > ah or cur_w + margin > aw:
cur_wh[0] = cur_w
cur_wh[1] = cur_h
return oi
w0 = tw[ci] # unswapped trimmed dims
h0 = th[ci]
W = stride_w[ci] # row stride of the untrimmed block
o = boff[ci]
step = min(w0, h0) // 8
if step < 1:
step = 1
cap_step = max(cur_w, cur_h) // sweep_cap
if cap_step > step:
step = cap_step
poff = (oi * attempts) % (n_pool - attempts + 1)
x_range = cur_w + 1 if cur_w > 0 else 1
y_range = cur_h + 1 if cur_h > 0 else 1
# candidate groups per orientation: skyline-flush sweep, y=0 / y=cur_h sweeps,
# x=0 / x=cur_w sweeps; then the shared random pool
nx = max(cur_w, 1) // step + 2
ny = max(cur_h, 1) // step + 2
n_det = nx * 3 + ny * 2
total = n_det * 2 + attempts
for t in range(nt):
t_score[t] = big
t_k[t] = big
for t2 in _prange(nt):
for k in range(t2, total, nt):
x = 0 # int inits and no body-level continue:
y = 0 # parfor lowering types undef-path
swap = 0 # variables as f64
valid = True
if k < 2 * n_det:
if k >= n_det:
swap = 1
kk = k - n_det if swap == 1 else k
cw = w0 if swap == 0 else h0
if kk < nx: # skyline-flush sweep
x = kk * step
if x > cur_w:
valid = False
else:
x_end = x + cw
if x_end > skyline.shape[0]:
x_end = skyline.shape[0]
for xs in range(x, x_end):
if skyline[xs] > y:
y = int(skyline[xs])
elif kk < 3 * nx: # y=0 and y=cur_h sweeps
kk2 = kk - nx
x = (kk2 % nx) * step
if x > cur_w:
valid = False
elif kk2 >= nx:
y = cur_h
else: # x=0 and x=cur_w sweeps
kk2 = kk - 3 * nx
if kk2 >= 2 * ny:
valid = False
else:
y = (kk2 % ny) * step
if y > cur_h:
valid = False
elif kk2 >= ny:
x = cur_w
else:
r = k - 2 * n_det
x = int(pool[poff + r, 0] % x_range)
y = int(pool[poff + r, 1] % y_range)
swap = int(r & 1)
if valid:
ch = h0 if swap == 0 else w0
cw = w0 if swap == 0 else h0
nw = cur_w if cur_w > x + cw else x + cw
nh = cur_h if cur_h > y + ch else y + ch
ext = nw if nw > nh else nh
score = ext * ext + nw * nh
if score < t_score[t2] or (score == t_score[t2] and k < t_k[t2]):
ok = True
for j in range(ch):
yy = int(y + j)
if yy >= ah:
continue
for i in range(cw):
if swap == 0:
bit = buf[o + j * W + i]
else:
# 90deg rotation: bm_rot[j, i] = bm[h0-1-i, j]
bit = buf[o + (h0 - 1 - i) * W + j]
if not bit:
continue
xx = int(x + i)
if xx >= aw:
continue
if atlas[yy, xx]:
ok = False
break
if not ok:
break
if ok:
t_score[t2] = score
t_k[t2] = k
t_x[t2] = x
t_y[t2] = y
t_sw[t2] = swap
best_x = -1
best_y = -1
best_swap = 0
bs = big
bk = big
for t in range(nt):
if t_score[t] < bs or (t_score[t] == bs and t_k[t] < bk):
bs = t_score[t]
bk = t_k[t]
best_x = t_x[t]
best_y = t_y[t]
best_swap = t_sw[t]
if best_x < 0: # fallback: extension corner
best_x = cur_w
best_y = 0
best_swap = 0
bh_ = h0 if best_swap == 0 else w0
bw_ = w0 if best_swap == 0 else h0
# blit + extents + skyline lift
for j in range(bh_):
for i in range(bw_):
if best_swap == 0:
bit = buf[o + j * W + i]
else:
bit = buf[o + (h0 - 1 - i) * W + j]
if bit:
atlas[best_y + j, best_x + i] = True
if best_x + bw_ > cur_w:
cur_w = best_x + bw_
if best_y + bh_ > cur_h:
cur_h = best_y + bh_
for i in range(bw_):
col_x = best_x + i
if col_x >= skyline.shape[0]:
continue
col_top = -1
for j in range(bh_ - 1, -1, -1):
if best_swap == 0:
bit = buf[o + j * W + i]
else:
bit = buf[o + (h0 - 1 - i) * W + j]
if bit:
col_top = j
break
if col_top >= 0:
nh2 = best_y + col_top + 1
if nh2 > skyline[col_x]:
skyline[col_x] = nh2
out_x[ci] = best_x
out_y[ci] = best_y
out_sw[ci] = best_swap
cur_wh[0] = cur_w
cur_wh[1] = cur_h
return stop
# Torch fallback (used when numba is unavailable; runs on GPU if present)
def _dilate_local(x: Tensor, p: int) -> Tensor:
"""4-connectivity dilation by p over a batch of (cnt,g,g) bitmaps. Dilation distributes
over union, so dilating per-triangle then OR-scattering equals dilating the chart."""
for _ in range(p):
y = x.clone()
y[:, 1:, :] |= x[:, :-1, :]
y[:, :-1, :] |= x[:, 1:, :]
y[:, :, 1:] |= x[:, :, :-1]
y[:, :, :-1] |= x[:, :, 1:]
x = y
return x
def _raster_all_torch(uvs_tex_pad, faces_pad, fmask, bw_t, bh_t, padding, device):
"""Rasterize every chart into one flat bool buffer; buf[cbase[i]:cbase[i+1]].view(bh,bw)
is chart i's bitmap. Triangles are bucketed by next-pow2 bbox size to bound memory."""
n = uvs_tex_pad.shape[0]
fmax = faces_pad.shape[1]
bwL, bhL = bw_t.long(), bh_t.long()
cbase = torch.zeros(n + 1, dtype=torch.long, device=device)
torch.cumsum(bwL * bhL, 0, out=cbase[1:])
buf = torch.zeros(int(cbase[-1].item()), dtype=torch.bool, device=device)
# gather all triangle coords, keep only valid faces -> (Ttot,3,2) + chart id per triangle
fp = faces_pad.reshape(n, fmax * 3)
tri = torch.gather(uvs_tex_pad, 1, fp[..., None].expand(-1, -1, 2)).reshape(n * fmax, 3, 2)
fm = fmask.reshape(-1)
tri_f = tri[fm]
if tri_f.shape[0] == 0:
return buf, cbase
cid = torch.arange(n, device=device).repeat_interleave(fmax)[fm]
# per-triangle pixel bbox, inflated by padding (origin >= 0); bucket by next-pow2 max-dim
tmin = tri_f.amin(1)
tmax = tri_f.amax(1)
x0 = (tmin[:, 0].floor().long() - padding).clamp_min(0)
y0 = (tmin[:, 1].floor().long() - padding).clamp_min(0)
bbw = (tmax[:, 0].ceil().long() + padding) - x0 + 1
bbh = (tmax[:, 1].ceil().long() + padding) - y0 + 1
mxd = torch.maximum(bbw, bbh).clamp_min(1)
bsz = (2 ** torch.ceil(torch.log2(mxd.float())).long()).long()
a = tri_f[:, 0]
b = tri_f[:, 1]
c = tri_f[:, 2]
v0 = b - a
v1 = c - a
d00 = (v0 * v0).sum(-1)
d01 = (v0 * v1).sum(-1)
d11 = (v1 * v1).sum(-1)
den = (d00 * d11 - d01 * d01).clamp(min=1e-20)
for g in sorted(set(bsz.tolist())): # one batch per pow2 grid
sel = (bsz == g).nonzero(as_tuple=True)[0]
m = sel.shape[0]
xs0 = x0[sel].view(m, 1, 1)
ys0 = y0[sel].view(m, 1, 1)
cc = cid[sel]
bwp = bwL[cc].view(m, 1, 1)
bhp = bhL[cc].view(m, 1, 1)
gi = torch.arange(g, device=device)
px = xs0 + gi.view(1, 1, g)
py = ys0 + gi.view(1, g, 1) # (m,g,g) int
pxf = px.float() + 0.5
pyf = py.float() + 0.5
v2x = pxf - a[sel, 0].view(m, 1, 1)
v2y = pyf - a[sel, 1].view(m, 1, 1)
d20 = v2x * v0[sel, 0].view(m, 1, 1) + v2y * v0[sel, 1].view(m, 1, 1)
d21 = v2x * v1[sel, 0].view(m, 1, 1) + v2y * v1[sel, 1].view(m, 1, 1)
idn = den[sel].view(m, 1, 1).reciprocal()
vv = torch.addcmul(d11[sel].view(m, 1, 1) * d20, d01[sel].view(m, 1, 1), d21, value=-1) * idn
ww = torch.addcmul(d00[sel].view(m, 1, 1) * d21, d01[sel].view(m, 1, 1), d20, value=-1) * idn
uu = 1.0 - vv - ww
inside = (uu >= -1e-6) & (vv >= -1e-6) & (ww >= -1e-6)
if padding > 0:
inside = _dilate_local(inside, padding)
valid = inside & (px < bwp) & (py < bhp)
flat = (cbase[cc].view(m, 1, 1) + py * bwp + px)[valid]
buf[flat] = True
return buf, cbase
def _build_candidates_gpu(sky_t, ar, cur_w, cur_h, bw0, bw1, step, rand01, device):
"""Candidate (x, y) positions as a (2, M, 2) tensor (dim 0 = orientation). The first
n_sky rows per orientation are skyline-flush and collision-free by construction.
rand01 is (2, rand_n, 2) pre-drawn uniforms; ar a preallocated arange."""
hi_x = max(cur_w, 1) + 1
hi_y = max(cur_h, 1) + 1
xs = ar[0:hi_x:step]
ys = ar[0:hi_y:step]
n_sky = (hi_x + step - 1) // step
zx = torch.zeros_like(xs)
zy = torch.zeros_like(ys)
common = torch.cat([
torch.stack([xs, zx], 1), torch.stack([xs, zx + cur_h], 1),
torch.stack([zy, ys], 1), torch.stack([zy + cur_w, ys], 1)])
wm = []
for cw in (bw0, bw1):
span = (n_sky - 1) * step + cw
wm.append(max_pool1d(sky_t[:span].view(1, 1, -1).float(), kernel_size=cw,
stride=step).view(-1))
sky = torch.stack([torch.stack([xs, wm[0].long()], 1),
torch.stack([xs, wm[1].long()], 1)])
lim = torch.tensor([hi_x, hi_y], dtype=rand01.dtype, device=device)
rnd = (rand01 * lim).long()
return torch.cat([sky, common.expand(2, -1, -1), rnd], 1), n_sky
def _best_placement_torch(atlas, pix0, dim0, dim1, cands, n_sky, cur_w, cur_h, device):
"""Lowest-score non-colliding placement as a (3,) int tensor [x, y, swap]. The best
skyline candidate bounds the score; only strictly better candidates are pixel-tested."""
m = cands.shape[1]
chw = torch.tensor([[dim0[0], dim0[1]], [dim1[0], dim1[1]]], device=device)
nw = torch.clamp(cands[..., 0] + chw[:, 1:], min=cur_w) # (2,M)
nh = torch.clamp(cands[..., 1] + chw[:, :1], min=cur_h)
ext = torch.maximum(nw, nh)
sc = ext * ext + nw * nh
js = sc[:, :n_sky].reshape(-1).argmin() # best skyline candidate
sky_o = js // n_sky
s_star = sc[:, :n_sky].reshape(-1)[js]
sky = torch.cat([cands[sky_o, js % n_sky], sky_o.reshape(1)])
cflat = cands.reshape(-1, 2)
surv = (sc.reshape(-1) < s_star).nonzero(as_tuple=True)[0] # compact once
total = surv.shape[0]
if total == 0:
return sky
k = pix0.shape[0]
if k == 0: # empty chart: anywhere free
j = surv[sc.reshape(-1)[surv].argmin()]
return torch.cat([cflat[j], (j // m).reshape(1)])
ordr = surv[torch.argsort(sc.reshape(-1)[surv], stable=True)]
# flattened-index collision test: one int32 gather index instead of two int64 rows/cols
aw = atlas.shape[1]
idt = torch.int32 if atlas.numel() < (1 << 31) else torch.long
lin0 = (pix0[:, 0] * aw + pix0[:, 1]).to(idt) # (y, x)
lin1 = (pix0[:, 1] * aw + (dim0[0] - 1 - pix0[:, 0])).to(idt) # rotated: (x, h-1-y)
linp = torch.stack([lin0, lin1])
aflat = atlas.view(-1)
og = (ordr >= m).long()
base = (cflat[ordr, 1] * aw + cflat[ordr, 0]).to(idt)
# prescreen survivors on ~128 strided pixels: a sampled hit proves collision, so only
# subsample-clean candidates need the exact test
stride = (k + 127) // 128
linp_sub = linp[:, ::stride].contiguous()
maybe = ~aflat[base[:, None] + linp_sub[og]].any(1)
passers = maybe.nonzero(as_tuple=True)[0] # ascending = score-sorted
npass = passers.shape[0]
if npass == 0:
return sky
if stride == 1: # prescreen was already exact
j = ordr[passers[0]]
return torch.cat([cflat[j], (j // m).reshape(1)])
budget = 1 << 22 # pixel-tests per chunk
start = 0
while start < npass:
take = max(1, budget // k)
pi = passers[start:start + take]
free = ~aflat[base[pi][:, None] + linp[og[pi]]].any(1) # (t,k) True-pixel gather
# single host read per chunk: whether a free hit exists and where
has, first = torch.stack([free.any().long(), free.long().argmax()]).tolist()
if has:
j = ordr[pi[first]] # lowest score: sorted order
return torch.cat([cflat[j], (j // m).reshape(1)])
start += take
budget = min(budget * 4, 1 << 25)
return sky
def _pack_bitmap_torch(chart_uvs, chart_3d_areas, chart_uv_areas, chart_faces,
texels_per_unit, padding_texels, attempts=4096, rng_seed=0,
progress_callback=None):
"""Torch rasterize-and-place packer (numba-free fallback). Returns (placements, atlas_w, atlas_h)."""
n = len(chart_uvs)
if n == 0:
return [], 1, 1
device = comfy.model_management.get_torch_device()
ang = torch.linspace(0.0, math.pi / 2.0, 37, device=device)[:-1]
cos_a, sin_a = ang.cos(), ang.sin()
# ---- Prepare pass 1: best-rotation + scale + bbox for ALL charts at once (batched) ----
vcount = [int(u.shape[0]) for u in chart_uvs]
fcount = [int(f.shape[0]) for f in chart_faces]
vmax = max(vcount)
fmax = max(fcount)
uvs_pad = torch.zeros(n, vmax, 2, device=device)
vmask = torch.zeros(n, vmax, dtype=torch.bool, device=device)
faces_pad = torch.zeros(n, fmax, 3, dtype=torch.long, device=device)
fmask = torch.zeros(n, fmax, dtype=torch.bool, device=device)
for i in range(n):
uvs_pad[i, :vcount[i]] = chart_uvs[i].to(device=device, dtype=torch.float32)
vmask[i, :vcount[i]] = True
if fcount[i]:
faces_pad[i, :fcount[i]] = chart_faces[i].to(device=device, dtype=torch.long)
fmask[i, :fcount[i]] = True
u0, u1 = uvs_pad[..., 0], uvs_pad[..., 1] # (N,Vmax)
BIG = 1e30
mlo = torch.where(vmask, torch.zeros_like(u0), u0.new_full((), BIG))
mhi = torch.where(vmask, torch.zeros_like(u0), u0.new_full((), -BIG))
xr = torch.addcmul(u0[:, :, None] * cos_a, u1[:, :, None], sin_a, value=-1) # (N,Vmax,A)
yr = torch.addcmul(u0[:, :, None] * sin_a, u1[:, :, None], cos_a)
xsp = (xr + mhi[:, :, None]).amax(1) - (xr + mlo[:, :, None]).amin(1) # (N,A) masked span
ysp = (yr + mhi[:, :, None]).amax(1) - (yr + mlo[:, :, None]).amin(1)
ti = (xsp * ysp).argmin(1) # (N,) best angle per chart
cc, ss = cos_a[ti][:, None], sin_a[ti][:, None] # (N,1)
rx = torch.addcmul(u0 * cc, u1, ss, value=-1) # (N,Vmax)
ry = torch.addcmul(u0 * ss, u1, cc)
rxmin = (rx + mlo).amin(1) # (N,)
rxmax = (rx + mhi).amax(1)
rymin = (ry + mlo).amin(1)
rymax = (ry + mhi).amax(1)
a3 = torch.tensor([max(a, 1e-12) for a in chart_3d_areas], device=device)
au = torch.tensor([max(a, 1e-12) for a in chart_uv_areas], device=device)
base = (a3 / au).sqrt() * texels_per_unit
maxb = (4.0 * a3.sqrt() * texels_per_unit).clamp_min(8.0)
bbm = torch.maximum(rxmax - rxmin, rymax - rymin).clamp_min(1e-12)
scale = torch.minimum(base, maxb / bbm) # (N,)
uvs_tex_pad = torch.stack([(rx - rxmin[:, None]) * scale[:, None],
(ry - rymin[:, None]) * scale[:, None]], dim=-1) # (N,Vmax,2)
bw_t = ((rxmax - rxmin) * scale).ceil().int() + padding_texels + 1
bh_t = ((rymax - rymin) * scale).ceil().int() + padding_texels + 1
# one sync: pull all per-chart scalars
thetas = ang[ti].cpu().tolist()
scales = scale.cpu().tolist()
# ---- Prepare pass 2: rasterize ALL charts at once, then derive per-chart sparse data ----
buf, cbase = _raster_all_torch(uvs_tex_pad, faces_pad, fmask, bw_t, bh_t, padding_texels, device)
# nonzero over the flat buffer is ascending, so pixels come out grouped by chart
nz = buf.nonzero(as_tuple=True)[0]
del buf
cid = torch.searchsorted(cbase, nz, right=True) - 1
bwl = bw_t.long()
local = nz - cbase[cid]
py = local // bwl[cid]
px = local - py * bwl[cid]
del nz, local
counts = torch.bincount(cid, minlength=n)
rmax = torch.full((n,), -1, dtype=torch.long, device=device)
cmax = torch.full((n,), -1, dtype=torch.long, device=device)
rmax.scatter_reduce_(0, cid, py, reduce="amax")
cmax.scatter_reduce_(0, cid, px, reduce="amax")
ht = (rmax + 1).clamp_min(1) # trimmed bitmap dims (1x1 when empty)
wt = (cmax + 1).clamp_min(1)
pix_all = torch.stack([py, px], 1) # True-pixel (row, col) offsets, sparse
pixr_all = torch.stack([px, rmax[cid] - py], 1) # 90deg rotation: (y, x) -> (x, h-1-y)
meta = torch.stack([ht, wt, counts.cumsum(0)], 1).cpu().tolist() # one sync for all charts
dim_l = [(m[0], m[1]) for m in meta]
dimr_l = [(w, h) for (h, w) in dim_l]
offs = [0] + [m[2] for m in meta]
pix_l = [pix_all[offs[i]:offs[i + 1]] for i in range(n)]
pixr_l = [pixr_all[offs[i]:offs[i + 1]] for i in range(n)]
# column tops (skyline lift), batched via flat scatter-amax over (chart, column) keys
wmax = max(max(h, w) for (h, w) in dim_l)
ct_pad = torch.full((n * wmax,), -1, dtype=torch.long, device=device)
ctr_pad = torch.full((n * wmax,), -1, dtype=torch.long, device=device)
ct_pad.scatter_reduce_(0, cid * wmax + px, py, reduce="amax")
ctr_pad.scatter_reduce_(0, cid * wmax + (rmax[cid] - py), px, reduce="amax")
ct_pad = ct_pad.view(n, wmax)
ctr_pad = ctr_pad.view(n, wmax)
del cid, py, px, rmax, cmax
# ---- Placement: skyline bin-pack on GPU ----
order = sorted(range(n), key=lambda i: -(dim_l[i][0] * dim_l[i][1])) # biggest bitmap first
max_b = max(max(d) for d in dim_l)
margin = max_b + 8
side_guess = int(math.sqrt(sum(d[0] * d[1] for d in dim_l)) * 2) + 16
cap = side_guess + margin
atlas = torch.zeros((cap, cap), dtype=torch.bool, device=device)
sky_t = torch.zeros(cap, dtype=torch.long, device=device)
ar = torch.arange(cap + 1, device=device)
cur_w = cur_h = 0
placements = [None] * n
gen = torch.Generator(device=device).manual_seed(rng_seed)
rand_n = min(512, attempts) # random samples per orientation
# no _SWEEP_CAP here: the skyline-bound pruning depends on the dense sweep
rand01 = torch.rand(n, 2, rand_n, 2, generator=gen, device=device) # all draws upfront
for t_i, ci in enumerate(order):
if progress_callback is not None and (t_i & 255) == 0:
progress_callback(n + t_i, 2 * n)
if cur_h + margin > atlas.shape[0] or cur_w + margin > atlas.shape[1]:
ns = max(atlas.shape[0], cur_h + margin, cur_w + margin)
na = torch.zeros((ns, ns), dtype=torch.bool, device=device)
na[:atlas.shape[0], :atlas.shape[1]] = atlas
atlas = na
nsk = torch.zeros(ns, dtype=torch.long, device=device)
nsk[:sky_t.shape[0]] = sky_t
sky_t = nsk
ar = torch.arange(ns + 1, device=device)
dim, dimr = dim_l[ci], dimr_l[ci]
step = max(1, min(dim[0], dim[1]) // 8)
cands, n_sky = _build_candidates_gpu(
sky_t, ar, cur_w, cur_h, dim[1], dimr[1], step, rand01[t_i], device)
res = _best_placement_torch(atlas, pix_l[ci], dim, dimr,
cands, n_sky, cur_w, cur_h, device)
bx, by, swap = (int(v) for v in res.tolist())
if bx < 0:
bx, by, swap = cur_w, 0, 0
pix = pixr_l[ci] if swap else pix_l[ci]
bh_, bw_ = (dimr if swap else dim)
atlas[by + pix[:, 0], bx + pix[:, 1]] = True # sparse blit
cur_w = max(cur_w, bx + bw_)
cur_h = max(cur_h, by + bh_)
ct = (ctr_pad if swap else ct_pad)[ci, :bw_] # GPU skyline lift
ix = ar[bx:bx + bw_]
sky_t[ix] = torch.where(ct >= 0, torch.maximum(sky_t[ix], by + ct + 1), sky_t[ix])
placements[ci] = ChartPlacement(chart_id=ci, offset=(float(bx), float(by)),
scale=scales[ci], rotation=thetas[ci], swap_xy=bool(swap),
chart_h=float(dim_l[ci][0]))
return placements, cur_w, cur_h
def pack_bitmap_concat(
uvs_cat: np.ndarray, # (sumV, 2) per-chart concatenated UVs
uv_offsets: np.ndarray, # (n+1,)
faces_cat: np.ndarray, # (sumF, 3) local vert ids per chart
face_offsets: np.ndarray, # (n+1,)
chart_3d_areas: np.ndarray,
chart_uv_areas: np.ndarray,
texels_per_unit: float = 256.0,
padding_texels: int = 2,
attempts: int = 4096,
rng_seed: int = 0,
progress_callback=None,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, int, int]:
"""Rasterize-and-place packer over concatenated chart arrays (no per-chart python).
Returns (x, y, swap, rotation, scale, chart_h, atlas_w, atlas_h) with one entry per chart.
progress_callback(done, total) is invoked periodically; total is 2*n_charts."""
n = int(uv_offsets.shape[0]) - 1
empty = np.zeros(n, dtype=np.int64)
if n == 0:
return empty, empty, empty, empty.astype(np.float64), empty.astype(np.float64), empty, 1, 1
if not _HAVE_NUMBA_PACK:
chart_uvs = [torch.from_numpy(np.ascontiguousarray(uvs_cat[uv_offsets[c]:uv_offsets[c + 1]]))
for c in range(n)]
chart_faces = [torch.from_numpy(np.ascontiguousarray(faces_cat[face_offsets[c]:face_offsets[c + 1]]))
for c in range(n)]
placements, w, h = _pack_bitmap_torch(
chart_uvs, [float(a) for a in chart_3d_areas], [float(a) for a in chart_uv_areas],
chart_faces, texels_per_unit, padding_texels, attempts=attempts,
rng_seed=rng_seed, progress_callback=progress_callback)
px = np.array([p.offset[0] for p in placements], dtype=np.int64)
py = np.array([p.offset[1] for p in placements], dtype=np.int64)
sw = np.array([1 if p.swap_xy else 0 for p in placements], dtype=np.int64)
th = np.array([p.rotation for p in placements], dtype=np.float64)
sc = np.array([p.scale for p in placements], dtype=np.float64)
chh = np.array([p.chart_h for p in placements], dtype=np.int64)
return px, py, sw, th, sc, chh, w, h
uvs64 = np.ascontiguousarray(uvs_cat, dtype=np.float64)
faces64 = np.ascontiguousarray(faces_cat, dtype=np.int64)
uv_off = np.ascontiguousarray(uv_offsets, dtype=np.int64)
f_off = np.ascontiguousarray(face_offsets, dtype=np.int64)
a3 = np.ascontiguousarray(chart_3d_areas, dtype=np.float64)
auv = np.ascontiguousarray(chart_uv_areas, dtype=np.float64)
theta = np.zeros(n, dtype=np.float64)
scale = np.zeros(n, dtype=np.float64)
bw = np.zeros(n, dtype=np.int64)
bh = np.zeros(n, dtype=np.int64)
rot_uv = np.empty_like(uvs64)
_prepare_dims_jit(uvs64, uv_off, a3, auv, float(texels_per_unit), int(padding_texels),
theta, scale, bw, bh, rot_uv)
boff = np.zeros(n + 1, dtype=np.int64)
np.cumsum(bw * bh, out=boff[1:])
buf = np.zeros(int(boff[-1]), dtype=np.bool_)
tw = np.zeros(n, dtype=np.int64)
th_arr = np.zeros(n, dtype=np.int64)
perim = np.zeros(n, dtype=np.float64)
_raster_all_jit(rot_uv, uv_off, faces64, f_off, bw, bh, boff, buf,
int(padding_texels), tw, th_arr, perim)
if progress_callback is not None:
progress_callback(n, 2 * n)
order = np.argsort(-perim, kind="stable")
max_b = int(max(int(tw.max()), int(th_arr.max())))
margin = max_b + 8
side_guess = int(math.sqrt(float((tw * th_arr).sum()))) * 2 + 16
cap = side_guess + margin
atlas = np.zeros((cap, cap), dtype=np.bool_)
skyline = np.zeros(cap, dtype=np.int64)
rng = np.random.default_rng(rng_seed)
# shared random pool, sliced at a rotating offset per chart
pool = rng.integers(0, 1 << 31, size=(attempts * 8, 2)).astype(np.int64)
out_x = np.full(n, -1, dtype=np.int64)
out_y = np.full(n, -1, dtype=np.int64)
out_sw = np.zeros(n, dtype=np.int64)
cur_wh = np.zeros(2, dtype=np.int64)
start = 0
while start < n:
stop = min(n, start + 1024)
nxt = _place_all_jit(buf, boff, bw, tw, th_arr, order, start, stop,
atlas, skyline, pool, int(attempts), int(_SWEEP_CAP),
int(margin), int(_nb_threads()), cur_wh, out_x, out_y, out_sw)
if nxt < stop: # atlas must grow before this chart fits
ns = max(atlas.shape[0], int(cur_wh[1]) + margin, int(cur_wh[0]) + margin)
na = np.zeros((ns, ns), dtype=np.bool_)
na[:atlas.shape[0], :atlas.shape[1]] = atlas
atlas = na
nsk = np.zeros(ns, dtype=np.int64)
nsk[:skyline.shape[0]] = skyline
skyline = nsk
start = nxt
if progress_callback is not None:
progress_callback(n + start, 2 * n)
return out_x, out_y, out_sw, theta, scale, th_arr, int(cur_wh[0]), int(cur_wh[1])
def apply_placements_concat(
uvs_cat: np.ndarray, uv_offsets: np.ndarray,
px: np.ndarray, py: np.ndarray, sw: np.ndarray,
theta: np.ndarray, scale: np.ndarray, chart_h: np.ndarray,
atlas_w: int, atlas_h: int,
) -> np.ndarray:
"""apply_placements over concatenated charts, fully vectorized. Returns (sumV, 2) float32."""
n = int(uv_offsets.shape[0]) - 1
side = float(max(atlas_w, atlas_h, 1))
cov = np.repeat(np.arange(n), np.diff(uv_offsets))
u_in = uvs_cat[:, 0].astype(np.float64)
v_in = uvs_cat[:, 1].astype(np.float64)
c = np.cos(theta)[cov]
s = np.sin(theta)[cov]
u = u_in * c - v_in * s
v = u_in * s + v_in * c
umin = np.full(n, np.inf)
vmin = np.full(n, np.inf)
np.minimum.at(umin, cov, u)
np.minimum.at(vmin, cov, v)
u = (u - umin[cov]) * scale[cov]
v = (v - vmin[cov]) * scale[cov]
swv = sw[cov].astype(bool)
# 90 deg rotation matching the rotated-bitmap access: (u, v) -> (chart_h - v, u)
u2 = np.where(swv, chart_h[cov] - v, u) + px[cov]
v2 = np.where(swv, u, v) + py[cov]
out = np.stack([u2, v2], axis=1) / side
np.clip(out, 0.0, 1.0, out=out) # slivers can stick sub-texel past extents
return out.astype(np.float32)

View File

@ -0,0 +1,565 @@
"""Chart parameterization: ortho PCA projection, falling back to ABF/LSCM."""
from __future__ import annotations
import warnings
from typing import List, Tuple
import numpy as np
import scipy.sparse as sp
import scipy.sparse.linalg as spla
import torch
from torch import Tensor
from . import mesh as _mesh
LSCM_BATCH_MAX_VERTS = 256 # charts above this solve per-chart sparse (lscm_chart)
def solve_least_squares(A: sp.csr_matrix, b: np.ndarray) -> np.ndarray:
"""Solve ||Ax - b||^2 by factorizing AtA."""
At = A.T.tocsr()
AtA = (At @ A).tocsc()
Atb = At @ b
return spla.spsolve(AtA, Atb)
def _triangle_local_2d(verts_3d: np.ndarray, faces: np.ndarray) -> np.ndarray:
"""Per-triangle 2D coords [F, 3, 2] with v0 at origin, v1 along +x."""
v0 = verts_3d[faces[:, 0]]
v1 = verts_3d[faces[:, 1]]
v2 = verts_3d[faces[:, 2]]
e01 = v1 - v0
e02 = v2 - v0
L01 = np.linalg.norm(e01, axis=1).clip(min=1e-20)
x_axis = e01 / L01[:, None]
n = np.cross(e01, e02)
n /= np.linalg.norm(n, axis=1, keepdims=True).clip(min=1e-20)
y_axis = np.cross(n, x_axis)
out = np.zeros((faces.shape[0], 3, 2), dtype=np.float64)
out[:, 1, 0] = L01
out[:, 2, 0] = (e02 * x_axis).sum(axis=1)
out[:, 2, 1] = (e02 * y_axis).sum(axis=1)
return out
def _pick_pins(loops: List[List[int]], verts_3d: np.ndarray) -> Tuple[int, int]:
"""Pick the longest-diameter axis-extremal boundary vertex pair across all boundary verts."""
if not loops:
# Closed surface: two far verts via two-pass farthest.
d2 = np.sum((verts_3d - verts_3d[0]) ** 2, axis=1)
a = int(np.argmax(d2))
d2 = np.sum((verts_3d - verts_3d[a]) ** 2, axis=1)
b = int(np.argmax(d2))
return a, b
boundary_verts: List[int] = []
for loop in loops:
boundary_verts.extend(loop)
seen = set()
uniq = []
for v in boundary_verts:
if v not in seen:
seen.add(v)
uniq.append(v)
bv = np.asarray(uniq, dtype=np.int64)
pts = verts_3d[bv]
pin_pairs = []
for axis in range(3):
i_min = int(bv[int(np.argmin(pts[:, axis]))])
i_max = int(bv[int(np.argmax(pts[:, axis]))])
d = float(np.linalg.norm(verts_3d[i_min] - verts_3d[i_max]))
pin_pairs.append((d, i_min, i_max))
d0, _, _ = pin_pairs[0]
d1, _, _ = pin_pairs[1]
d2, _, _ = pin_pairs[2]
if d0 > d1 and d0 > d2:
_, a, b = pin_pairs[0]
elif d1 > d2:
_, a, b = pin_pairs[1]
else:
_, a, b = pin_pairs[2]
return a, b
def _ortho_project(verts_3d: np.ndarray) -> np.ndarray:
"""PCA-fit plane normal, axis-aligned tangent, project verts to 2D."""
centroid = verts_3d.mean(axis=0)
pts = verts_3d - centroid
cov = pts.T @ pts
_w, ev = np.linalg.eigh(cov)
normal = ev[:, 0]
a = np.abs(normal)
if a[0] < a[1] and a[0] < a[2]:
t = np.array([1.0, 0.0, 0.0])
elif a[1] < a[2]:
t = np.array([0.0, 1.0, 0.0])
else:
t = np.array([0.0, 0.0, 1.0])
t = t - normal * float(np.dot(normal, t))
t /= max(float(np.linalg.norm(t)), 1e-20)
b = np.cross(normal, t)
return np.stack([verts_3d @ t, verts_3d @ b], axis=1)
def ortho_project_concat(verts: np.ndarray, chart_of_vert: np.ndarray, n_charts: int) -> np.ndarray:
"""_ortho_project for every chart at once over concatenated per-chart vertices."""
cnt = np.bincount(chart_of_vert, minlength=n_charts).clip(min=1).astype(np.float64)
cen = np.stack([np.bincount(chart_of_vert, weights=verts[:, i], minlength=n_charts)
for i in range(3)], axis=1) / cnt[:, None]
d = verts - cen[chart_of_vert]
cov = np.zeros((n_charts, 3, 3), dtype=np.float64)
for i in range(3):
for j in range(i, 3):
s = np.bincount(chart_of_vert, weights=d[:, i] * d[:, j], minlength=n_charts)
cov[:, i, j] = s
cov[:, j, i] = s
_w, ev = np.linalg.eigh(cov)
normal = ev[:, :, 0]
t = np.eye(3, dtype=np.float64)[np.argmin(np.abs(normal), axis=1)]
t = t - normal * (normal * t).sum(axis=1, keepdims=True)
t /= np.linalg.norm(t, axis=1, keepdims=True).clip(min=1e-20)
b = np.cross(normal, t)
tt, bb = t[chart_of_vert], b[chart_of_vert]
return np.stack([(verts * tt).sum(1), (verts * bb).sum(1)], axis=1)
def stretch_metrics_concat(
verts: np.ndarray, uvs: np.ndarray, faces: np.ndarray,
chart_of_face: np.ndarray, n_charts: int,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Per-chart Sander stretch metrics (rms, max, n_flipped, n_zero_area); rms/max inf where undefined."""
p = verts[faces]
t = uvs[faces]
pa_signed = 0.5 * (
(t[:, 1, 1] - t[:, 0, 1]) * (t[:, 2, 0] - t[:, 0, 0])
- (t[:, 2, 1] - t[:, 0, 1]) * (t[:, 1, 0] - t[:, 0, 0]))
n_flip = np.bincount(chart_of_face[pa_signed < -1e-12], minlength=n_charts)
n_zero = np.bincount(chart_of_face[np.abs(pa_signed) < 1e-12], minlength=n_charts)
pa = np.abs(pa_signed).clip(min=1e-20)
ga = 0.5 * np.linalg.norm(np.cross(p[:, 1] - p[:, 0], p[:, 2] - p[:, 0]), axis=1)
keep = (ga > 1e-12) & (np.abs(pa_signed) > 1e-12)
t1, s1 = t[:, 0, 0], t[:, 0, 1]
t2, s2 = t[:, 1, 0], t[:, 1, 1]
t3, s3 = t[:, 2, 0], t[:, 2, 1]
inv_2pa = 1.0 / (2.0 * pa)
Ss = (p[:, 0] * (t2 - t3)[:, None] + p[:, 1] * (t3 - t1)[:, None]
+ p[:, 2] * (t1 - t2)[:, None]) * inv_2pa[:, None]
St = (p[:, 0] * (s3 - s2)[:, None] + p[:, 1] * (s1 - s3)[:, None]
+ p[:, 2] * (s2 - s1)[:, None]) * inv_2pa[:, None]
a = (Ss * Ss).sum(axis=1)
bb = (Ss * St).sum(axis=1)
c = (St * St).sum(axis=1)
sigma2_sq = 0.5 * (a + c + np.sqrt(np.maximum(0.0, (a - c) ** 2 + 4 * bb ** 2)))
rms_sq = (a + c) * 0.5
cf = chart_of_face[keep]
tg = np.bincount(cf, weights=ga[keep], minlength=n_charts)
tp = np.bincount(cf, weights=pa[keep], minlength=n_charts)
rs = np.bincount(cf, weights=(rms_sq * ga)[keep], minlength=n_charts)
smax = np.zeros(n_charts, dtype=np.float64)
np.maximum.at(smax, cf, sigma2_sq[keep])
ok = tg > 0.0
tg_safe = np.where(ok, tg, 1.0)
norm = np.sqrt(tp / tg_safe)
rms = np.where(ok, np.sqrt(rs / tg_safe) * norm, np.inf)
mx = np.where(ok, np.sqrt(smax) * norm, np.inf)
return rms, mx, n_flip, n_zero
def _segment_argmax(vals: np.ndarray, seg: np.ndarray, n: int) -> np.ndarray:
"""Index of the (first) max element per segment; -1 for empty segments."""
amax = np.full(n, -np.inf)
np.maximum.at(amax, seg, vals)
hit = vals == amax[seg]
out = np.full(n, np.iinfo(np.int64).max, dtype=np.int64)
np.minimum.at(out, seg[hit], np.nonzero(hit)[0])
return np.where(out == np.iinfo(np.int64).max, -1, out)
def lscm_charts_batch(
verts: np.ndarray, # (sumV, 3) float64, per-chart concatenated
uv_pins: np.ndarray, # (sumV, 2) float64, ortho UVs (pin values + fallback)
faces_gl: np.ndarray, # (sumF, 3) global-local ids into verts
face_pos: np.ndarray, # (sumF,) row index of each face within its chart
chart_of_face: np.ndarray, # (sumF,)
chart_of_vert: np.ndarray, # (sumV,)
vert_offsets: np.ndarray, # (n_charts+1,)
chart_ids: np.ndarray, # charts to solve (each with >=3 verts, >=1 face)
n_charts: int,
max_bucket_verts: int = LSCM_BATCH_MAX_VERTS,
device: "torch.device | None" = None,
) -> dict:
"""Batched dense ABF/LSCM; returns {chart_id: (Vc, 2) float32}. Charts larger than
max_bucket_verts are left out (the caller solves those sparse)."""
out: dict = {}
if chart_ids.size == 0:
return out
sel = np.zeros(n_charts, dtype=bool)
sel[chart_ids] = True
vcounts = np.diff(vert_offsets)
# ABF coefficients for all selected faces in one shot
fmask = sel[chart_of_face]
f_ids = np.nonzero(fmask)[0]
abf_ids, abf_cos, abf_sin, abf_valid = _abf_face_coefficients(verts, faces_gl[f_ids])
# farthest-point pin pair per chart (two passes)
vmask = sel[chart_of_vert]
v_ids = np.nonzero(vmask)[0]
cv = chart_of_vert[v_ids]
first = vert_offsets[:-1]
d0 = ((verts[v_ids] - verts[first[cv]]) ** 2).sum(1)
pin_a = _segment_argmax(d0, cv, n_charts) # global vert index (into v_ids space)
pin_a = np.where(pin_a >= 0, v_ids[pin_a.clip(min=0)], -1)
d1 = ((verts[v_ids] - verts[pin_a.clip(min=0)[cv]]) ** 2).sum(1)
pin_b = _segment_argmax(d1, cv, n_charts)
pin_b = np.where(pin_b >= 0, v_ids[pin_b.clip(min=0)], -1)
# degenerate (all verts coincide): any distinct vert within the chart (Vc >= 3 guaranteed)
alt = np.where(pin_a == first, first + 1, first)
pin_b = np.where(pin_a == pin_b, alt, pin_b)
fcounts = np.bincount(chart_of_face[f_ids], minlength=n_charts)
# size-sorted chunks padded to their own max, bounded by an element budget so one
# face-heavy chart can't inflate a whole chunk
small = chart_ids[vcounts[chart_ids] <= max_bucket_verts]
sorted_ids = small[np.argsort(vcounts[small], kind="stable")]
budget = (96 << 20) // 8 # float64 elements in a chunk's A
chunks = []
cs = 0
fmax_r = vmax_r = 0
for idx in range(sorted_ids.size):
c2 = sorted_ids[idx]
fm2 = max(fmax_r, int(fcounts[c2]))
vm2 = max(vmax_r, int(vcounts[c2]))
nb = idx - cs + 1
if nb > 1 and (nb > 128 or nb * 4 * fm2 * vm2 > budget):
chunks.append((cs, idx))
cs = idx
fmax_r, vmax_r = int(fcounts[c2]), int(vcounts[c2])
else:
fmax_r, vmax_r = fm2, vm2
if sorted_ids.size:
chunks.append((cs, sorted_ids.size))
for s, e in chunks:
cids = sorted_ids[s:e]
B = cids.size
Vmax = int(vcounts[cids].max())
Fmax = int(fcounts[cids].max())
N = 2 * Vmax
R = 2 * Fmax
compact = np.full(n_charts, -1, dtype=np.int64)
compact[cids] = np.arange(B)
fm = compact[chart_of_face[f_ids]] >= 0
fi = f_ids[fm] # face rows for this chunk
bi = compact[chart_of_face[fi]] # chart slot per face
frow = face_pos[fi]
v0 = vert_offsets[chart_of_face[fi]] # local id = global-local - v0
am = fm.nonzero()[0] # index into abf_* arrays
pieces_i: list = []
pieces_v: list = []
def scatter(rows, cols, vals, bsel):
pieces_i.append((bsel * R + rows) * N + cols)
pieces_v.append(vals)
val = abf_valid[am]
ii = am[val]
ids = abf_ids[ii] - v0[val, None] # local vert ids, reordered
cosf, sinf = abf_cos[ii], abf_sin[ii]
rr, bsel = frow[val] * 2, bi[val]
ones = np.ones(ii.size)
for cc2, vv in ((ids[:, 0], cosf - 1.0), (ids[:, 0] + Vmax, -sinf),
(ids[:, 1], -cosf), (ids[:, 1] + Vmax, sinf), (ids[:, 2], ones)):
scatter(rr, cc2, vv, bsel)
for cc2, vv in ((ids[:, 0], sinf), (ids[:, 0] + Vmax, cosf - 1.0),
(ids[:, 1], -sinf), (ids[:, 1] + Vmax, -cosf), (ids[:, 2] + Vmax, ones)):
scatter(rr + 1, cc2, vv, bsel)
inv = ~val
if inv.any():
jj = fi[inv]
tri2d = _triangle_local_2d(verts, faces_gl[jj])
twice = tri2d[:, 1, 0] * tri2d[:, 2, 1] - tri2d[:, 1, 1] * tri2d[:, 2, 0]
w = 1.0 / np.sqrt(2.0 * np.abs(twice).clip(min=1e-20))
rr2, bs2 = frow[inv] * 2, bi[inv]
lids = faces_gl[jj] - v0[inv, None]
for j in range(3):
jp1, jp2 = (j + 1) % 3, (j + 2) % 3
aj = (tri2d[:, jp1, 0] - tri2d[:, jp2, 0]) * w
bj = (tri2d[:, jp1, 1] - tri2d[:, jp2, 1]) * w
vc2 = lids[:, j]
scatter(rr2, vc2, aj, bs2)
scatter(rr2, vc2 + Vmax, -bj, bs2)
scatter(rr2 + 1, vc2, bj, bs2)
scatter(rr2 + 1, vc2 + Vmax, aj, bs2)
flat = np.concatenate(pieces_i)
A = np.bincount(flat, weights=np.concatenate(pieces_v),
minlength=B * R * N).reshape(B, R, N)
# pins: move their columns to the RHS, then constrain via identity rows
voff = vert_offsets[cids]
pa_l = pin_a[cids] - voff
pb_l = pin_b[cids] - voff
pin_cols = np.stack([pa_l, pb_l, pa_l + Vmax, pb_l + Vmax], 1) # (B,4)
pin_vals = np.stack([uv_pins[pin_a[cids], 0], uv_pins[pin_b[cids], 0],
uv_pins[pin_a[cids], 1], uv_pins[pin_b[cids], 1]], 1)
rhs = np.zeros((B, R), dtype=np.float64)
barange = np.arange(B)
for k in range(4):
rhs -= A[barange, :, pin_cols[:, k]] * pin_vals[:, k, None]
A[barange, :, pin_cols[:, k]] = 0.0
# constrained columns: the 4 pins + padding beyond each chart's vert count
vcs = vcounts[cids]
padm = np.arange(Vmax)[None, :] >= vcs[:, None]
con = np.concatenate([padm, padm], axis=1) # (B,N)
np.put_along_axis(con, pin_cols, True, axis=1)
cval = np.zeros((B, N), dtype=np.float64)
np.put_along_axis(cval, pin_cols, pin_vals, axis=1)
# normal equations + batched solve; the fp64 dense algebra goes to the GPU when available
use_gpu = device is not None and device.type == "cuda"
if use_gpu:
A_t = torch.from_numpy(A).to(device)
At = A_t.transpose(1, 2)
AtA = At @ A_t
Atb = (At @ torch.from_numpy(rhs).to(device).unsqueeze(2)).squeeze(2)
con_t = torch.from_numpy(con).to(device)
free2 = (~con_t[:, :, None]) & (~con_t[:, None, :])
AtA = AtA * free2
diag = torch.diagonal(AtA, dim1=1, dim2=2)
# median (not max) positive diagonal: a degenerate face's ~1e19 squared row
# weight would blow a max-scaled eps past the unit ABF rows
dpos = torch.where(diag > 0, diag, torch.full_like(diag, float("nan")))
dsc = 1e-12 * torch.nan_to_num(dpos.nanmedian(dim=1).values, nan=1e-8).clamp_min(1e-20)
diag += torch.where(con_t, torch.ones_like(diag), dsc[:, None].expand_as(diag))
Atb = torch.where(con_t, torch.from_numpy(cval).to(device), Atb)
x = torch.linalg.solve(AtA, Atb).cpu().numpy()
else:
At = A.transpose(0, 2, 1)
AtA = At @ A # batched BLAS dgemm
Atb = (At @ rhs[:, :, None])[:, :, 0]
AtA *= (~con[:, :, None]) & (~con[:, None, :])
dg = AtA.reshape(B, -1)[:, ::N + 1]
# median positive diagonal (see GPU branch): robust to degenerate-face weights
dpos = np.where(dg > 0, dg, np.nan)
with np.errstate(all="ignore"):
dsc = 1e-12 * np.nan_to_num(np.nanmedian(dpos, axis=1), nan=1e-8).clip(min=1e-20)
dg += np.where(con, 1.0, dsc[:, None])
Atb2 = np.where(con, cval, Atb)
x = np.linalg.solve(AtA, Atb2)
for i2, c2 in enumerate(cids):
vc3 = int(vcs[i2])
out[int(c2)] = np.stack([x[i2, :vc3], x[i2, Vmax:Vmax + vc3]], 1).astype(np.float32)
return out
def _uv_boundary_self_intersects(
uvs: np.ndarray, faces: np.ndarray, face_face: np.ndarray, eps: float = 1e-9
) -> bool:
"""True if any chart-boundary edge pair crosses in 2D (ortho folded the chart)."""
fi, ei = np.nonzero(face_face < 0)
n = fi.size
if n < 2:
return False
a = uvs[faces[fi, ei]].astype(np.float64)
b = uvs[faces[fi, (ei + 1) % 3]].astype(np.float64)
d = b - a
# Pairwise segment crossings, row-chunked to bound memory at chunk*n.
chunk = max(1, min(n, 1_000_000 // max(n, 1)))
for s in range(0, n, chunk):
e = min(s + chunk, n)
d1 = d[s:e, None, :]
denom = d1[:, :, 0] * d[None, :, 1] - d1[:, :, 1] * d[None, :, 0]
rx = a[None, :, 0] - a[s:e, None, 0]
ry = a[None, :, 1] - a[s:e, None, 1]
with np.errstate(divide="ignore", invalid="ignore"):
t = (rx * d[None, :, 1] - ry * d[None, :, 0]) / denom
u = (rx * d1[:, :, 1] - ry * d1[:, :, 0]) / denom
cross = (
(np.abs(denom) >= eps)
& (t > eps) & (t < 1.0 - eps)
& (u > eps) & (u < 1.0 - eps)
)
if bool(cross.any()):
return True
return False
def _abf_face_coefficients(
verts_3d: np.ndarray, faces: np.ndarray
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Per-face ABF constraint (largest-sine vertex at local index 2); returns (faces_reordered, cosine, sine, valid_mask) with valid_mask False for degenerate tris."""
p0 = verts_3d[faces[:, 0]]
p1 = verts_3d[faces[:, 1]]
p2 = verts_3d[faces[:, 2]]
e01 = p1 - p0
e12 = p2 - p1
e20 = p0 - p2
L01 = np.linalg.norm(e01, axis=1).clip(min=1e-20)
L12 = np.linalg.norm(e12, axis=1).clip(min=1e-20)
L20 = np.linalg.norm(e20, axis=1).clip(min=1e-20)
cos_a0 = ((-e20) * e01).sum(axis=1) / (L20 * L01)
cos_a1 = ((-e01) * e12).sum(axis=1) / (L01 * L12)
cos_a2 = ((-e12) * e20).sum(axis=1) / (L12 * L20)
cos_a0 = cos_a0.clip(-1.0, 1.0)
cos_a1 = cos_a1.clip(-1.0, 1.0)
cos_a2 = cos_a2.clip(-1.0, 1.0)
a = np.arccos(cos_a0)
b_ang = np.arccos(cos_a1)
c_ang = np.arccos(cos_a2)
angles = np.stack([a, b_ang, c_ang], axis=1)
sines = np.stack([np.sin(a), np.sin(b_ang), np.sin(c_ang)], axis=1)
valid = (angles > 1e-12).all(axis=1)
ids = faces.astype(np.int64).copy()
s0, s1, s2 = sines[:, 0], sines[:, 1], sines[:, 2]
pattA = (s1 > s0) & (s1 > s2)
pattB = (~pattA) & (s0 > s1) & (s0 > s2)
if pattA.any():
old_a = angles[pattA].copy()
old_s = sines[pattA].copy()
old_id = ids[pattA].copy()
angles[pattA] = old_a[:, [2, 0, 1]]
sines[pattA] = old_s[:, [2, 0, 1]]
ids[pattA] = old_id[:, [2, 0, 1]]
if pattB.any():
old_a = angles[pattB].copy()
old_s = sines[pattB].copy()
old_id = ids[pattB].copy()
angles[pattB] = old_a[:, [1, 2, 0]]
sines[pattB] = old_s[:, [1, 2, 0]]
ids[pattB] = old_id[:, [1, 2, 0]]
a0 = angles[:, 0]
s0 = sines[:, 0]
s1 = sines[:, 1]
s2 = sines[:, 2]
c0 = np.cos(a0)
ratio = np.where(s2 > 0.0, s1 / s2.clip(min=1e-20), 1.0)
cosine = c0 * ratio
sine = s0 * ratio
return ids, cosine, sine, valid
def lscm_chart(
local_verts: Tensor,
local_faces: Tensor,
local_face_face: Tensor,
pin_positions: "np.ndarray | None" = None,
) -> Tensor:
"""ABF parameterization on one chart (degenerate faces use plain LSCM rows; two pins fix gauge at pin_positions)."""
verts_np = local_verts.detach().cpu().numpy().astype(np.float64)
faces_np = local_faces.detach().cpu().numpy().astype(np.int64)
Vc = verts_np.shape[0]
Fc = faces_np.shape[0]
if Vc < 3 or Fc == 0:
return torch.zeros((Vc, 2), dtype=torch.float32, device=local_verts.device)
loops = _mesh.chart_boundary_loops(local_faces, local_face_face)
pin_a, pin_b = _pick_pins(loops, verts_np)
if pin_positions is not None and pin_positions.shape == (Vc, 2):
pa = pin_positions[pin_a]
pb = pin_positions[pin_b]
u_a, v_a = float(pa[0]), float(pa[1])
u_b, v_b = float(pb[0]), float(pb[1])
else:
u_a, v_a = 0.0, 0.0
u_b, v_b = 1.0, 0.0
abf_ids, abf_cos, abf_sin, abf_valid = _abf_face_coefficients(verts_np, faces_np)
rows_list: List[np.ndarray] = []
cols_list: List[np.ndarray] = []
vals_list: List[np.ndarray] = []
# ABF rows for valid faces.
valid_idx = np.nonzero(abf_valid)[0]
if valid_idx.size:
Nv = valid_idx.size
id0 = abf_ids[valid_idx, 0]
id1 = abf_ids[valid_idx, 1]
id2 = abf_ids[valid_idx, 2]
cosf = abf_cos[valid_idx]
sinf = abf_sin[valid_idx]
r_real = valid_idx * 2
r_imag = valid_idx * 2 + 1
ones = np.ones(Nv, dtype=np.float64)
rows_list.extend([r_real] * 5)
cols_list.extend([id0, id0 + Vc, id1, id1 + Vc, id2])
vals_list.extend([cosf - 1.0, -sinf, -cosf, sinf, ones])
rows_list.extend([r_imag] * 5)
cols_list.extend([id0, id0 + Vc, id1, id1 + Vc, id2 + Vc])
vals_list.extend([sinf, cosf - 1.0, -sinf, -cosf, ones])
# Plain-LSCM rows for invalid (degenerate) faces.
invalid_idx = np.nonzero(~abf_valid)[0]
if invalid_idx.size:
tri2d_inv = _triangle_local_2d(verts_np, faces_np[invalid_idx])
twice_area_inv = (
tri2d_inv[:, 1, 0] * tri2d_inv[:, 2, 1]
- tri2d_inv[:, 1, 1] * tri2d_inv[:, 2, 0]
)
weight_inv = 1.0 / np.sqrt(2.0 * np.abs(twice_area_inv).clip(min=1e-20))
r_real_inv = invalid_idx * 2
r_imag_inv = invalid_idx * 2 + 1
for j in range(3):
jp1 = (j + 1) % 3
jp2 = (j + 2) % 3
a_j = (tri2d_inv[:, jp1, 0] - tri2d_inv[:, jp2, 0]) * weight_inv
b_j = (tri2d_inv[:, jp1, 1] - tri2d_inv[:, jp2, 1]) * weight_inv
v_idx = faces_np[invalid_idx, j]
rows_list.extend([r_real_inv, r_real_inv, r_imag_inv, r_imag_inv])
cols_list.extend([v_idx, v_idx + Vc, v_idx, v_idx + Vc])
vals_list.extend([a_j, -b_j, b_j, a_j])
rows = np.concatenate(rows_list) if rows_list else np.empty(0, dtype=np.int64)
cols = np.concatenate(cols_list) if cols_list else np.empty(0, dtype=np.int64)
vals = np.concatenate(vals_list) if vals_list else np.empty(0, dtype=np.float64)
A_full = sp.csr_matrix((vals, (rows, cols)), shape=(2 * Fc, 2 * Vc))
pin_cols = np.array([pin_a, pin_b, pin_a + Vc, pin_b + Vc], dtype=np.int64)
pin_vals = np.array([u_a, u_b, v_a, v_b], dtype=np.float64)
free_mask = np.ones(2 * Vc, dtype=bool)
free_mask[pin_cols] = False
free_cols = np.nonzero(free_mask)[0]
A_pinned = A_full[:, pin_cols]
A_free = A_full[:, free_cols]
b = -(A_pinned @ pin_vals)
# Singular system (under-constrained chart) falls back to ortho.
fallback_to_ortho = False
try:
with warnings.catch_warnings():
warnings.simplefilter("error", category=sp.linalg.MatrixRankWarning)
x_free = solve_least_squares(A_free, b)
if not np.all(np.isfinite(x_free)):
fallback_to_ortho = True
except (sp.linalg.MatrixRankWarning, RuntimeError):
fallback_to_ortho = True # singular / under-constrained system
if fallback_to_ortho:
if pin_positions is not None and pin_positions.shape == (Vc, 2):
uvs = pin_positions.astype(np.float32)
else:
uvs = _ortho_project(verts_np).astype(np.float32)
return torch.from_numpy(uvs).to(local_verts.device)
full = np.zeros(2 * Vc, dtype=np.float64)
full[free_cols] = x_free
full[pin_cols] = pin_vals
uvs = np.stack([full[:Vc], full[Vc:]], axis=1).astype(np.float32)
if not np.all(np.isfinite(uvs)):
if pin_positions is not None and pin_positions.shape == (Vc, 2):
uvs = pin_positions.astype(np.float32)
else:
uvs = _ortho_project(verts_np).astype(np.float32)
return torch.from_numpy(uvs).to(local_verts.device)

View File

@ -0,0 +1,414 @@
"""Adaptive cost-grow chart segmentation (vectorized torch, CPU or GPU)."""
from __future__ import annotations
from typing import Tuple
import torch
from torch import Tensor
from tqdm import tqdm
from .mesh import MeshData, face_edge_lengths
DEFAULT_W_NORMAL_DEVIATION = 2.0
DEFAULT_W_ROUNDNESS = 0.01
DEFAULT_W_STRAIGHTNESS = 6.0
DEFAULT_MAX_COST = 2.0
NORMAL_DEVIATION_HARD_CUTOFF = 0.707 # ~75°
def _grow_iter(face_chart, frontier, ff, fn, fa, fel, basis, nsum, area, perim, K,
nd_cutoff, tau, w_nd, w_round, w_straight):
"""One grow pass: each frontier face joins its lowest-cost adjacent chart if cost <= tau;
returns the number of faces assigned."""
u = frontier.nonzero(as_tuple=True)[0]
if u.numel() == 0:
return 0
nb = ff[u] # (U,3) neighbor face ids
nbc = torch.where(nb >= 0, face_chart[nb.clamp_min(0)], nb.new_full((), -1))
valid = nbc >= 0
d = (fn[u][:, None, :] * basis[nbc.clamp_min(0)]).sum(-1)
nd = (1.0 - d).clamp(0.0, 1.0)
valid &= nd < nd_cutoff
el = fel[u] # (U,3)
# l_in per candidate chart j: edge k counts if its (assigned) neighbor is in chart j
inm = (nbc[:, :, None] == nbc[:, None, :]) & valid[:, None, :]
l_in = (el[:, None, :] * inm).sum(-1) # (U,3)
tot = el.sum(-1, keepdim=True)
l_out = tot - l_in
ca = area[nbc.clamp_min(0)]
cp = perim[nbc.clamp_min(0)]
new_perim = cp - l_in + l_out
new_r = new_perim * new_perim / (ca + fa[u][:, None]).clamp_min(1e-20)
round_cost = torch.where((cp <= 1e-20) | (ca <= 1e-20) | (new_r <= 1e-20),
torch.zeros_like(new_r),
1.0 - (cp * cp / ca.clamp_min(1e-20)) / new_r.clamp_min(1e-20))
straight_cost = ((l_out - l_in) / tot.clamp_min(1e-20)).clamp(max=0.0)
cost = w_nd * nd + w_round * round_cost + w_straight * straight_cost
cost = torch.where(valid, cost, cost.new_full((), float("inf")))
best_cost, best_j = cost.min(1)
acc = best_cost <= tau
n_acc = int(acc.sum())
if n_acc == 0:
return 0
f_acc = u[acc]
c_acc = nbc.gather(1, best_j[:, None]).squeeze(1)[acc]
nbc_old = nbc[acc] # neighbor charts before this commit
face_chart[f_acc] = c_acc
nb_acc = nb[acc]
nbs_acc = nb_acc.clamp_min(0)
nbc_post = torch.where(nb_acc >= 0, face_chart[nbs_acc], nb_acc.new_full((), -1))
# frontier update: committed faces leave; their still-unassigned neighbors enter
frontier[f_acc] = False
grow_nb = nbs_acc[(nb_acc >= 0) & (nbc_post < 0)]
frontier[grow_nb] = True
el_acc = el[acc]
cx = c_acc[:, None]
dper = torch.where(nbc_old == cx, -el_acc, # was member: edge turns interior
torch.where(nbc_post == cx, torch.zeros_like(el_acc), # co-committer
el_acc)).sum(1) # boundary / other chart
perim.scatter_add_(0, c_acc, dper)
area.scatter_add_(0, c_acc, fa[f_acc])
nsum.index_add_(0, c_acc, fn[f_acc] * fa[f_acc, None])
nl = nsum[:K].norm(dim=1, keepdim=True)
basis[:K] = torch.where(nl > 1e-20, nsum[:K] / nl.clamp_min(1e-20), basis[:K])
return n_acc
def segment_charts(
mesh: MeshData,
max_cost: float = DEFAULT_MAX_COST,
w_normal_deviation: float = DEFAULT_W_NORMAL_DEVIATION,
w_roundness: float = DEFAULT_W_ROUNDNESS,
w_straightness: float = DEFAULT_W_STRAIGHTNESS,
progress_callback=None,
) -> Tensor:
"""Segment mesh into charts (parallel batch cost-grow). Returns face -> chart_id."""
F = mesh.faces.shape[0]
device = mesh.faces.device
if F == 0:
return torch.zeros(0, dtype=torch.long, device=device)
fn = mesh.face_normal.detach().to(torch.float32)
fa = mesh.face_area.detach().to(torch.float32)
fc = mesh.face_centroid.detach().to(torch.float32)
ff = mesh.face_face.detach().long()
fel = face_edge_lengths(mesh.vertices, mesh.faces).detach().to(torch.float32)
nd_cutoff = NORMAL_DEVIATION_HARD_CUTOFF
# one seed per connected component (first face of each)
comp = mesh.component.detach().long().to(device)
ncomp = int(comp.max()) + 1 if comp.numel() else 0
if ncomp:
seeds = torch.full((ncomp,), F, dtype=torch.long, device=device)
seeds.scatter_reduce_(0, comp, torch.arange(F, device=device), reduce="amin")
else:
seeds = torch.zeros(1, dtype=torch.long, device=device)
K = seeds.shape[0]
max_total_charts = max(F, 8000)
cap = K + F + 1 # every re-seed assigns a face, so K < K0 + F
face_chart = torch.full((F,), -1, dtype=torch.long, device=device)
basis = torch.zeros(cap, 3, dtype=torch.float32, device=device)
nsum = torch.zeros(cap, 3, dtype=torch.float32, device=device)
area = torch.zeros(cap, dtype=torch.float32, device=device)
perim = torch.zeros(cap, dtype=torch.float32, device=device)
face_chart[seeds] = torch.arange(K, device=device)
basis[:K] = fn[seeds]
nsum[:K] = fn[seeds] * fa[seeds, None]
area[:K] = fa[seeds]
perim[:K] = fel[seeds].sum(1)
frontier = torch.zeros(F, dtype=torch.bool, device=device)
seed_nb = ff[seeds]
seed_nb = seed_nb[seed_nb >= 0]
frontier[seed_nb] = True
frontier &= face_chart < 0
min_d2 = torch.full((F,), float("inf"), dtype=torch.float32, device=device)
for i in range(0, K, 32): # chunked: (F, <=32, 3) stays small
d2 = ((fc[:, None, :] - fc[seeds[i:i + 32]][None, :, :]) ** 2).sum(-1)
min_d2 = torch.minimum(min_d2, d2.amin(1))
# Multi-pass threshold schedule (low-cost first); tau cap 0.5 keeps cones ~30deg.
tau_final = min(max_cost * 0.25, 0.5)
thresholds = [t for t in (0.05, 0.1, 0.25) if t < tau_final] + [tau_final]
max_inner = max(64, int(F ** 0.5) * 2)
outer_iter = 0
assigned = 0
tq = tqdm(total=F, desc="unwrap: segment (adaptive)", unit="face", leave=False)
while True:
outer_iter += 1
if outer_iter > F + 16:
break
for tau in thresholds:
for _ in range(max_inner):
n_added = _grow_iter(face_chart, frontier, ff, fn, fa, fel, basis, nsum,
area, perim, K, nd_cutoff, tau, w_normal_deviation,
w_roundness, w_straightness)
if n_added == 0:
break
tq.update(n_added)
assigned += n_added
if progress_callback is not None:
progress_callback(assigned, F)
unassigned = face_chart < 0
if int(unassigned.sum()) == 0:
break
if K >= max_total_charts:
break
# re-seed at the unassigned face farthest from every existing seed
new_seed = int(torch.where(unassigned, min_d2,
min_d2.new_full((), float("-inf"))).argmax())
face_chart[new_seed] = K
basis[K] = fn[new_seed]
nsum[K] = fn[new_seed] * fa[new_seed]
area[K] = fa[new_seed]
perim[K] = fel[new_seed].sum()
K += 1
min_d2 = torch.minimum(min_d2, ((fc - fc[new_seed]) ** 2).sum(-1))
tq.update(1)
frontier[new_seed] = False
ns_nb = ff[new_seed]
ns_nb = ns_nb[ns_nb >= 0]
frontier[ns_nb[face_chart[ns_nb] < 0]] = True
tq.close()
# Orphan cleanup: leftover faces join their best-matching neighbor's chart.
while True:
orphans = (face_chart < 0).nonzero(as_tuple=True)[0]
if orphans.numel() == 0:
break
nb = ff[orphans]
nbc = torch.where(nb >= 0, face_chart[nb.clamp_min(0)], nb.new_full((), -1))
valid = nbc >= 0
assignable = valid.any(1)
if not bool(assignable.any()):
break
d = (fn[orphans][:, None, :] * basis[nbc.clamp_min(0)]).sum(-1)
ndv = torch.where(valid, 1.0 - d, d.new_full((), float("inf")))
best_c = nbc.gather(1, ndv.argmin(1, keepdim=True)).squeeze(1)
face_chart[orphans[assignable]] = best_c[assignable]
leftover = (face_chart < 0).nonzero(as_tuple=True)[0]
if leftover.numel(): # isolated faces become singleton charts
face_chart[leftover] = K + torch.arange(leftover.numel(), device=device)
_, inverse = torch.unique(face_chart, sorted=True, return_inverse=True)
return inverse
# Parallel edge-collapse (PEC) chart clustering (GPU)
def _combine_normal_cones(
axis_a: Tensor, half_a: Tensor,
axis_b: Tensor, half_b: Tensor,
) -> Tuple[Tensor, Tensor, Tensor]:
"""Merge two normal cones along the great circle from axis_a; returns (combined_axis, combined_half_angle, axis_angle)."""
cos_angle = (axis_a * axis_b).sum(dim=-1).clamp(-1.0, 1.0)
axis_angle = torch.acos(cos_angle)
new_low = torch.minimum(-half_a, axis_angle - half_b)
new_high = torch.maximum(half_a, axis_angle + half_b)
new_half = (new_high - new_low) * 0.5
rot_angle = (new_high + new_low) * 0.5
b_perp = axis_b - axis_a * cos_angle.unsqueeze(-1)
b_perp_norm = b_perp.norm(dim=-1, keepdim=True).clamp_min(1e-12)
b_perp_unit = b_perp / b_perp_norm
new_axis = (
axis_a * torch.cos(rot_angle).unsqueeze(-1)
+ b_perp_unit * torch.sin(rot_angle).unsqueeze(-1)
)
new_axis_norm = new_axis.norm(dim=-1, keepdim=True).clamp_min(1e-12)
new_axis = new_axis / new_axis_norm
return new_axis, new_half, axis_angle
def _build_chart_edges(
face_face: Tensor,
chart_id: Tensor,
face_edge_len: Tensor,
) -> Tuple[Tensor, Tensor]:
"""Build chart-edge list (chart_pairs[E,2] with a<b, edge_length[E]); same-chart edges dropped, duplicates summed."""
F = face_face.shape[0]
device = face_face.device
f_idx = torch.arange(F, device=device).repeat_interleave(3)
nb = face_face.flatten()
valid = nb >= 0
f_idx = f_idx[valid]
nb = nb[valid]
el = face_edge_len.flatten()[valid]
ca = chart_id[f_idx]
cb = chart_id[nb]
diff = ca != cb
ca = ca[diff]
cb = cb[diff]
el = el[diff]
if ca.numel() == 0:
return (
torch.empty((0, 2), dtype=torch.long, device=device),
torch.empty(0, device=device),
)
lo = torch.minimum(ca, cb)
hi = torch.maximum(ca, cb)
V = int(chart_id.max().item()) + 1
key = lo * V + hi
sort_idx = torch.argsort(key)
sorted_key = key[sort_idx]
sorted_lo = lo[sort_idx]
sorted_hi = hi[sort_idx]
sorted_el = el[sort_idx]
unique_key, inverse, counts = torch.unique(
sorted_key, return_inverse=True, return_counts=True
)
n_unique = unique_key.shape[0]
reduced_el = torch.zeros(n_unique, device=device, dtype=el.dtype)
reduced_el.scatter_add_(0, inverse, sorted_el)
first_idx = torch.cat([
torch.zeros(1, dtype=torch.long, device=device),
counts.cumsum(0)[:-1],
])
pair_lo = sorted_lo[first_idx]
pair_hi = sorted_hi[first_idx]
chart_pairs = torch.stack([pair_lo, pair_hi], dim=1)
return chart_pairs, reduced_el
def _merge_small_charts(
chart_id: Tensor, face_normal: Tensor, face_area: Tensor,
face_face: Tensor, face_edge_len: Tensor,
min_faces: int, cost_cap: float,
) -> Tensor:
"""Absorb charts under min_faces faces into their lowest-cone-cost neighbor (capped at cost_cap)."""
if chart_id.numel() == 0:
return chart_id
device = chart_id.device
for _ in range(16):
N = int(chart_id.max().item()) + 1
sizes = torch.bincount(chart_id, minlength=N)
# recompute cones from scratch: area-weighted mean axis, max deviation as half-angle
axis = torch.zeros(N, 3, dtype=torch.float32, device=device)
axis.index_add_(0, chart_id, face_normal * face_area[:, None])
axis = axis / axis.norm(dim=1, keepdim=True).clamp_min(1e-12)
dev = torch.acos((face_normal * axis[chart_id]).sum(1).clamp(-1.0, 1.0))
half = torch.zeros(N, dtype=torch.float32, device=device)
half.scatter_reduce_(0, chart_id, dev, reduce="amax")
edges, _ = _build_chart_edges(face_face, chart_id, face_edge_len)
if edges.shape[0] == 0:
break
a, b = edges[:, 0], edges[:, 1]
_, new_half, _ = _combine_normal_cones(axis[a], half[a], axis[b], half[b])
ok = new_half <= cost_cap
E = edges.shape[0]
key = (torch.clamp(new_half * 1e6, max=2e9).to(torch.int64) << 32) \
| torch.arange(E, dtype=torch.long, device=device)
best = torch.full((N,), 1 << 62, dtype=torch.long, device=device)
va = (sizes[a] < min_faces) & ok
vb = (sizes[b] < min_faces) & ok
best.scatter_reduce_(0, a[va], key[va], reduce="amin")
best.scatter_reduce_(0, b[vb], key[vb], reduce="amin")
src = (best < (1 << 62)).nonzero(as_tuple=True)[0]
if src.numel() == 0:
break
eid = best[src] & 0xFFFFFFFF
ea, eb = a[eid], b[eid]
tgt = torch.where(ea == src, eb, ea)
# cycle break: keep src->tgt only if tgt merges nowhere itself or src > tgt;
# the kept graph is then a DAG, so the pointer-doubling below terminates
prop = torch.arange(N, dtype=torch.long, device=device)
prop[src] = tgt
keepm = (prop[tgt] == tgt) | (src > tgt)
remap = torch.arange(N, dtype=torch.long, device=device)
remap[src[keepm]] = tgt[keepm]
for _ in range(32):
nr = remap[remap]
if torch.equal(nr, remap):
break
remap = nr
chart_id = remap[chart_id]
_, chart_id = torch.unique(chart_id, return_inverse=True)
return chart_id
def cluster_charts_pec(
mesh: MeshData,
max_cost: float = 0.7,
max_iters: int = 1024,
min_faces: int = 8,
progress_callback=None,
) -> Tensor:
"""Parallel edge-collapse clustering; returns face_chart [F]. max_cost is the per-merge
cutoff (~0.7 rad ~ 40deg); charts under min_faces are then absorbed at a relaxed 2x cutoff."""
device = mesh.faces.device
F = mesh.faces.shape[0]
faces = mesh.faces.to(torch.long)
vertices = mesh.vertices.to(torch.float32)
face_normal = mesh.face_normal.to(torch.float32)
face_face = mesh.face_face.to(torch.long)
face_edge_len = face_edge_lengths(vertices, faces)
chart_id = torch.arange(F, dtype=torch.long, device=device)
chart_axis = face_normal.clone()
chart_half = torch.zeros(F, dtype=torch.float32, device=device)
for it in range(max_iters):
edges, _ = _build_chart_edges(face_face, chart_id, face_edge_len)
if edges.shape[0] == 0:
break
a = edges[:, 0]
b = edges[:, 1]
axis_a = chart_axis[a]
axis_b = chart_axis[b]
half_a = chart_half[a]
half_b = chart_half[b]
_, new_half, _ = _combine_normal_cones(axis_a, half_a, axis_b, half_b)
cost = new_half.clone()
# Pack (cost, edge_id) so scatter_reduce amin picks the right edge.
E = edges.shape[0]
N = int(chart_id.max().item()) + 1
edge_ids = torch.arange(E, dtype=torch.long, device=device)
cost_i32 = torch.clamp(cost * 1e6, max=2e9).to(torch.int64)
key = (cost_i32 << 32) | edge_ids
chart_min = torch.full((N,), (2**62), dtype=torch.long, device=device)
chart_min.scatter_reduce_(0, a, key, reduce="amin", include_self=True)
chart_min.scatter_reduce_(0, b, key, reduce="amin", include_self=True)
# Mutual-min collapse: each chart in at most one merge per iter (winners are disjoint pairs).
is_a_min = chart_min[a] == key
is_b_min = chart_min[b] == key
mutual = is_a_min & is_b_min
within = cost <= max_cost
winners = mutual & within
n_merge = int(winners.sum().item())
if n_merge == 0:
break
if progress_callback is not None:
progress_callback(F - N + n_merge, F) # saturating: charts remaining vs faces
win_a = a[winners]
win_b = b[winners]
axis_a_w = chart_axis[win_a]
half_a_w = chart_half[win_a]
axis_b_w = chart_axis[win_b]
half_b_w = chart_half[win_b]
new_axis, new_half_w, _ = _combine_normal_cones(
axis_a_w, half_a_w, axis_b_w, half_b_w,
)
chart_axis[win_a] = new_axis
chart_half[win_a] = new_half_w
remap = torch.arange(N, dtype=torch.long, device=device)
remap[win_b] = win_a
chart_id = remap[chart_id]
if min_faces > 1:
chart_id = _merge_small_charts(chart_id, face_normal, mesh.face_area.to(torch.float32),
face_face, face_edge_len, min_faces, 2.0 * max_cost)
_, inverse = torch.unique(chart_id, sorted=True, return_inverse=True)
return inverse

File diff suppressed because it is too large Load Diff

View File

@ -277,13 +277,30 @@ class RescaleCFG:
CATEGORY = "model/patch"
def patch(self, model, multiplier):
model_sampling = model.get_model_object("model_sampling")
is_x0_space = not isinstance(model_sampling, comfy.model_sampling.EPS)
def rescale_cfg(args):
x_orig = args["input"]
cond_scale = args["cond_scale"]
if is_x0_space:
# Flow-matching / X0 models: cond_denoised/uncond_denoised are x_0 estimates,
# so the eps↔v conversion below would be wrong. Rescale directly in x_0 space.
x_0_cond = args["cond_denoised"]
x_0_uncond = args["uncond_denoised"]
x_0_cfg = x_0_uncond + cond_scale * (x_0_cond - x_0_uncond)
dims = tuple(range(1, x_0_cond.ndim))
ro_pos = x_0_cond.std(dim=dims, keepdim=True)
ro_cfg = x_0_cfg.std(dim=dims, keepdim=True).clamp(min=1e-8)
x_0_rescaled = x_0_cfg * (ro_pos / ro_cfg)
x_0_final = multiplier * x_0_rescaled + (1.0 - multiplier) * x_0_cfg
return x_orig - x_0_final
cond = args["cond"]
uncond = args["uncond"]
cond_scale = args["cond_scale"]
sigma = args["sigma"]
sigma = sigma.view(sigma.shape[:1] + (1,) * (cond.ndim - 1))
x_orig = args["input"]
#rescale cfg has to be done on v-pred model output
x = x_orig / (sigma * sigma + 1.0)

View File

@ -2,6 +2,7 @@
import torch
import math
import comfy.utils
import folder_paths
@ -391,10 +392,57 @@ class MoGePointMapToMesh(io.ComfyNode):
return io.NodeOutput(mesh)
class MoGeGeometryToFOV(io.ComfyNode):
"""Extract horizontal/vertical FOV from MoGe intrinsics, e.g. fov_y to feed SAM3DBody_Predict."""
@classmethod
def define_schema(cls):
return io.Schema(
node_id="MoGeGeometryToFOV",
search_aliases=["moge", "fov", "geometry", "intrinsics", "field of view"],
display_name="Get FoV from MoGe Geometry",
description="Derive the field of view and focal length from MoGe intrinsics.",
category="image/geometry estimation",
inputs=[
MoGeGeometry.Input("moge_geometry"),
io.Combo.Input("axis", options=["vertical", "horizontal", "diagonal"], default="vertical",
tooltip="'vertical' (fov_y), 'horizontal' (fov_x), or 'diagonal'."),
io.Combo.Input("unit", options=["degrees", "radians"], default="degrees",
tooltip="Output unit for the FOV."),
],
outputs=[
io.Float.Output(display_name="fov"),
io.Float.Output(display_name="focal_pixels"),
],
)
@classmethod
def execute(cls, moge_geometry, axis, unit) -> io.NodeOutput:
K = moge_geometry.get("intrinsics") if isinstance(moge_geometry, dict) else None
if K is None:
raise ValueError("moge_geometry has no intrinsics (panorama geometry has none).")
if K.ndim == 3:
K = K[0]
# MoGe normalizes fx by width and fy by height; with cx=cy=0.5 the half-extent
# in normalized units is 0.5, so fov = 2*atan(0.5 / f) per axis (hypot for diagonal).
hx = 0.5 / float(K[0, 0].item())
hy = 0.5 / float(K[1, 1].item())
half_tan = {"horizontal": hx, "vertical": hy, "diagonal": math.hypot(hx, hy)}[axis]
fov_radians = 2.0 * math.atan(half_tan)
fov = fov_radians if unit == "radians" else math.degrees(fov_radians)
# Pixels are square here, so fy*H == fx*W is the single lens focal in pixels.
src = next((moge_geometry[k] for k in ("image", "points", "depth") if k in moge_geometry), None)
if src is None:
raise ValueError("moge_geometry has no image/points/depth to read the pixel height from.")
H = int(src.shape[1])
focal_pixels = float(K[1, 1].item()) * H
return io.NodeOutput(fov, focal_pixels)
class MoGeExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[io.ComfyNode]]:
return [LoadMoGeModel, MoGeInference, MoGePanoramaInference, MoGeRender, MoGePointMapToMesh]
return [LoadMoGeModel, MoGeInference, MoGePanoramaInference, MoGeRender, MoGePointMapToMesh, MoGeGeometryToFOV]
async def comfy_entrypoint() -> MoGeExtension:

View File

@ -1,10 +1,13 @@
"""Save-side 3D nodes: mesh packing/slicing helpers + GLB writer + SaveGLB node."""
import copy
import json
import logging
import math
import os
import struct
from io import BytesIO
from typing import TypedDict
import numpy as np
from PIL import Image
@ -13,14 +16,16 @@ from typing_extensions import override
import folder_paths
from comfy.cli_args import args
from comfy_api.latest import ComfyExtension, IO, Types
from comfy_api.latest import ComfyExtension, IO, Types, UI
from server import PromptServer
def pack_variable_mesh_batch(vertices, faces, colors=None, uvs=None, texture=None, unlit=False):
# Pack lists of (Nᵢ, *) vertex/face/color/uv tensors into padded batched tensors,
# stashing per-item lengths as runtime attrs so consumers can recover the real slice.
# colors and uvs are 1:1 with vertices, so they're padded to max_vertices and read with vertex_counts.
# texture is (B, H, W, 3) — passed through unchanged
def pack_variable_mesh_batch(vertices, faces, colors=None, uvs=None, texture=None, unlit=False,
normals=None, metallic_roughness=None, tangents=None, normal_map=None,
occlusion_in_mr=False, material=None, emissive=None):
# Pack per-item tensors into padded batches, stashing per-item lengths as runtime attrs.
# colors/uvs/normals/tangents are 1:1 with vertices (padded to max_vertices); texture/
# metallic_roughness/normal_map are (B,H,W,*) image stacks passed through unchanged.
batch_size = len(vertices)
max_vertices = max(v.shape[0] for v in vertices)
max_faces = max(f.shape[0] for f in faces)
@ -52,43 +57,86 @@ def pack_variable_mesh_batch(vertices, faces, colors=None, uvs=None, texture=Non
)
packed_uvs[i, :u.shape[0]] = u
packed_normals = None
if normals is not None:
packed_normals = normals[0].new_zeros((batch_size, max_vertices, normals[0].shape[1]))
for i, nrm in enumerate(normals):
assert nrm.shape[0] == vertices[i].shape[0], (
f"normals[{i}] has {nrm.shape[0]} entries, expected {vertices[i].shape[0]} (1:1 with vertices)"
)
packed_normals[i, :nrm.shape[0]] = nrm
packed_tangents = None
if tangents is not None:
packed_tangents = tangents[0].new_zeros((batch_size, max_vertices, tangents[0].shape[1]))
for i, tn in enumerate(tangents):
assert tn.shape[0] == vertices[i].shape[0], (
f"tangents[{i}] has {tn.shape[0]} entries, expected {vertices[i].shape[0]} (1:1 with vertices)"
)
packed_tangents[i, :tn.shape[0]] = tn
return Types.MESH(packed_vertices, packed_faces,
uvs=packed_uvs, vertex_colors=packed_colors, texture=texture,
vertex_counts=vertex_counts, face_counts=face_counts, unlit=unlit)
metallic_roughness=metallic_roughness,
vertex_counts=vertex_counts, face_counts=face_counts, unlit=unlit,
normals=packed_normals, tangents=packed_tangents,
normal_map=normal_map, occlusion_in_mr=occlusion_in_mr,
material=material, emissive=emissive)
def get_mesh_batch_item(mesh, index):
# Returns (vertices, faces, colors, uvs) for batch index, slicing to real lengths
# if the mesh carries per-item counts (variable-size batch).
v_colors = getattr(mesh, "vertex_colors", None)
v_uvs = getattr(mesh, "uvs", None)
if getattr(mesh, "vertex_counts", None) is not None:
v_colors = mesh.vertex_colors
v_uvs = mesh.uvs
v_normals = mesh.normals
if mesh.vertex_counts is not None:
vertex_count = int(mesh.vertex_counts[index].item())
face_count = int(mesh.face_counts[index].item())
vertices = mesh.vertices[index, :vertex_count]
faces = mesh.faces[index, :face_count]
colors = v_colors[index, :vertex_count] if v_colors is not None else None
uvs = v_uvs[index, :vertex_count] if v_uvs is not None else None
return vertices, faces, colors, uvs
normals = v_normals[index, :vertex_count] if v_normals is not None else None
return vertices, faces, colors, uvs, normals
colors = v_colors[index] if v_colors is not None else None
uvs = v_uvs[index] if v_uvs is not None else None
return mesh.vertices[index], mesh.faces[index], colors, uvs
normals = v_normals[index] if v_normals is not None else None
return mesh.vertices[index], mesh.faces[index], colors, uvs, normals
def save_glb(vertices, faces, filepath, metadata=None,
uvs=None, vertex_colors=None, texture_image=None, unlit=False):
def save_glb(vertices, faces, filepath=None, metadata=None,
uvs=None, vertex_colors=None, texture_image=None,
metallic_roughness_image=None, unlit=False,
normals=None, normal_map_image=None, tangents=None, occlusion_in_mr=False,
material=None, emissive_image=None):
"""
Save PyTorch tensor vertices and faces as a GLB file without external dependencies.
Parameters:
vertices: torch.Tensor of shape (N, 3) - The vertex coordinates
faces: torch.Tensor of shape (M, 3) - The face indices (triangle faces)
filepath: str - Output filepath (should end with .glb)
filepath: str - Output filepath (should end with .glb). None returns the GLB bytes instead of writing.
metadata: dict - Optional asset.extras metadata
uvs: torch.Tensor of shape (N, 2) - Optional per-vertex texture coordinates
vertex_colors: torch.Tensor of shape (N, 3) or (N, 4) - Optional per-vertex colors in [0, 1]
texture_image: PIL.Image - Optional baseColor texture, embedded as PNG
metallic_roughness_image: PIL.Image - Optional glTF metallicRoughness texture
(R unused, G=roughness, B=metallic), embedded as PNG
normals: torch.Tensor of shape (N, 3) - Optional per-vertex normals, written as the
glTF NORMAL attribute. When omitted, NO normals are written and viewers fall back
to flat (per-face) shading use the MeshSmoothNormals node to generate them.
normal_map_image: PIL.Image - Optional tangent-space normal map (glTF/OpenGL +Y),
written as the material normalTexture. Needs TEXCOORD_0.
tangents: torch.Tensor of shape (N, 4) - Optional per-vertex tangents (xyz + handedness w),
written as the glTF TANGENT attribute. Without it viewers derive tangents in-shader.
occlusion_in_mr: bool - When True, R of metallic_roughness_image holds AO (ORM packing) and
occlusionTexture is pointed at that same image.
material: dict - Optional scalar overrides from SetMeshMaterial (base_color_factor,
metallic/roughness_factor with <0 = auto, emissive_factor/strength, normal_scale,
occlusion_strength, double_sided).
emissive_image: PIL.Image - Optional emissive (glow) texture, written as emissiveTexture.
"""
# Convert tensors to numpy arrays
@ -117,44 +165,82 @@ def save_glb(vertices, faces, filepath, metadata=None,
raise ValueError(
f"save_glb: vertex_colors has {colors_np.shape[0]} entries but vertex count is {n_verts}"
)
normals_np = normals.cpu().numpy().astype(np.float32) if normals is not None else None
if normals_np is not None and normals_np.shape[0] != n_verts:
raise ValueError(
f"save_glb: normals has {normals_np.shape[0]} entries but vertex count is {n_verts}"
)
tangents_np = tangents.cpu().numpy().astype(np.float32) if tangents is not None else None
if tangents_np is not None and tangents_np.shape != (n_verts, 4):
raise ValueError(
f"save_glb: tangents must be (N, 4) with N={n_verts}, got {tuple(tangents_np.shape)}"
)
faces_np = faces_signed.astype(np.uint32)
texture_png_bytes = None
if texture_image is not None:
buf = BytesIO()
texture_image.save(buf, format="PNG")
texture_png_bytes = buf.getvalue()
mr_png_bytes = None
if metallic_roughness_image is not None:
buf = BytesIO()
metallic_roughness_image.save(buf, format="PNG")
mr_png_bytes = buf.getvalue()
nm_png_bytes = None
if normal_map_image is not None:
buf = BytesIO()
normal_map_image.save(buf, format="PNG")
nm_png_bytes = buf.getvalue()
em_png_bytes = None
if emissive_image is not None:
buf = BytesIO()
emissive_image.save(buf, format="PNG")
em_png_bytes = buf.getvalue()
vertices_buffer = vertices_np.tobytes()
indices_buffer = faces_np.tobytes()
uvs_buffer = uvs_np.tobytes() if uvs_np is not None else b""
colors_buffer = colors_np.tobytes() if colors_np is not None else b""
normals_buffer = normals_np.tobytes() if normals_np is not None else b""
tangents_buffer = tangents_np.tobytes() if tangents_np is not None else b""
texture_buffer = texture_png_bytes if texture_png_bytes is not None else b""
mr_buffer = mr_png_bytes if mr_png_bytes is not None else b""
nm_buffer = nm_png_bytes if nm_png_bytes is not None else b""
em_buffer = em_png_bytes if em_png_bytes is not None else b""
def pad_to_4_bytes(buffer):
padding_length = (4 - (len(buffer) % 4)) % 4
return buffer + b'\x00' * padding_length
vertices_buffer_padded = pad_to_4_bytes(vertices_buffer)
indices_buffer_padded = pad_to_4_bytes(indices_buffer)
uvs_buffer_padded = pad_to_4_bytes(uvs_buffer)
colors_buffer_padded = pad_to_4_bytes(colors_buffer)
texture_buffer_padded = pad_to_4_bytes(texture_buffer)
buffer_data = b"".join([
vertices_buffer_padded,
indices_buffer_padded,
uvs_buffer_padded,
colors_buffer_padded,
texture_buffer_padded,
])
# Blob order in one place; offsets accumulated in a pass so adding a buffer is one entry.
_blobs = [
("vertices", vertices_buffer), ("indices", indices_buffer), ("uvs", uvs_buffer),
("colors", colors_buffer), ("normals", normals_buffer), ("tangents", tangents_buffer),
("texture", texture_buffer), ("mr", mr_buffer), ("nm", nm_buffer), ("em", em_buffer),
]
byte_offset = {}
acc = 0
parts = []
for name, b in _blobs:
padded = pad_to_4_bytes(b)
byte_offset[name] = acc
acc += len(padded)
parts.append(padded)
buffer_data = b"".join(parts)
vertices_byte_length = len(vertices_buffer)
vertices_byte_offset = 0
indices_byte_length = len(indices_buffer)
indices_byte_offset = len(vertices_buffer_padded)
uvs_byte_offset = indices_byte_offset + len(indices_buffer_padded)
colors_byte_offset = uvs_byte_offset + len(uvs_buffer_padded)
texture_byte_offset = colors_byte_offset + len(colors_buffer_padded)
vertices_byte_offset = byte_offset["vertices"]
indices_byte_offset = byte_offset["indices"]
uvs_byte_offset = byte_offset["uvs"]
colors_byte_offset = byte_offset["colors"]
normals_byte_offset = byte_offset["normals"]
tangents_byte_offset = byte_offset["tangents"]
texture_byte_offset = byte_offset["texture"]
mr_byte_offset = byte_offset["mr"]
nm_byte_offset = byte_offset["nm"]
em_byte_offset = byte_offset["em"]
buffer_views = [
{
@ -224,6 +310,40 @@ def save_glb(vertices, faces, filepath, metadata=None,
})
primitive_attributes["COLOR_0"] = accessor_idx
if normals_np is not None and len(normals_np) > 0:
buffer_views.append({
"buffer": 0,
"byteOffset": normals_byte_offset,
"byteLength": len(normals_buffer),
"target": 34962
})
accessor_idx = len(accessors)
accessors.append({
"bufferView": len(buffer_views) - 1,
"byteOffset": 0,
"componentType": 5126, # FLOAT
"count": len(normals_np),
"type": "VEC3",
})
primitive_attributes["NORMAL"] = accessor_idx
if tangents_np is not None and len(tangents_np) > 0:
buffer_views.append({
"buffer": 0,
"byteOffset": tangents_byte_offset,
"byteLength": len(tangents_buffer),
"target": 34962
})
accessor_idx = len(accessors)
accessors.append({
"bufferView": len(buffer_views) - 1,
"byteOffset": 0,
"componentType": 5126, # FLOAT
"count": len(tangents_np),
"type": "VEC4", # xyz tangent + w handedness (glTF TANGENT)
})
primitive_attributes["TANGENT"] = accessor_idx
primitive = {
"attributes": primitive_attributes,
"indices": 1,
@ -235,9 +355,24 @@ def save_glb(vertices, faces, filepath, metadata=None,
samplers = []
materials = []
extensions_used = []
def add_image_texture(png_byte_offset, png_byte_length):
"""Append an embedded PNG image + a texture referencing it; return the texture index."""
buffer_views.append({"buffer": 0, "byteOffset": png_byte_offset, "byteLength": png_byte_length})
images.append({"bufferView": len(buffer_views) - 1, "mimeType": "image/png"})
if not samplers:
samplers.append({"magFilter": 9729, "minFilter": 9729, "wrapS": 33071, "wrapT": 33071})
textures.append({"source": len(images) - 1, "sampler": 0})
return len(textures) - 1
has_uv = "TEXCOORD_0" in primitive_attributes
if unlit and texture_png_bytes is None:
# Flat, light-independent shading (KHR_materials_unlit): COLOR_0 is shown as-is, matching how a
# gaussian splat renders (emissive). Without this the viewer lights the mesh and washes the colours.
if nm_png_bytes is not None or em_png_bytes is not None or occlusion_in_mr or material is not None:
logging.warning(
"save_glb: unlit material ignores normal/occlusion/emissive maps and SetMeshMaterial "
"overrides — those are PBR-lit features. Disable unlit to export them.")
materials.append({
"pbrMetallicRoughness": {"baseColorFactor": [1.0, 1.0, 1.0, 1.0], "metallicFactor": 0.0, "roughnessFactor": 1.0},
"extensions": {"KHR_materials_unlit": {}},
@ -245,23 +380,67 @@ def save_glb(vertices, faces, filepath, metadata=None,
})
extensions_used.append("KHR_materials_unlit")
primitive["material"] = 0
if texture_png_bytes is not None and "TEXCOORD_0" in primitive_attributes:
buffer_views.append({
"buffer": 0,
"byteOffset": texture_byte_offset,
"byteLength": len(texture_buffer),
})
images.append({"bufferView": len(buffer_views) - 1, "mimeType": "image/png"})
samplers.append({"magFilter": 9729, "minFilter": 9729, "wrapS": 33071, "wrapT": 33071})
textures.append({"source": 0, "sampler": 0})
materials.append({
"pbrMetallicRoughness": {
"baseColorTexture": {"index": 0, "texCoord": 0},
"metallicFactor": 0.0,
"roughnessFactor": 1.0,
},
"doubleSided": True,
})
else:
pbr = {
"metallicFactor": 0.0,
"roughnessFactor": 0.5,
"baseColorFactor": [0.22, 0.22, 0.22, 1.0], # neutral-gray fallback for bare geometry only
}
if texture_png_bytes is not None and has_uv:
pbr["baseColorTexture"] = {"index": add_image_texture(texture_byte_offset, len(texture_buffer)), "texCoord": 0}
if (texture_png_bytes is not None and has_uv) or "COLOR_0" in primitive_attributes:
pbr["baseColorFactor"] = [1.0, 1.0, 1.0, 1.0]
pbr["roughnessFactor"] = 1.0
if mr_png_bytes is not None and has_uv:
mr_texture_index = add_image_texture(mr_byte_offset, len(mr_buffer))
pbr["metallicRoughnessTexture"] = {"index": mr_texture_index, "texCoord": 0}
# When a metallicRoughness texture is present, the factors scale it; use 1.0
# so the texture values pass through unchanged (glTF convention).
pbr["metallicFactor"] = 1.0
pbr["roughnessFactor"] = 1.0
mat = material if isinstance(material, dict) else {}
# Scalar overrides from SetMeshMaterial (factor < 0 means "leave auto").
if mat.get("base_color_factor") is not None:
pbr["baseColorFactor"] = [float(x) for x in mat["base_color_factor"]]
if mat.get("metallic_factor", -1.0) >= 0.0:
pbr["metallicFactor"] = float(mat["metallic_factor"])
if mat.get("roughness_factor", -1.0) >= 0.0:
pbr["roughnessFactor"] = float(mat["roughness_factor"])
material = {
"pbrMetallicRoughness": pbr,
"doubleSided": bool(mat.get("double_sided", True)),
}
if occlusion_in_mr and mr_png_bytes is not None and has_uv:
# ORM packing: occlusionTexture reuses the MR image (glTF reads its R channel).
material["occlusionTexture"] = {"index": mr_texture_index, "texCoord": 0,
"strength": float(mat.get("occlusion_strength", 1.0))}
if nm_png_bytes is not None and has_uv:
material["normalTexture"] = {"index": add_image_texture(nm_byte_offset, len(nm_buffer)),
"texCoord": 0, "scale": float(mat.get("normal_scale", 1.0))}
emissive_factor = [float(x) for x in mat.get("emissive_factor", [0.0, 0.0, 0.0])]
emissive_strength = float(mat.get("emissive_strength", 1.0))
has_em_tex = em_png_bytes is not None and has_uv
if any(c > 0.0 for c in emissive_factor) or has_em_tex:
# glTF multiplies emissiveFactor × texture, so a texture with no color would go black;
# default the factor to white in that case.
if has_em_tex and not any(c > 0.0 for c in emissive_factor):
emissive_factor = [1.0, 1.0, 1.0]
material["emissiveFactor"] = [min(1.0, c) for c in emissive_factor]
if has_em_tex:
material["emissiveTexture"] = {"index": add_image_texture(em_byte_offset, len(em_buffer)),
"texCoord": 0}
if emissive_strength != 1.0:
material.setdefault("extensions", {})["KHR_materials_emissive_strength"] = {
"emissiveStrength": emissive_strength}
if "KHR_materials_emissive_strength" not in extensions_used:
extensions_used.append("KHR_materials_emissive_strength")
materials.append(material)
primitive["material"] = 0
gltf = {
@ -306,17 +485,48 @@ def save_glb(vertices, faces, filepath, metadata=None,
# Create BIN chunk header (chunk type 1)
bin_chunk_header = struct.pack('<II', len(buffer_data), 0x004E4942) # "BIN\0" in little endian
# Write the GLB file
glb = b"".join([glb_header, json_chunk_header, gltf_json_padded, bin_chunk_header, buffer_data])
if filepath is None:
return glb # in-memory GLB bytes (e.g. for a File3D object)
with open(filepath, 'wb') as f:
f.write(glb_header)
f.write(json_chunk_header)
f.write(gltf_json_padded)
f.write(bin_chunk_header)
f.write(buffer_data)
f.write(glb)
return filepath
def mesh_item_to_glb_bytes(mesh, index, metadata=None):
"""Serialize one batch item of a MESH to in-memory GLB bytes, carrying every PBR attribute
(uvs, colors, normals, texture, ORM/occlusion, normal map + tangents, emissive, material).
Returns None for an empty item. Shared by SaveGLB (per item) and MeshToFile3D."""
vertices_i, faces_i, v_colors, uvs_i, normals_i = get_mesh_batch_item(mesh, index)
if vertices_i.shape[0] == 0 or faces_i.shape[0] == 0:
return None
def _img(attr):
t = getattr(mesh, attr, None)
if t is None:
return None
a = (t[index].clamp(0.0, 1.0).cpu().numpy() * 255).astype(np.uint8)
assert a.ndim == 3 and a.shape[-1] == 3, f"{attr} must be (B, H, W, 3), got {tuple(t.shape)}"
return Image.fromarray(a, mode="RGB")
tangents_b = mesh.tangents
tangents_i = tangents_b[index, :vertices_i.shape[0]] if tangents_b is not None else None
return save_glb(
vertices_i, faces_i, None, metadata,
uvs=uvs_i,
vertex_colors=v_colors,
texture_image=_img("texture"),
metallic_roughness_image=_img("metallic_roughness"),
unlit=mesh.unlit,
normals=normals_i,
normal_map_image=_img("normal_map"),
tangents=tangents_i,
occlusion_in_mr=mesh.occlusion_in_mr,
material=mesh.material,
emissive_image=_img("emissive"),
)
class SaveGLB(IO.ComfyNode):
@classmethod
def define_schema(cls):
@ -378,25 +588,14 @@ class SaveGLB(IO.ComfyNode):
counter += 1
else:
# Handle Mesh input - save vertices and faces as GLB; carry optional UVs / colors / texture.
texture_b = getattr(mesh, "texture", None)
texture_np = None
if texture_b is not None:
texture_np = (texture_b.clamp(0.0, 1.0).cpu().numpy() * 255).astype(np.uint8)
assert texture_np.ndim == 4 and texture_np.shape[-1] == 3, (
f"texture must be (B, H, W, 3) RGB, got shape {tuple(texture_np.shape)}"
)
for i in range(mesh.vertices.shape[0]):
vertices_i, faces_i, v_colors, uvs_i = get_mesh_batch_item(mesh, i)
if vertices_i.shape[0] == 0 or faces_i.shape[0] == 0:
glb = mesh_item_to_glb_bytes(mesh, i, metadata)
if glb is None:
logging.warning(f"SaveGLB: skipping empty mesh at batch index {i}")
continue
tex_img = Image.fromarray(texture_np[i], mode="RGB") if texture_np is not None else None
f = f"{filename}_{counter:05}_.glb"
save_glb(vertices_i, faces_i, os.path.join(full_output_folder, f), metadata,
uvs=uvs_i,
vertex_colors=v_colors,
texture_image=tex_img,
unlit=getattr(mesh, "unlit", False))
with open(os.path.join(full_output_folder, f), "wb") as fh:
fh.write(glb)
results.append({
"filename": f,
"subfolder": subfolder,
@ -406,10 +605,273 @@ class SaveGLB(IO.ComfyNode):
return IO.NodeOutput(ui={"3d": results})
class MeshToFile3D(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="MeshToFile3D",
display_name="Create 3D File (from Mesh)",
search_aliases=["mesh to glb", "mesh to file", "export mesh"],
category="3d",
description="Serialize a mesh to a GLB File3D object for Save / Preview 3D nodes, "
"carrying its UVs, colors, normals, texture, normal/occlusion/emissive "
"maps and material. Supports one item per batch only.",
inputs=[IO.Mesh.Input("mesh")],
outputs=[IO.File3DGLB.Output(display_name="model_3d")],
)
@classmethod
def execute(cls, mesh) -> IO.NodeOutput:
if mesh.vertices.shape[0] > 1:
logging.warning("MeshToFile3D supports one item per batch only. Got %d; using first.",
mesh.vertices.shape[0])
glb = mesh_item_to_glb_bytes(mesh, 0)
if glb is None:
raise ValueError("MeshToFile3D: mesh is empty (no vertices/faces).")
return IO.NodeOutput(Types.File3D(BytesIO(glb), file_format="glb"))
class RotateMesh(IO.ComfyNode):
class ModeValues(TypedDict, total=False):
mode: str
angle_x: float
angle_y: float
angle_z: float
qw: float
qx: float
qy: float
qz: float
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="RotateMesh",
display_name="Rotate Mesh",
category="3d/mesh",
description=(
"Rotate a mesh. Euler XYZ applies X then Y then Z about the world axes (degrees). "
"Quaternion is (w, x, y, z), auto-normalized."
),
inputs=[
IO.Mesh.Input("mesh"),
IO.DynamicCombo.Input(
"mode",
options=[
IO.DynamicCombo.Option("euler_xyz", [
IO.Float.Input("angle_x", default=0.0, min=-360.0, max=360.0, step=0.1,
tooltip="Rotation around the X axis in degrees."),
IO.Float.Input("angle_y", default=0.0, min=-360.0, max=360.0, step=0.1,
tooltip="Rotation around the Y axis in degrees."),
IO.Float.Input("angle_z", default=0.0, min=-360.0, max=360.0, step=0.1,
tooltip="Rotation around the Z axis in degrees."),
]),
IO.DynamicCombo.Option("quaternion", [
IO.Float.Input("qw", default=1.0, min=-1.0, max=1.0, step=0.001),
IO.Float.Input("qx", default=0.0, min=-1.0, max=1.0, step=0.001),
IO.Float.Input("qy", default=0.0, min=-1.0, max=1.0, step=0.001),
IO.Float.Input("qz", default=0.0, min=-1.0, max=1.0, step=0.001),
]),
],
),
],
outputs=[IO.Mesh.Output("mesh")],
)
@classmethod
def execute(cls, mesh: Types.MESH, mode: ModeValues) -> IO.NodeOutput:
mode_name = mode["mode"]
if mode_name == "euler_xyz":
ax = math.radians(mode["angle_x"])
ay = math.radians(mode["angle_y"])
az = math.radians(mode["angle_z"])
if ax == 0.0 and ay == 0.0 and az == 0.0:
return IO.NodeOutput(mesh)
cx, sx = math.cos(ax), math.sin(ax)
cy, sy = math.cos(ay), math.sin(ay)
cz, sz = math.cos(az), math.sin(az)
R_rows = [
[cy * cz, sx * sy * cz - cx * sz, cx * sy * cz + sx * sz],
[cy * sz, sx * sy * sz + cx * cz, cx * sy * sz - sx * cz],
[-sy, sx * cy, cx * cy],
]
elif mode_name == "quaternion":
qw, qx, qy, qz = mode["qw"], mode["qx"], mode["qy"], mode["qz"]
n = math.sqrt(qw * qw + qx * qx + qy * qy + qz * qz)
if n < 1e-8:
raise ValueError("RotateMesh: quaternion has zero magnitude")
qw, qx, qy, qz = qw / n, qx / n, qy / n, qz / n
if qw == 1.0 and qx == 0.0 and qy == 0.0 and qz == 0.0:
return IO.NodeOutput(mesh)
R_rows = [
[1 - 2 * (qy * qy + qz * qz), 2 * (qx * qy - qz * qw), 2 * (qx * qz + qy * qw)],
[2 * (qx * qy + qz * qw), 1 - 2 * (qx * qx + qz * qz), 2 * (qy * qz - qx * qw)],
[2 * (qx * qz - qy * qw), 2 * (qy * qz + qx * qw), 1 - 2 * (qx * qx + qy * qy)],
]
else:
raise ValueError(f"RotateMesh: unknown mode {mode_name!r}")
def rotate(v: torch.Tensor) -> torch.Tensor:
R = torch.tensor(R_rows, device=v.device, dtype=v.dtype)
return v @ R.T
out = copy.copy(mesh)
if isinstance(mesh.vertices, list):
out.vertices = [rotate(v) for v in mesh.vertices]
else:
out.vertices = rotate(mesh.vertices)
# Normals are directions; rotate them too (R is orthogonal) so they stay valid.
nrm = mesh.normals
if nrm is not None:
out.normals = [rotate(n) for n in nrm] if isinstance(nrm, list) else rotate(nrm)
return IO.NodeOutput(out)
class MergeMeshes(IO.ComfyNode):
@classmethod
def define_schema(cls):
autogrow_template = IO.Autogrow.TemplatePrefix(
IO.Mesh.Input("mesh"), prefix="mesh", min=2, max=50,
)
return IO.Schema(
node_id="MergeMeshes",
display_name="Merge Meshes",
category="3d/mesh",
description=(
"Concatenate N meshes into one by offsetting face indices and stacking verts, "
"faces, uvs, and colors."
),
inputs=[
IO.Autogrow.Input("meshes", template=autogrow_template),
],
outputs=[IO.Mesh.Output("mesh")],
)
@classmethod
def execute(cls, meshes: IO.Autogrow.Type) -> IO.NodeOutput:
# Concatenate the input meshes into one (B=1) mesh: cumulative face-index offset,
# missing uvs/colors padded (zeros/white), texture from the first input that has one
# (later dropped — a single-primitive glb can't carry multiple atlases).
meshes = list(meshes.values())
if not meshes:
raise ValueError("MergeMeshes: need at least one mesh")
def _b0(t):
return t[0] if t.ndim == 3 else t
any_uvs = any(m.uvs is not None for m in meshes)
any_colors = any(m.vertex_colors is not None for m in meshes)
verts_list, faces_list, uvs_list, colors_list = [], [], [], []
texture = None
offset = 0
for m in meshes:
# Coerce to CPU so CUDA-side (MoGe) meshes merge cleanly with our outputs.
v = _b0(m.vertices).cpu()
f = _b0(m.faces).cpu()
verts_list.append(v)
faces_list.append(f + offset)
offset += v.shape[0]
if any_uvs:
mu = m.uvs
uvs_list.append(_b0(mu).cpu() if mu is not None else v.new_zeros((v.shape[0], 2)))
if any_colors:
mc = m.vertex_colors
c = _b0(mc).cpu() if mc is not None else v.new_ones((v.shape[0], 3))
colors_list.append(c)
mt = m.texture
if mt is not None:
if texture is None:
texture = mt.cpu()
else:
logging.warning("MergeMeshes: dropping extra texture from input; only one texture is kept.")
merged_verts = torch.cat(verts_list, dim=0).unsqueeze(0)
merged_faces = torch.cat(faces_list, dim=0).unsqueeze(0)
merged_uvs = torch.cat(uvs_list, dim=0).unsqueeze(0) if any_uvs else None
merged_colors = torch.cat(colors_list, dim=0).unsqueeze(0) if any_colors else None
return IO.NodeOutput(Types.MESH(
vertices=merged_verts,
faces=merged_faces,
uvs=merged_uvs,
vertex_colors=merged_colors,
texture=texture,
))
class GetMeshInfo(IO.ComfyNode):
"""Report vertex / face counts and attributes for a MESH, displayed on the
node (and as a string output). Counts are comma-formatted since meshes can
run into the millions of faces. Passes the mesh through unchanged."""
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="GetMeshInfo",
display_name="Get Mesh Info",
category="3d/mesh",
inputs=[IO.Mesh.Input("mesh")],
outputs=[
IO.Mesh.Output(display_name="mesh"),
IO.String.Output(display_name="info"),
],
hidden=[IO.Hidden.unique_id],
)
@staticmethod
def _fmt(n: int) -> str:
# e.g. 1234567 -> "1,234,567 (1.23M)"; small numbers stay plain.
s = f"{n:,}"
if n >= 1_000_000:
s += f" ({n / 1_000_000:.2f}M)"
elif n >= 10_000:
s += f" ({n / 1_000:.1f}K)"
return s
@classmethod
def execute(cls, mesh):
B = mesh.vertices.shape[0]
# Honour per-item counts when the batch is zero-padded; else use the row sizes.
if mesh.vertex_counts is not None:
v_counts = [int(x) for x in mesh.vertex_counts.tolist()]
f_counts = [int(x) for x in mesh.face_counts.tolist()]
else:
v_counts = [int(mesh.vertices.shape[1])] * B
f_counts = [int(mesh.faces.shape[1])] * B
attrs = []
for name in ("uvs", "vertex_colors", "normals", "tangents", "texture", "metallic_roughness", "normal_map"):
t = getattr(mesh, name, None)
if t is not None:
if name in ("texture", "metallic_roughness", "normal_map"):
attrs.append(f"{name} {int(t.shape[-3])}×{int(t.shape[-2])}") # H×W
else:
attrs.append(name)
lines = []
if B > 1:
lines.append(f"Batch: {B} meshes")
lines.append(f"Vertices: {cls._fmt(sum(v_counts))} total")
lines.append(f"Faces: {cls._fmt(sum(f_counts))} total")
for i in range(B):
lines.append(f" [{i}] {v_counts[i]:>10,} verts · {f_counts[i]:>10,} faces")
else:
lines.append(f"Vertices: {cls._fmt(v_counts[0])}")
lines.append(f"Faces: {cls._fmt(f_counts[0])}")
lines.append(f"Attributes: {', '.join(attrs) if attrs else 'none'}")
info = "\n".join(lines)
logging.info("[GetMeshInfo]\n%s", info)
if cls.hidden.unique_id:
PromptServer.instance.send_progress_text(info, cls.hidden.unique_id)
return IO.NodeOutput(mesh, info, ui=UI.PreviewText(info))
class Save3DExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
return [SaveGLB]
return [SaveGLB, MeshToFile3D, RotateMesh, MergeMeshes, GetMeshInfo]
async def comfy_entrypoint() -> Save3DExtension:

View File

@ -0,0 +1,947 @@
from typing_extensions import override
from comfy_api.latest import ComfyExtension, IO, Types, io
from comfy.ldm.trellis2.vae import SparseTensor
from comfy.ldm.trellis2.model import build_proj_transform_matrix, compute_stage_proj_feats
from comfy_extras.nodes_mesh_postprocess import pack_variable_mesh_batch
import comfy.latent_formats
import comfy.model_management
import comfy.utils
import logging
import math
import torch
ShapeSubdivides = io.Custom("SHAPE_SUBDIVIDES")
shape_slat_format = comfy.latent_formats.Trellis2ShapeSLAT()
tex_slat_format = comfy.latent_formats.Trellis2TexSLAT()
def shape_norm(shape_latent, coords):
feats = shape_slat_format.process_out(shape_latent)
return SparseTensor(feats=feats, coords=coords)
def infer_batched_coord_layout(coords):
if coords.ndim != 2 or coords.shape[1] != 4:
raise ValueError(f"Expected Trellis2 coords with shape [N, 4], got {tuple(coords.shape)}")
if coords.shape[0] == 0:
raise ValueError("Trellis2 coords can't be empty")
batch_ids = coords[:, 0].to(torch.int64)
if (batch_ids < 0).any():
raise ValueError(f"Trellis2 batch ids must be non-negative, got {batch_ids.unique(sorted=True).tolist()}")
batch_size = int(batch_ids.max().item()) + 1
counts = torch.bincount(batch_ids, minlength=batch_size)
if (counts == 0).any():
raise ValueError(f"Non-contiguous Trellis2 batch ids in coords: {batch_ids.unique(sorted=True).tolist()}")
max_tokens = int(counts.max().item())
return batch_size, counts, max_tokens
def split_batched_coords(coords, coord_counts):
if coord_counts.ndim != 1:
raise ValueError(f"Trellis2 coord_counts must be 1D, got shape {tuple(coord_counts.shape)}")
if (coord_counts < 0).any():
raise ValueError(f"Trellis2 coord_counts must be non-negative, got {coord_counts.tolist()}")
if int(coord_counts.sum().item()) != coords.shape[0]:
raise ValueError(
f"Trellis2 coord_counts total {int(coord_counts.sum().item())} does not match coords rows {coords.shape[0]}"
)
batch_ids = coords[:, 0].to(torch.int64)
order = torch.argsort(batch_ids, stable=True)
sorted_coords = coords.index_select(0, order)
sorted_batch_ids = batch_ids.index_select(0, order)
offsets = coord_counts.cumsum(0) - coord_counts
items = []
for i in range(coord_counts.shape[0]):
count = int(coord_counts[i].item())
start = int(offsets[i].item())
coords_i = sorted_coords[start:start + count]
ids_i = sorted_batch_ids[start:start + count]
if coords_i.shape[0] != count or not torch.all(ids_i == i):
raise ValueError(f"Trellis2 coords rows for batch {i} expected {count}, got {coords_i.shape[0]}")
items.append(coords_i)
return items
def flatten_batched_sparse_latent(samples, coords, coord_counts):
samples = samples.squeeze(-1).transpose(1, 2)
if coord_counts is None:
return samples.reshape(-1, samples.shape[-1]), coords
coords_items = split_batched_coords(coords, coord_counts)
feat_list = []
coord_list = []
for i, coords_i in enumerate(coords_items):
count = int(coord_counts[i].item())
feat_list.append(samples[i, :count])
coord_list.append(coords_i)
return torch.cat(feat_list, dim=0), torch.cat(coord_list, dim=0)
def split_batched_sparse_latent(samples, coords, coord_counts):
samples = samples.squeeze(-1).transpose(1, 2)
if coord_counts is None:
return [(samples.reshape(-1, samples.shape[-1]), coords)]
coords_items = split_batched_coords(coords, coord_counts)
items = []
for i, coords_i in enumerate(coords_items):
count = int(coord_counts[i].item())
items.append((samples[i, :count], coords_i))
return items
class VaeDecodeShapeTrellis(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="VaeDecodeShapeTrellis",
category="latent/3d",
inputs=[
IO.Latent.Input("samples"),
IO.Vae.Input("vae"),
],
outputs=[
IO.Mesh.Output("mesh"),
ShapeSubdivides.Output(display_name = "shape_subdivides"),
]
)
@classmethod
def execute(cls, samples, vae):
# Mesh grid_size must match the actual coord resolution the upstream
# stage was run at (1024 cascade -> 64, 1536 cascade -> 96). The VAE's
# built-in `.resolution` buffer defaults to 1024 and is otherwise stale;
# take coord_resolution from the latent dict if the stage node set it.
coord_resolution = samples.get("coord_resolution")
if coord_resolution is not None:
resolution = int(coord_resolution) * 16
else:
resolution = int(vae.first_stage_model.resolution.item())
model_frame = samples.get("model_frame", "y_up")
sample_tensor = samples["samples"]
device = comfy.model_management.get_torch_device()
coords = samples["coords"]
vae.prepare_decode(sample_tensor.shape)
trellis_vae = vae.first_stage_model
coord_counts = samples.get("coord_counts")
samples = samples["samples"]
if coord_counts is None:
samples, coords = flatten_batched_sparse_latent(samples, coords, coord_counts)
samples = shape_norm(samples.to(device), coords.to(device))
mesh, subs = trellis_vae.decode_shape_slat(samples.to(vae.vae_dtype), resolution)
else:
split_items = split_batched_sparse_latent(samples, coords, coord_counts)
mesh = []
subs_per_sample = []
for feats_i, coords_i in split_items:
coords_i = coords_i.to(device).clone()
coords_i[:, 0] = 0
sample_i = shape_norm(feats_i.to(device), coords_i)
mesh_i, subs_i = trellis_vae.decode_shape_slat(sample_i.to(vae.vae_dtype), resolution)
mesh.append(mesh_i[0])
subs_per_sample.append(subs_i)
subs = []
for stage_index in range(len(subs_per_sample[0])):
stage_tensors = [sample_subs[stage_index] for sample_subs in subs_per_sample]
feats_list = [stage_tensor.feats for stage_tensor in stage_tensors]
coords_list = [stage_tensor.coords for stage_tensor in stage_tensors]
subs.append(SparseTensor.from_tensor_list(feats_list, coords_list))
# Rotate Z-up (Trellis2 training frame) vertices to glTF Y-up. Pixal3D outputs are already Y-up.
if model_frame == "z_up":
vert_list = [torch.stack([v[..., 0], v[..., 2], -v[..., 1]], dim=-1).float().cpu()
for v, _ in mesh]
else:
vert_list = [v.float().cpu() for v, _ in mesh]
face_list = [f.int().cpu() for _, f in mesh]
if all(v.shape == vert_list[0].shape for v in vert_list) and all(f.shape == face_list[0].shape for f in face_list):
mesh = Types.MESH(vertices=torch.stack(vert_list), faces=torch.stack(face_list))
else:
mesh = pack_variable_mesh_batch(vert_list, face_list)
return IO.NodeOutput(mesh, subs)
class VaeDecodeTextureTrellis(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="VaeDecodeTextureTrellis",
category="latent/3d",
inputs=[
IO.Latent.Input("samples"),
IO.Vae.Input("vae"),
ShapeSubdivides.Input("shape_subdivides",
tooltip=(
"Shape information used to guide higher-detail reconstruction during decoding. "
"Helps preserve structure consistency at higher resolutions."
)),
],
outputs=[
IO.Voxel.Output("voxel_colors"),
]
)
@classmethod
def execute(cls, samples, vae, shape_subdivides):
sample_tensor = samples["samples"]
device = comfy.model_management.get_torch_device()
coords = samples["coords"]
vae.prepare_decode(sample_tensor.shape)
trellis_vae = vae.first_stage_model
coord_counts = samples.get("coord_counts")
model_frame = samples.get("model_frame", "y_up")
coord_resolution = samples.get("coord_resolution")
samples = samples["samples"]
samples, coords = flatten_batched_sparse_latent(samples, coords, coord_counts)
samples = samples.to(device)
feats = tex_slat_format.process_out(samples)
samples = SparseTensor(feats=feats, coords=coords.to(device))
voxel = trellis_vae.decode_tex_slat(samples.to(vae.vae_dtype), shape_subdivides)
# Keep all decoded channels. The texture VAE emits 6: base_color (0:3),
# metallic (3), roughness (4), alpha (5) — all in [0, 1]. Vertex-color
# consumers (PaintMesh) slice [:3]
color_feats = voxel.feats
voxel_coords = voxel.coords
if coord_resolution is not None:
tex_resolution = int(coord_resolution) * 16
elif voxel_coords.numel() > 0 and voxel_coords.shape[-1] >= 3:
spatial = voxel_coords[:, -3:] if voxel_coords.shape[-1] == 4 else voxel_coords
max_idx = int(spatial.max().item()) + 1
tex_resolution = next((r for r in (256, 512, 1024, 1536, 2048) if r >= max_idx), max_idx)
else:
tex_resolution = 1024
# Remap Z-up voxel coords to Y-up: (x, y, z) -> (x, z, R-1-y), matching the
# R_x(-90°) applied to mesh vertices in VaeDecodeShapeTrellis. Keeps PaintMesh's
# NN lookup correctly aligned without it needing to know the source frame.
if model_frame == "z_up" and voxel_coords.numel() > 0 and voxel_coords.shape[-1] >= 3:
R = tex_resolution
if voxel_coords.shape[-1] == 4:
batch_col = voxel_coords[:, :1]
spatial = voxel_coords[:, 1:]
spatial_yup = torch.stack(
[spatial[:, 0], spatial[:, 2], (R - 1) - spatial[:, 1]], dim=-1
)
voxel_coords = torch.cat([batch_col, spatial_yup], dim=-1)
else:
voxel_coords = torch.stack(
[voxel_coords[:, 0], voxel_coords[:, 2], (R - 1) - voxel_coords[:, 1]],
dim=-1,
)
voxel = Types.VOXEL(voxel_coords, color_feats, tex_resolution)
return IO.NodeOutput(voxel)
class VaeDecodeStructureTrellis2(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="VaeDecodeStructureTrellis2",
category="latent/3d",
inputs=[
IO.Latent.Input("samples"),
IO.Vae.Input("vae"),
IO.Combo.Input("resolution", options=["32", "64"], default="32"),
],
outputs=[
IO.Voxel.Output("voxel"),
]
)
@classmethod
def execute(cls, samples, vae, resolution):
resolution = int(resolution)
sample_tensor = samples["samples"]
sample_tensor = sample_tensor[:, :8]
batch_number = vae.prepare_decode(sample_tensor.shape)
shape_vae = vae.first_stage_model
load_device = comfy.model_management.get_torch_device()
decoded_batches = []
for start in range(0, sample_tensor.shape[0], batch_number):
sample_chunk = sample_tensor[start:start + batch_number].to(load_device)
decoded_batches.append(shape_vae.decode_structure(sample_chunk.to(vae.vae_dtype)) > 0)
decoded = torch.cat(decoded_batches, dim=0)
current_res = decoded.shape[2]
if current_res != resolution:
ratio = current_res // resolution
decoded = torch.nn.functional.max_pool3d(decoded.float(), ratio, ratio, 0) > 0.5
voxel_data = decoded.squeeze(1).float()
return IO.NodeOutput(Types.VOXEL(voxel_data))
class Trellis2UpsampleStage(IO.ComfyNode):
"""Cascade-upsamples a 512-resolution shape latent into high-resolution
sparse coords and sets up the second shape-stage sampling pass at the
target resolution, attaching per-stage metadata to the conditioning for
the model to consume via extra_conds. target_resolution is reduced in
128-step decrements until the unique upsampled coord count fits under
max_tokens (floor 1024)."""
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="Trellis2UpsampleStage",
category="model/conditioning/trellis2",
display_name="Trellis2 Upsample Stage",
inputs=[
IO.Conditioning.Input("positive"),
IO.Conditioning.Input("negative"),
IO.Latent.Input("shape_latent", tooltip="The 512-resolution shape latent output from the first shape-stage KSampler."),
IO.Vae.Input("vae"),
IO.Combo.Input("target_resolution", options=["1024", "1536"], default="1024", tooltip="Controls output detail level for upsampling."),
IO.Int.Input("max_tokens", default=49152, min=1024, max=100000,
tooltip=(
"Maximum number of output elements (coordinates) allowed after upsampling. "
"Used to limit memory usage and control mesh density."
)),
],
outputs=[
IO.Conditioning.Output(display_name="positive"),
IO.Conditioning.Output(display_name="negative"),
IO.Latent.Output(),
]
)
@staticmethod
def _quantize_unique(hr_coords: torch.Tensor, lr_resolution: int, hr_resolution: int, pixal3d_mode: bool = False) -> torch.Tensor:
# Trellis2 uses `floor((c+0.5) * grid_res / lr_res)
# Pixal3D uses `round((c+0.5) * (grid_res-1) / lr_res)`
# this is a half-cell spatial shift. Branch so each upstream is matched bit-for-bit.
grid_res = hr_resolution // 16
spatial = hr_coords[:, 1:].float()
if pixal3d_mode:
spatial.add_(0.5).mul_((grid_res - 1) / lr_resolution).round_()
else:
spatial.add_(0.5).mul_(grid_res / lr_resolution)
quant = torch.cat([hr_coords[:, :1], spatial.int()], dim=1)
return quant.unique(dim=0)
@classmethod
def execute(cls, positive, negative, shape_latent, vae, target_resolution, max_tokens):
device = comfy.model_management.get_torch_device()
vae.prepare_decode(shape_latent["samples"].shape)
coord_counts = shape_latent.get("coord_counts")
shape_vae = vae.first_stage_model
lr_resolution = 512
target_resolution = int(target_resolution)
proj_pack = _proj_pack_from_conditioning(positive)
pixal3d_mode = proj_pack is not None
# Decode each sample's HR coords, then search for the largest hr_resolution
# that fits under max_tokens across all samples.
if coord_counts is None:
feats, coords_512 = flatten_batched_sparse_latent(
shape_latent["samples"], shape_latent["coords"], coord_counts,
)
slat = shape_norm(feats.to(device), coords_512.to(device))
sample_hr_coords = [shape_vae.upsample_shape(slat.to(vae.vae_dtype), upsample_times=4)]
else:
items = split_batched_sparse_latent(
shape_latent["samples"], shape_latent["coords"], coord_counts,
)
sample_hr_coords = []
for feats_i, coords_i in items:
coords_i = coords_i.to(device).clone()
coords_i[:, 0] = 0
slat_i = shape_norm(feats_i.to(device), coords_i)
sample_hr_coords.append(shape_vae.upsample_shape(slat_i.to(vae.vae_dtype), upsample_times=4))
# Resolution search — cache the final iteration's quantized unique tensors
# so we don't recompute .unique() per sample after picking hr_resolution.
hr_resolution = target_resolution
quant_unique_list = []
while True:
quant_unique_list = []
exceeds_limit = False
for hr_coords_i in sample_hr_coords:
qu = cls._quantize_unique(hr_coords_i, lr_resolution, hr_resolution, pixal3d_mode)
quant_unique_list.append(qu)
if qu.shape[0] >= max_tokens:
exceeds_limit = True
break
if not exceeds_limit:
break
if hr_resolution <= 1024:
for k in range(len(quant_unique_list), len(sample_hr_coords)):
quant_unique_list.append(
cls._quantize_unique(sample_hr_coords[k], lr_resolution, hr_resolution, pixal3d_mode)
)
break
hr_resolution -= 128
# Rewrite batch column to match per-sample offset and concat.
per_sample_counts = []
for sample_offset, qu in enumerate(quant_unique_list):
qu[:, 0] = sample_offset
per_sample_counts.append(int(qu.shape[0]))
coords = torch.cat(quant_unique_list, dim=0)
counts = torch.tensor(per_sample_counts, dtype=torch.int64)
coord_resolution = hr_resolution // 16
batch_size, _, max_tokens_out = infer_batched_coord_layout(coords)
latent = torch.zeros(batch_size, 32, max_tokens_out, 1)
extras = {
"trellis2_generation_mode": "shape_generation",
"trellis2_coords": coords,
"trellis2_coord_counts": counts,
}
if proj_pack is not None:
extras["trellis2_proj_feats"] = compute_stage_proj_feats(
proj_pack, "shape_1024", coords=coords, coord_resolution=coord_resolution,
)
positive_out = _conditioning_set_extras(positive, extras)
negative_out = _conditioning_set_extras(negative, extras)
out_latent = {"samples": latent, "coords": coords, "coord_counts": counts,
"coord_resolution": coord_resolution, "type": "trellis2",
"model_frame": shape_latent.get("model_frame",
"y_up" if proj_pack is not None else "z_up")}
return IO.NodeOutput(positive_out, negative_out, out_latent)
def _dinov3_encode(model, image_bchw, image_size, want_patches=False):
"""Run DINOv3 once at the requested resolution.
image_bchw: [B, 3, H, W] float in [0, 1] (any source resolution; resized here).
Returns the full sequence tensor (Trellis2 path) or a dict with the global
tokens split out + a 2D patch grid (Pixal3D path) when `want_patches=True`.
"""
model_internal = model.model
device = comfy.model_management.get_torch_device()
img_t = comfy.utils.common_upscale(image_bchw, image_size, image_size, "lanczos", "disabled").to(device)
mean = torch.tensor(model.image_mean or [0.485, 0.456, 0.406], device=device).view(1, 3, 1, 1)
std = torch.tensor(model.image_std or [0.229, 0.224, 0.225], device=device).view(1, 3, 1, 1)
img_t = (img_t - mean) / std
tokens = model_internal(img_t, skip_norm_elementwise=True)[0]
if not want_patches:
return tokens
h_p = w_p = image_size // 16
n_reg = tokens.shape[1] - 1 - h_p * w_p
return {"tokens": tokens[:, :1 + n_reg], "patches_2d": _dinov3_patches_to_2d(tokens, image_size)}
class Trellis2Conditioning(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="Trellis2Conditioning",
category="model/conditioning/trellis2",
inputs=[
IO.ClipVision.Input("clip_vision_model"),
IO.Image.Input("image", tooltip="Preprocessed image from ImageCropToMask (pad_factor=1.0 for TRELLIS.2)."),
],
outputs=[
IO.Conditioning.Output(display_name="positive"),
IO.Conditioning.Output(display_name="negative"),
]
)
@classmethod
def execute(cls, clip_vision_model, image) -> IO.NodeOutput:
out_device = comfy.model_management.intermediate_device()
cond = _dino_encode_batch(clip_vision_model, image, out_device)
cond_512_batched, cond_1024_batched = cond["global_512"], cond["global_1024"]
neg_cond_batched = torch.zeros_like(cond_512_batched)
neg_embeds_batched = torch.zeros_like(cond_1024_batched)
positive = [[cond_512_batched, {"embeds": cond_1024_batched}]]
negative = [[neg_cond_batched, {"embeds": neg_embeds_batched}]]
return IO.NodeOutput(positive, negative)
def _proj_pack_from_conditioning(conditioning):
"""Return the proj_feat_pack dict embedded in a Pixal3D conditioning (or None
for vanilla Trellis2 / no conditioning connected). Pixal3DConditioning ships
the pack in cond[0][1]["proj_feat_pack"]; Trellis2Conditioning doesn't set it."""
if not conditioning:
return None
entry = conditioning[0]
if not isinstance(entry, (list, tuple)) or len(entry) < 2 or not isinstance(entry[1], dict):
return None
return entry[1].get("proj_feat_pack")
def _conditioning_set_extras(conditioning, extras: dict):
"""Return a copy of `conditioning` with `extras` merged into each entry's
dict same shallow-copy pattern ControlNetApplyAdvanced uses. The dicts
are copied so we don't mutate upstream conditioning."""
out = []
for entry in conditioning:
if isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict):
new_dict = entry[1].copy()
new_dict.update(extras)
out.append([entry[0], new_dict])
else:
out.append(entry)
return out
class Trellis2ShapeStage(IO.ComfyNode):
"""Sets up the first shape-stage sampling pass: extracts sparse coords from
the dense structure voxel produced by VaeDecodeStructureTrellis2, builds an
empty sparse latent, and attaches per-stage metadata to the conditioning so
the model reads it via extra_conds at sample time. For the second shape pass
(post-upsample), use Trellis2UpsampleStage instead it combines the cascade
and the second-pass stage setup."""
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="Trellis2ShapeStage",
category="model/conditioning/trellis2",
inputs=[
IO.Conditioning.Input("positive"),
IO.Conditioning.Input("negative"),
IO.Voxel.Input(
"voxel",
tooltip="Dense structure voxel from VaeDecodeStructureTrellis2.",
),
],
outputs=[
IO.Conditioning.Output(display_name="positive"),
IO.Conditioning.Output(display_name="negative"),
IO.Latent.Output(),
]
)
@classmethod
def execute(cls, positive, negative, voxel):
decoded = voxel.data.unsqueeze(1)
coords = torch.argwhere(decoded.bool())[:, [0, 2, 3, 4]].int()
coord_resolution = int(decoded.shape[-1])
# Dispatch based on the upstream voxel resolution, mirroring upstream's
# pipeline_type → ss_res table:
# coord_res == 32 → first cascade shape pass OR pure-512 pipeline
# (img2shape_512 + shape_512 proj stage, 512 DINO).
# coord_res > 32 → pure-1024 non-cascade pipeline
# (img2shape + shape_1024 proj stage, 1024 DINO).
if coord_resolution <= 32:
mode = "shape_generation_512"
stage = "shape_512"
else:
mode = "shape_generation"
stage = "shape_1024"
batch_size, counts, max_tokens = infer_batched_coord_layout(coords)
latent = torch.zeros(batch_size, 32, max_tokens, 1)
extras = {
"trellis2_generation_mode": mode,
"trellis2_coords": coords,
"trellis2_coord_counts": counts,
}
proj_pack = _proj_pack_from_conditioning(positive)
if proj_pack is not None:
extras["trellis2_proj_feats"] = compute_stage_proj_feats(
proj_pack, stage, coords=coords, coord_resolution=coord_resolution,
)
positive_out = _conditioning_set_extras(positive, extras)
negative_out = _conditioning_set_extras(negative, extras)
out_latent = {"samples": latent, "coords": coords, "coord_counts": counts,
"coord_resolution": coord_resolution, "type": "trellis2",
"model_frame": "y_up" if proj_pack is not None else "z_up"}
return IO.NodeOutput(positive_out, negative_out, out_latent)
class Trellis2TextureStage(IO.ComfyNode):
"""Sets up the texture-stage sampling pass. Reads coords / coord_counts /
coord_resolution and the shape_slat (the per-voxel shape latent) from the
incoming shape_latent dict set there by Trellis2ShapeStage or
Trellis2UpsampleStage. Builds an empty sparse latent at the same coord
layout and attaches per-stage metadata to the conditioning."""
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="Trellis2TextureStage",
category="model/conditioning/trellis2",
inputs=[
IO.Conditioning.Input("positive"),
IO.Conditioning.Input("negative"),
IO.Latent.Input("shape_latent"),
],
outputs=[
IO.Conditioning.Output(display_name="positive"),
IO.Conditioning.Output(display_name="negative"),
IO.Latent.Output(),
]
)
@classmethod
def execute(cls, positive, negative, shape_latent):
channels = 32
coords = shape_latent["coords"]
coord_resolution = shape_latent.get("coord_resolution")
batch_size, counts, max_tokens = infer_batched_coord_layout(coords)
shape_slat = shape_latent["samples"]
if shape_slat.ndim == 4:
shape_slat = shape_slat.squeeze(-1).transpose(1, 2).reshape(-1, channels)
latent = torch.zeros(batch_size, channels, max_tokens, 1)
proj_pack = _proj_pack_from_conditioning(positive)
model_frame = shape_latent.get("model_frame",
"y_up" if proj_pack is not None else "z_up")
extras = {
"trellis2_generation_mode": "texture_generation",
"trellis2_coords": coords,
"trellis2_coord_counts": counts,
"trellis2_shape_slat": shape_slat,
"trellis2_model_frame": model_frame,
}
if proj_pack is not None and coord_resolution is not None:
extras["trellis2_proj_feats"] = compute_stage_proj_feats(
proj_pack, "tex_1024", coords=coords, coord_resolution=coord_resolution,
)
positive_out = _conditioning_set_extras(positive, extras)
negative_out = _conditioning_set_extras(negative, extras)
out_latent = {"samples": latent, "type": "trellis2", "coords": coords, "coord_counts": counts,
"model_frame": shape_latent.get("model_frame",
"y_up" if proj_pack is not None else "z_up")}
if coord_resolution is not None:
out_latent["coord_resolution"] = coord_resolution
return IO.NodeOutput(positive_out, negative_out, out_latent)
class EmptyTrellis2LatentStructure(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="EmptyTrellis2LatentStructure",
category="latent/3d",
inputs=[
IO.Int.Input("batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch."),
],
outputs=[
IO.Latent.Output(),
]
)
@classmethod
def execute(cls, batch_size):
in_channels = 32
resolution = 16
latent = torch.zeros(batch_size, in_channels, resolution, resolution, resolution)
return IO.NodeOutput({"samples": latent, "type": "trellis2"})
def _dinov3_patches_to_2d(tokens, image_size, patch_size=16):
h_p = w_p = image_size // patch_size
n_patches = h_p * w_p
n_reg = tokens.shape[1] - 1 - n_patches
if n_reg < 0 or tokens.shape[1] != 1 + n_reg + n_patches:
raise ValueError(
f"_dinov3_patches_to_2d: got {tokens.shape[1]} tokens, expected "
f"1 (CLS) + N_reg + {h_p}*{w_p}={n_patches} patches at image_size={image_size}, "
f"patch_size={patch_size}. Inferred N_reg={n_reg} which is invalid."
)
start = 1 + n_reg
patches = tokens[:, start:start + n_patches]
return patches.transpose(1, 2).reshape(tokens.shape[0], -1, h_p, w_p).contiguous()
def _crop_image_with_mask(item_image, item_mask, max_image_size=1024, pad_factor=1.1,
mask_offset=0, mask_threshold=0.05, bg_rgb=(0.0, 0.0, 0.0),
aspect_ratio=1.0):
img = item_image.permute(2, 0, 1).unsqueeze(0).cpu().float().clamp(0, 1)
mask = item_mask.unsqueeze(0).unsqueeze(0).cpu().float().clamp(0, 1)
# Detect and correct an inverted mask, only when border and center have opposite polarity.
m2d = mask[0, 0]
h, w = m2d.shape
border = torch.cat([m2d[0, :], m2d[-1, :], m2d[:, 0], m2d[:, -1]])
center = m2d[h // 4:h - h // 4, w // 4:w - w // 4]
if float(border.mean()) > 0.5 and float(center.mean()) < 0.5:
mask = 1.0 - mask
if mask_offset > 0:
r = mask_offset
mask = torch.nn.functional.max_pool2d(mask, kernel_size=2 * r + 1, stride=1, padding=r)
elif mask_offset < 0:
r = -mask_offset
mask = 1.0 - torch.nn.functional.max_pool2d(1.0 - mask, kernel_size=2 * r + 1, stride=1, padding=r)
if mask_threshold > 0.0:
mask = torch.where(mask < mask_threshold, torch.zeros_like(mask), mask)
H, W = img.shape[-2:]
if max(H, W) > max_image_size:
scale = max_image_size / max(H, W)
new_w, new_h = int(W * scale), int(H * scale)
img = comfy.utils.common_upscale(img, new_w, new_h, "lanczos", "disabled")
mask = comfy.utils.common_upscale(mask, new_w, new_h, "lanczos", "disabled")
# common_upscale's lanczos path drops the singleton channel dim for masks (utils.py:1062).
if mask.ndim == 3:
mask = mask.unsqueeze(1)
H, W = new_h, new_w
scene_size = (W, H)
alpha_u8 = (mask[0, 0].clamp(0, 1) * 255.0).to(torch.uint8)
fg_pixels = (alpha_u8 > 204).nonzero()
if fg_pixels.numel() == 0:
# Try the inverted mask — auto-invert above may have been too conservative.
inv_fg = ((255 - alpha_u8) > 204).nonzero()
if inv_fg.numel() > 0:
logging.info("Trellis2 preprocess: mask bbox empty, using inverted mask.")
mask = 1.0 - mask
fg_pixels = inv_fg
if fg_pixels.numel() > 0:
y_min, x_min = fg_pixels.min(dim=0).values.tolist()
y_max, x_max = fg_pixels.max(dim=0).values.tolist()
center_y, center_x = (y_min + y_max) / 2.0, (x_min + x_max) / 2.0
bw = x_max - x_min
bh = y_max - y_min
# Grow the bbox so its aspect matches `aspect_ratio` (width/height),
# anchored on the max side. Then apply pad_factor.
if bw / max(bh, 1) >= aspect_ratio:
crop_w = int(bw * pad_factor)
crop_h = int(bw / aspect_ratio * pad_factor)
else:
crop_h = int(bh * pad_factor)
crop_w = int(bh * aspect_ratio * pad_factor)
half_w, half_h = crop_w // 2, crop_h // 2
crop_x1 = int(center_x - half_w)
crop_y1 = int(center_y - half_h)
crop_x2 = crop_x1 + 2 * half_w
crop_y2 = crop_y1 + 2 * half_h
else:
logging.warning("Mask for the image is empty; a clean foreground mask is required for best quality.")
crop_x1, crop_y1, crop_x2, crop_y2 = 0, 0, W, H
crop_bbox = (crop_x1, crop_y1, crop_x2, crop_y2)
# Zero-pad out-of-bounds slice (PIL.crop semantics).
pad_l = max(0, -crop_x1)
pad_t = max(0, -crop_y1)
pad_r = max(0, crop_x2 - W)
pad_b = max(0, crop_y2 - H)
if pad_l or pad_t or pad_r or pad_b:
img = torch.nn.functional.pad(img, (pad_l, pad_r, pad_t, pad_b), value=0.0)
mask = torch.nn.functional.pad(mask, (pad_l, pad_r, pad_t, pad_b), value=0.0)
crop_x1 += pad_l
crop_x2 += pad_l
crop_y1 += pad_t
crop_y2 += pad_t
cropped_img = img [..., crop_y1:crop_y2, crop_x1:crop_x2]
cropped_mask = mask[..., crop_y1:crop_y2, crop_x1:crop_x2]
bg = torch.tensor(bg_rgb, dtype=cropped_img.dtype, device=cropped_img.device).view(1, 3, 1, 1)
composite = (cropped_img * cropped_mask + bg * (1.0 - cropped_mask)).clamp(0, 1)
return composite, crop_bbox, scene_size
def _dino_encode_batch(clip_vision_model, image, out_device, *, want_patches=False):
"""Encode an already-preprocessed image through DINOv3 at 512 and 1024.
Expects `image` to be a comfy IMAGE tensor [B, H, W, 3] of squared composites
(from ImageCropToMask). Returns batched global tokens; with want_patches also
the 2D patch grids and the per-item BCHW composites that the Pixal3D NAF path needs."""
image = image[..., :3]
batch_size = image.shape[0]
cond_512_list, cond_1024_list = [], []
patches_512_list, patches_1024_list = [], []
composite_list = []
for b in range(batch_size):
item = image[b].movedim(-1, -3).unsqueeze(0).contiguous().float().clamp(0, 1)
c512 = _dinov3_encode(clip_vision_model, item, 512, want_patches=want_patches)
c1024 = _dinov3_encode(clip_vision_model, item, 1024, want_patches=want_patches)
if want_patches:
cond_512_list.append(c512["tokens"].to(out_device))
cond_1024_list.append(c1024["tokens"].to(out_device))
patches_512_list.append(c512["patches_2d"].to(out_device))
patches_1024_list.append(c1024["patches_2d"].to(out_device))
composite_list.append(item)
else:
cond_512_list.append(c512.to(out_device))
cond_1024_list.append(c1024.to(out_device))
out = {
"batch_size": batch_size,
"global_512": torch.cat(cond_512_list, dim=0),
"global_1024": torch.cat(cond_1024_list, dim=0),
}
if want_patches:
out["patches_512"] = torch.cat(patches_512_list, dim=0)
out["patches_1024"] = torch.cat(patches_1024_list, dim=0)
out["composites"] = composite_list
return out
class ImageCropToMask(IO.ComfyNode):
"""Crop an image to its mask's bounding box (centered square, with pad_factor
margin), then composite `img * mask` and resize to a square. Handles OOB crops
with zero-padding. Useful for 3D pipelines that expect a centered, background-free
subject at a fixed input resolution (Trellis2, Pixal3D, Hunyuan3D, TripoSR, etc.)."""
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="ImageCropToMask",
display_name="Image Crop to Mask",
category="image/transform",
search_aliases=["crop to mask", "mask crop", "crop mask", "mask crop resize", "crop mask resize", "trellis2", "pixal3d"],
inputs=[
IO.Image.Input("image"),
IO.Mask.Input("mask"),
IO.Int.Input("width", default=1024, min=64, max=4096, step=8, tooltip="Output width in pixels."),
IO.Int.Input("height", default=1024, min=64, max=4096, step=8, tooltip="Output height in pixels."),
IO.Float.Input("pad_factor", default=1.0, min=1.0, max=2.0, step=0.01, tooltip="Extra margin around the mask bbox as a multiplier."),
IO.Int.Input("mask_offset", default=0, min=-32, max=32, step=1, tooltip="Grow or shrink the mask by this many pixels before cropping."),
IO.Color.Input("background", default="#000000", tooltip="Fill color behind the masked subject."),
],
outputs=[IO.Image.Output(display_name="image")],
)
@classmethod
def execute(cls, image, mask, width, height, pad_factor, mask_offset, background) -> IO.NodeOutput:
h = background.lstrip("#")
bg_rgb = (int(h[0:2], 16) / 255.0, int(h[2:4], 16) / 255.0, int(h[4:6], 16) / 255.0) if len(h) == 6 else (0.0, 0.0, 0.0)
image = image[..., :3]
batch_size = image.shape[0]
if mask.shape[0] == 1 and batch_size > 1:
mask = mask.expand(batch_size, -1, -1)
elif mask.shape[0] != batch_size:
raise ValueError(f"Mask batch {mask.shape[0]} does not match image batch {batch_size}")
out_images = []
for b in range(batch_size):
composite, _, _ = _crop_image_with_mask(
image[b], mask[b], max_image_size=max(width, height), pad_factor=pad_factor,
mask_offset=mask_offset, bg_rgb=bg_rgb, aspect_ratio=width / height,
)
composite = comfy.utils.common_upscale(composite, width, height, "lanczos", "disabled")
out_images.append(composite.movedim(-3, -1))
result = torch.cat(out_images, dim=0).to(
device=comfy.model_management.intermediate_device(),
dtype=comfy.model_management.intermediate_dtype(),
)
return IO.NodeOutput(result)
class Pixal3DConditioning(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
node_id="Pixal3DConditioning",
category="model/conditioning/trellis2",
inputs=[
IO.ClipVision.Input("clip_vision_model", tooltip="DINOv3 ViT-L/16 ClipVision."),
IO.Image.Input("image", tooltip="Preprocessed image from ImageCropToMask (pad_factor=1.1 for Pixal3D)."),
IO.Float.Input(
"camera_angle_x", display_name="fov",
default=49.13, min=1.0, max=170.0, step=0.01, advanced=True,
tooltip="Horizontal FOV in degrees. Wire a MoGeGeometryToFOV "
"(axis='horizontal', unit='degrees') for a per-image FoV (matches upstream default).",
),
],
outputs=[
IO.Conditioning.Output(display_name="positive"),
IO.Conditioning.Output(display_name="negative"),
],
)
@classmethod
def execute(cls, clip_vision_model, image, camera_angle_x) -> IO.NodeOutput:
naf_model = clip_vision_model.naf
out_device = comfy.model_management.intermediate_device()
compute_device = comfy.model_management.get_torch_device()
cond = _dino_encode_batch(clip_vision_model, image, out_device, want_patches=True)
batch_size = cond["batch_size"]
global_512, global_1024 = cond["global_512"], cond["global_1024"]
fm_512_dino, fm_1024_dino = cond["patches_512"], cond["patches_1024"]
composite_list = cond["composites"]
# The LR DINO grid AND the NAF HR grid are sampled separately
# NAF targets per stage: shape_512=512, shape_1024=512, tex_1024=1024.
def _naf_hr(lr_feat, composites, image_size, naf_target):
if naf_model is None or naf_target is None:
return None
comfy.model_management.load_model_gpu(naf_model)
inner = naf_model.model
model_dtype = next(inner.parameters()).dtype # set at load time (see clip_vision NAF)
hrs = []
for i, c in enumerate(composites):
img_i = comfy.utils.common_upscale(c, image_size, image_size, "lanczos", "disabled")\
.to(compute_device).to(model_dtype)
lr_i = lr_feat[i:i + 1].to(compute_device).to(model_dtype)
hr_i = inner(img_i, lr_i, naf_target, output_device=out_device)
hrs.append(hr_i)
return torch.cat(hrs, dim=0)
hr_shape_512 = _naf_hr(fm_512_dino, composite_list, 512, (512, 512))
hr_shape_1024 = _naf_hr(fm_1024_dino, composite_list, 1024, (512, 512))
hr_tex_1024 = _naf_hr(fm_1024_dino, composite_list, 1024, (1024, 1024))
# distance_from_fov: grid_point (-1, 0, 0) projects to pixel (0, image_resolution-1).
# FOV widget is in degrees for UX; trig + downstream projection expect radians.
camera_angle_x = math.radians(float(camera_angle_x))
distance = 0.5 / math.tan(camera_angle_x / 2.0)
cam_angle_t = torch.tensor([camera_angle_x] * batch_size, device=out_device, dtype=torch.float32)
dist_t = torch.tensor([distance] * batch_size, device=out_device, dtype=torch.float32)
scale_t = torch.ones(batch_size, device=out_device, dtype=torch.float32)
T = build_proj_transform_matrix(dist_t, batch_size, device=out_device, dtype=torch.float32)
proj_pack = {
"stages": {
"ss": {"feature_map": fm_512_dino, "feature_map_hr": None, "image_resolution": 512},
"shape_512": {"feature_map": fm_512_dino, "feature_map_hr": hr_shape_512, "image_resolution": 512},
"shape_1024": {"feature_map": fm_1024_dino, "feature_map_hr": hr_shape_1024,"image_resolution": 1024},
"tex_1024": {"feature_map": fm_1024_dino, "feature_map_hr": hr_tex_1024, "image_resolution": 1024},
},
"transform_matrix": T,
"camera_angle_x": cam_angle_t,
"mesh_scale": scale_t,
"distance": dist_t,
"patch_size": 16,
}
# global_512 → SS/shape_512 cross-attn; global_1024 → shape_1024/tex_1024.
ss_proj_feats = compute_stage_proj_feats(
proj_pack, "ss", dense_grid_resolution=16, batch_size=batch_size,
device=compute_device,
)
neg_global = torch.zeros_like(global_512)
neg_embeds = torch.zeros_like(global_1024)
base_extras = {
"embeds": global_1024, "proj_feat_pack": proj_pack,
"trellis2_proj_feats": ss_proj_feats,
}
neg_extras = {
"embeds": neg_embeds, "proj_feat_pack": proj_pack,
"trellis2_proj_feats": ss_proj_feats,
}
positive = [[global_512, base_extras]]
negative = [[neg_global, neg_extras]]
return IO.NodeOutput(positive, negative)
class Trellis2Extension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
return [
ImageCropToMask,
Trellis2Conditioning,
Pixal3DConditioning,
Trellis2ShapeStage,
EmptyTrellis2LatentStructure,
Trellis2TextureStage,
VaeDecodeTextureTrellis,
VaeDecodeShapeTrellis,
VaeDecodeStructureTrellis2,
Trellis2UpsampleStage,
]
async def comfy_entrypoint() -> Trellis2Extension:
return Trellis2Extension()

View File

@ -2482,6 +2482,8 @@ async def init_builtin_extra_nodes():
"nodes_toolkit.py",
"nodes_replacements.py",
"nodes_nag.py",
"nodes_trellis2.py",
"nodes_mesh_postprocess.py",
"nodes_sdpose.py",
"nodes_math.py",
"nodes_number_convert.py",