feat: add torchao INT4 weight-only quantization backend

comfy/quantization/torchao/__init__.py

@@ -0,0 +1,24 @@
+"""
+ComfyUI TorchAO INT4 weight-only quantization backend.
+
+W4A16 scheme: weights quantized to INT4 via torchao,
+activations remain FP16. Supports CUDA, Intel XPU, and CPU.
+
+Includes:
+- TINT4Linear: INT4 linear layer with LoRA forward injection + QuaRot
+- quantize_model: torchao INT4 quantization entry point
+- reconstruct_int4_state_dict: rebuild TINT4Linear from safetensors
+- build_hadamard / rotate_weight: QuaRot Hadamard rotation
+"""
+
+from .linear import TINT4Linear
+from .quantize import quantize_model, reconstruct_int4_state_dict
+from .quarot import build_hadamard, rotate_weight
+
+__all__ = [
+    "TINT4Linear",
+    "quantize_model",
+    "reconstruct_int4_state_dict",
+    "build_hadamard",
+    "rotate_weight",
+]

comfy/quantization/torchao/linear.py

@@ -0,0 +1,154 @@
+"""
+TINT4Linear — INT4 weight-only Linear with LoRA + QuaRot.
+
+Backed by torchao Int4PlainInt32Tensor. Activations stay FP16.
+LoRA entries are injected at forward time — no kernel modification needed.
+QuaRot activation rotation is applied online when enabled.
+"""
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torchao.quantization.quantize_.workflows.int4.int4_plain_int32_tensor import (
+    Int4PlainInt32Tensor,
+)
+
+
+class TINT4Linear(nn.Module):
+    """
+    INT4 weight-only linear layer.
+
+    Stores raw int32 qdata + fp16 scales + int8 zero-points.
+    Int4PlainInt32Tensor is lazily constructed and rebuilt on device change.
+
+    Slots (set externally by loaders):
+      _use_quarot: bool         — enable activation Hadamard rotation
+      _group_size: int          — QuaRot group size
+      _hadamard_H: Tensor|None  — normalized Hadamard matrix
+      _tint4_lora_entries: dict|None — {lora_name: [(type, ...), ...]}
+    """
+
+    def __init__(self, in_features: int, out_features: int,
+                 qdata: torch.Tensor, scale: torch.Tensor,
+                 zp: torch.Tensor, block_size: tuple,
+                 bias: torch.Tensor | None = None):
+        super().__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+
+        if bias is not None:
+            self.bias = nn.Parameter(bias)
+        else:
+            self.register_parameter('bias', None)
+
+        self._qdata = qdata           # int32  (out_f, in_f // 8)
+        self._scale = scale           # fp16   (n_blocks, out_f)
+        self._zp = zp                 # int8   (n_blocks, out_f)
+        self._block_size = block_size # (b0, b1)
+        self._qt: Int4PlainInt32Tensor | None = None
+
+        # QuaRot (optional)
+        self._use_quarot: bool = False
+        self._group_size: int = 128
+        self._hadamard_H: torch.Tensor | None = None
+
+        # LoRA entries slot
+        self._tint4_lora_entries: dict | None = None
+
+    @property
+    def weight(self) -> Int4PlainInt32Tensor:
+        if self._qt is None:
+            self._qt = Int4PlainInt32Tensor(
+                self._qdata, self._scale, self._zp,
+                self._block_size, [self.out_features, self.in_features],
+            )
+        return self._qt
+
+    @weight.setter
+    def weight(self, value: Int4PlainInt32Tensor) -> None:
+        if isinstance(value, Int4PlainInt32Tensor):
+            self._qt = value
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x_flat = x.reshape(-1, x.shape[-1])
+
+        # ── QuaRot activation rotation (online) ──────────────────
+        if self._use_quarot and self._hadamard_H is not None:
+            try:
+                H_dev = self._hadamard_H.to(
+                    device=x.device,
+                    dtype=(x.dtype if x.dtype in (
+                        torch.float16, torch.bfloat16)
+                           else torch.float16),
+                )
+                n_groups = x_flat.shape[-1] // self._group_size
+                x_g = x_flat.view(-1, n_groups, self._group_size)
+                x_flat = torch.matmul(x_g, H_dev).view(-1, x_g.shape[2] * n_groups)
+            except Exception:
+                pass
+
+        dev = x.device
+
+        # ── Rebuild Int4PlainInt32Tensor on device change ────────
+        if self._qt is None or self._qt.device != dev:
+            self._qt = Int4PlainInt32Tensor(
+                self._qdata.to(dev), self._scale.to(dev), self._zp.to(dev),
+                self._block_size, [self.out_features, self.in_features],
+            )
+
+        out = F.linear(x_flat, self._qt, None)
+
+        # ── LoRA forward injection ───────────────────────────────
+        entries = self._tint4_lora_entries
+        if entries is not None and entries:
+            cd = (x.dtype if x.dtype in (torch.float16, torch.bfloat16)
+                  else torch.float16)
+            for lora_entries in entries.values():
+                for e in lora_entries:
+                    if isinstance(e[0], str) and e[0] == "lokr":
+                        _, w1, w2, mult, factor = e[:5]
+                        sl = e[5] if len(e) > 5 else None
+                        se = e[6] if len(e) > 6 else None
+                        w1d = w1.to(device=dev, dtype=cd)
+                        w2d = w2.to(device=dev, dtype=cd)
+                        of2, if2 = w2d.shape
+                        w1x = w1d.repeat_interleave(
+                            of2 // factor, dim=0).repeat_interleave(
+                            if2 // factor, dim=1)
+                        dw = (w1x * w2d).mul_(mult)
+                        lo = x_flat.to(cd) @ dw.T
+                        if sl is not None:
+                            if lo.shape[1] == (se - sl):
+                                out[:, sl:se] += lo
+                        elif lo.shape == out.shape:
+                            out += lo
+                        continue
+
+                    # standard LoRA
+                    A, B, mult = e[:3]
+                    sl = e[3] if len(e) > 3 else None
+                    se = e[4] if len(e) > 4 else None
+                    Ad = A.to(device=dev, dtype=cd)
+                    Bd = B.to(device=dev, dtype=cd)
+                    lo = (x_flat.to(cd) @ Ad.T) @ Bd.T * mult
+                    if sl is not None:
+                        if lo.shape[1] == (se - sl):
+                            out[:, sl:se] += lo
+                    elif lo.shape[1] == out.shape[1]:
+                        out += lo
+
+        if self.bias is not None:
+            out += self.bias.to(device=dev, dtype=out.dtype)
+
+        return out.reshape(*x.shape[:-1], out.shape[-1])
+
+    def release(self) -> None:
+        """Drop cached Int4PlainInt32Tensor to free VRAM."""
+        self._qt = None
+
+    def __del__(self):
+        self._qdata = None
+        self._scale = None
+        self._zp = None
+        self._qt = None
+        self._tint4_lora_entries = None

comfy/quantization/torchao/quantize.py

@@ -0,0 +1,111 @@
+"""
+TorchAO INT4 quantization helpers for ComfyUI.
+
+- quantize_model: in-place INT4 quantization via torchao
+- reconstruct_int4_state_dict: rebuild TINT4Linear layers from safetensors
+"""
+
+import torch
+import torch.nn as nn
+from torchao.quantization import Int4WeightOnlyConfig, quantize_
+
+from .linear import TINT4Linear
+
+
+def quantize_model(
+    model: nn.Module,
+    group_size: int = 128,
+    filter_fn=None,
+) -> nn.Module:
+    """
+    Quantize all nn.Linear layers in-place via torchao INT4 weight-only.
+
+    Args:
+        model:      PyTorch model to quantize.
+        group_size: Quantization group size (32/64/128/256).
+        filter_fn:  Optional callable(module, name) → bool.
+                    Return True to skip that module.
+
+    Returns:
+        Same model (quantized in-place).
+    """
+    config = Int4WeightOnlyConfig(
+        group_size=group_size,
+        int4_packing_format="plain_int32",
+    )
+    quantize_(model, config, filter_fn=filter_fn)
+    return model
+
+
+# torchao plain_int32 safetensors suffixes
+_QUANT_SUFFIXES = (
+    ".weight_scale", ".weight_zp",
+    ".weight_b0", ".weight_b1",
+    ".weight_sh0", ".weight_sh1",
+    ".comfy_quant",
+)
+
+
+def _is_quantized_weight(key: str, sd: dict) -> bool:
+    """Check whether a safetensors key is a torchao-quantized weight."""
+    if not key.endswith(".weight"):
+        return False
+    base = key[:-len(".weight")]
+    return f"{base}.weight_scale" in sd and f"{base}.weight_zp" in sd
+
+
+def reconstruct_int4_state_dict(
+    sd: dict,
+    device: torch.device = torch.device("cpu"),
+) -> dict[str, TINT4Linear]:
+    """
+    Extract TINT4Linear layers from a torchao-quantized safetensors dict.
+
+    Pops quantized weight tensors from sd and replaces them with
+    fully-constructed TINT4Linear modules keyed by base layer name.
+
+    Caller is responsible for placing the modules into the target model.
+
+    Args:
+        sd:     Safetensors state dict with torchao int4 packed weights.
+        device: Target device for the reconstructed layers.
+
+    Returns:
+        Dict mapping base key (e.g. "blocks.0.attn.qkv") → TINT4Linear.
+    """
+    replaced: dict[str, TINT4Linear] = {}
+
+    for key in list(sd.keys()):
+        if not _is_quantized_weight(key, sd):
+            continue
+
+        base = key[:-len(".weight")]
+        qdata = sd.pop(key)
+        scale = sd.pop(f"{base}.weight_scale")
+        zp = sd.pop(f"{base}.weight_zp")
+        b0 = sd.pop(f"{base}.weight_b0")
+        b1 = sd.pop(f"{base}.weight_b1")
+
+        for suffix in _QUANT_SUFFIXES:
+            sd.pop(f"{base}{suffix}", None)
+
+        in_f = qdata.shape[1] * 8   # int32 packs 8 int4 values
+        out_f = qdata.shape[0]
+
+        layer = TINT4Linear(
+            in_features=in_f,
+            out_features=out_f,
+            qdata=qdata.to(torch.int32),
+            scale=scale.to(torch.float16),
+            zp=zp.to(torch.int8),
+            block_size=(b0.item(), b1.item()),
+        )
+
+        if device.type != "cpu":
+            layer._qdata = layer._qdata.to(device)
+            layer._scale = layer._scale.to(device)
+            layer._zp = layer._zp.to(device)
+
+        replaced[base] = layer
+
+    return replaced

comfy/quantization/torchao/quarot.py

@@ -0,0 +1,80 @@
+"""
+QuaRot: Group-wise Hadamard rotation for INT quantization quality.
+
+Spreads activation outliers across channels via orthogonal Hadamard matrices.
+Based on QuaRot (2024). Pure PyTorch — scipy optional for large matrices.
+"""
+
+import torch
+
+try:
+    from scipy.linalg import hadamard as _scipy_hadamard
+    _SCIPY_AVAILABLE = True
+except ImportError:
+    _SCIPY_AVAILABLE = False
+
+_HADAMARD_CACHE: dict = {}
+
+
+def build_hadamard(
+    size: int,
+    device: str = "cpu",
+    dtype: torch.dtype = torch.float32,
+) -> torch.Tensor:
+    """
+    Build normalized orthogonal Hadamard matrix (size must be power of 2).
+    Returns H such that H @ H^T = I.
+    Uses scipy if available, otherwise pure-PyTorch Sylvester construction.
+    """
+    if size & (size - 1) != 0:
+        raise ValueError(f"Hadamard size must be a power of 2, got {size}")
+
+    cache_key = (size, str(device), dtype)
+    if cache_key in _HADAMARD_CACHE:
+        return _HADAMARD_CACHE[cache_key]
+
+    if _SCIPY_AVAILABLE:
+        import numpy as np
+        H_np = _scipy_hadamard(size)
+        H = torch.tensor(H_np, dtype=dtype, device=device) / (size ** 0.5)
+    else:
+        H = torch.tensor([[1.0]], dtype=dtype, device=device)
+        n = 1
+        while n < size:
+            H = torch.cat([torch.cat([H,  H], dim=1),
+                           torch.cat([H, -H], dim=0)], dim=0)
+            n *= 2
+        H = H / (size ** 0.5)
+
+    _HADAMARD_CACHE[cache_key] = H
+    return H
+
+
+def rotate_weight(
+    weight: torch.Tensor,
+    H: torch.Tensor,
+    group_size: int,
+) -> torch.Tensor:
+    """
+    Rotate weight matrix offline: W_rot = W @ H_block^T.
+
+    For Linear(in, out) with weight (out_f, in_f):
+    each row is split into groups of group_size and rotated by H^T.
+
+    Args:
+        weight:     (out_features, in_features)
+        H:          Normalized Hadamard matrix (group_size, group_size)
+        group_size: Group size for block-diagonal rotation
+
+    Returns:
+        Rotated weight, same shape.
+    """
+    out_f, in_f = weight.shape
+    if in_f % group_size != 0:
+        raise ValueError(
+            f"in_features {in_f} not divisible by group_size {group_size}")
+    n_groups = in_f // group_size
+    W_grouped = weight.view(out_f, n_groups, group_size)
+    H_t = H.T.to(dtype=weight.dtype, device=weight.device)
+    W_rot = torch.matmul(W_grouped, H_t)
+    return W_rot.reshape(out_f, in_f)