ComfyUI/comfy/text_encoders/anima_cache.py

import contextvars
import logging
import numbers
import weakref

import torch


_cache_scope = contextvars.ContextVar("anima_prefix_cache", default=None)


def begin_cache_scope(enabled=True):
    cache = weakref.WeakKeyDictionary() if enabled else None
    return cache, _cache_scope.set(cache)


def end_cache_scope(scope):
    cache, token = scope
    try:
        if cache is not None:
            cache.clear()
    finally:
        _cache_scope.reset(token)


def set_owner(model, owner):
    object.__setattr__(model, "_anima_cache_owner", weakref.ref(owner))


def get_owner(model):
    owner = getattr(model, "_anima_cache_owner", None)
    return owner() if owner is not None else None


def _token_ids(tokens):
    if len(tokens) != 1:
        return None
    sequence = tokens[0]
    if torch.is_tensor(sequence):
        sequence = sequence.tolist()
    token_ids = []
    for token in sequence:
        if isinstance(token, numbers.Integral):
            token_ids.append(int(token))
        elif isinstance(token, (tuple, list)) and len(token) == 2 and isinstance(token[0], numbers.Integral) and isinstance(token[1], numbers.Real) and token[1] == 1:
            token_ids.append(int(token[0]))
        else:
            return None
    return tuple(token_ids)


def _common_prefix(first, second):
    length = min(len(first), len(second))
    for index in range(length):
        if first[index] != second[index]:
            return index
    return length


def _copy_key_values(key_values, length):
    return [(key[:, :, :length].clone(), value[:, :, :length].clone(), length) for key, value, _ in key_values]


def forward(transformer, cache_owner, tokens, attention_mask, embeds, num_tokens, intermediate_output, final_layer_norm_intermediate, dtype, embeds_info):
    prefix_cache = _cache_scope.get()
    token_ids = _token_ids(tokens)
    weight_uuid = getattr(cache_owner, "current_weight_patches_uuid", None)
    if prefix_cache is None or cache_owner is None or weight_uuid is None or token_ids is None or embeds_info or intermediate_output is not None or attention_mask is not None and not bool(torch.all(attention_mask)):
        return transformer(None, attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, embeds_info=embeds_info)

    cached = prefix_cache.get(cache_owner)
    if cached is not None and (cached[0]() is not transformer or cached[1] != weight_uuid or cached[3].device != embeds.device or cached[3].dtype != embeds.dtype):
        cached = None
    common = _common_prefix(cached[2], token_ids) if cached is not None else 0
    if common > 0:
        logging.debug("Anima Qwen cache reused %d prefix tokens", common)
    if cached is not None and common == len(token_ids) == len(cached[2]):
        return cached[3].clone(), None

    if cached is not None and common == len(token_ids):
        hidden = cached[3][:, :common].clone()
        prefix_cache[cache_owner] = (weakref.ref(transformer), weight_uuid, token_ids, hidden.clone(), _copy_key_values(cached[4], common))
        return hidden, None

    past_key_values = []
    prefix_hidden = None
    if cached is not None and common > 0:
        prefix_hidden = cached[3][:, :common].clone()
        past_key_values = _copy_key_values(cached[4], common)

    suffix_embeds = embeds[:, common:]
    if common > 0 and suffix_embeds.shape[1] > 1:
        suffix_outputs = []
        next_key_values = past_key_values
        for index in range(suffix_embeds.shape[1]):
            output = transformer(
                None,
                None,
                embeds=suffix_embeds[:, index:index + 1],
                num_tokens=[1],
                intermediate_output=intermediate_output,
                final_layer_norm_intermediate=final_layer_norm_intermediate,
                dtype=dtype,
                embeds_info=embeds_info,
                past_key_values=next_key_values,
            )
            suffix_outputs.append(output[0])
            next_key_values = output[2]
        suffix_hidden = torch.cat(suffix_outputs, dim=1)
    else:
        output = transformer(
            None,
            attention_mask,
            embeds=suffix_embeds,
            num_tokens=[1] if common > 0 else num_tokens,
            intermediate_output=intermediate_output,
            final_layer_norm_intermediate=final_layer_norm_intermediate,
            dtype=dtype,
            embeds_info=embeds_info,
            past_key_values=past_key_values,
        )
        suffix_hidden = output[0]
        next_key_values = output[2]
    hidden = suffix_hidden if prefix_hidden is None else torch.cat((prefix_hidden, suffix_hidden), dim=1)
    prefix_cache[cache_owner] = (weakref.ref(transformer), weight_uuid, token_ids, hidden.clone(), _copy_key_values(next_key_values, len(token_ids)))
    return hidden, None