mirror of
https://github.com/comfyanonymous/ComfyUI.git
synced 2026-07-18 20:38:15 +08:00
300 lines
11 KiB
Python
300 lines
11 KiB
Python
import asyncio
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from comfy.text_encoders import anima_cache, llama
|
|
|
|
|
|
def tokens(*ids):
|
|
return [list(ids)]
|
|
|
|
|
|
class TinyCausalTransformer:
|
|
def __init__(self):
|
|
self.calls = []
|
|
self.num_tokens = []
|
|
|
|
def __call__(self, _, attention_mask, embeds, num_tokens, intermediate_output, final_layer_norm_intermediate, dtype, embeds_info, past_key_values=None):
|
|
self.calls.append((embeds.shape[1], past_key_values is not None))
|
|
self.num_tokens.append(tuple(num_tokens))
|
|
prefix = 0
|
|
if past_key_values:
|
|
prefix = past_key_values[0][2]
|
|
previous = past_key_values[0][0][:, :, :prefix].reshape(embeds.shape[0], prefix, 1)
|
|
else:
|
|
previous = embeds[:, :0]
|
|
sequence = torch.cat((previous, embeds), dim=1)
|
|
hidden = sequence.cumsum(dim=1)[:, prefix:]
|
|
key = sequence.reshape(sequence.shape[0], 1, sequence.shape[1], 1).clone()
|
|
value = (sequence * 2).reshape(sequence.shape[0], 1, sequence.shape[1], 1).clone()
|
|
if past_key_values is None:
|
|
return hidden, None
|
|
return hidden, None, [(key, value, sequence.shape[1])]
|
|
|
|
|
|
class CacheOwner:
|
|
def __init__(self, weight_uuid="weights-a"):
|
|
self.current_weight_patches_uuid = weight_uuid
|
|
|
|
|
|
def cached_forward(transformer, token_ids, dtype=torch.float32, attention_mask=None, intermediate_output=None, embeds_info=None, owner=None):
|
|
embeds = torch.tensor(token_ids, dtype=dtype).reshape(1, -1, 1)
|
|
owner = owner or CacheOwner()
|
|
return anima_cache.forward(transformer, owner, tokens(*token_ids), attention_mask, embeds, [len(token_ids)], intermediate_output, False, torch.float32, embeds_info or [])
|
|
|
|
|
|
@pytest.fixture
|
|
def prefix_cache():
|
|
scope = anima_cache.begin_cache_scope()
|
|
try:
|
|
yield scope[0]
|
|
finally:
|
|
anima_cache.end_cache_scope(scope)
|
|
|
|
|
|
def test_token_ids_accept_runtime_lists_tensors_and_unit_weight_legacy_pairs():
|
|
assert anima_cache._token_ids([[1, 2, 3]]) == (1, 2, 3)
|
|
assert anima_cache._token_ids(torch.tensor([[1, 2, 3]])) == (1, 2, 3)
|
|
assert anima_cache._token_ids([[(1, 1.0), (2, 1)]]) == (1, 2)
|
|
assert anima_cache._token_ids([[(1, 0.5), (2, 1.0)]]) is None
|
|
assert anima_cache._token_ids([[(1, 1.0, "custom")]]) is None
|
|
assert anima_cache._token_ids([[{"type": "embedding"}]]) is None
|
|
|
|
|
|
def test_non_unit_legacy_weight_bypasses_cache(prefix_cache):
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
embeds = torch.tensor([1.0, 2.0]).reshape(1, -1, 1)
|
|
weighted_tokens = [[(1, 0.5), (2, 1.0)]]
|
|
|
|
anima_cache.forward(transformer, owner, weighted_tokens, None, embeds, [2], None, False, torch.float32, [])
|
|
anima_cache.forward(transformer, owner, weighted_tokens, None, embeds, [2], None, False, torch.float32, [])
|
|
|
|
assert transformer.calls == [(2, False), (2, False)]
|
|
assert len(prefix_cache) == 0
|
|
|
|
|
|
def test_reuses_prefix_for_extension_exactly(caplog, prefix_cache):
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
cached_forward(transformer, [1, 2, 3], owner=owner)
|
|
with caplog.at_level("DEBUG"):
|
|
output, _ = cached_forward(transformer, [1, 2, 3, 4], owner=owner)
|
|
|
|
assert torch.equal(output, torch.tensor([[[1.0], [3.0], [6.0], [10.0]]]))
|
|
assert transformer.calls == [(3, True), (1, True)]
|
|
assert transformer.num_tokens == [(3,), (1,)]
|
|
assert "Anima Qwen cache reused 3 prefix tokens" in caplog.messages
|
|
|
|
|
|
def test_exact_hit_does_not_call_transformer(prefix_cache):
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
expected, _ = cached_forward(transformer, [1, 2, 3], owner=owner)
|
|
call_count = len(transformer.calls)
|
|
|
|
output, _ = cached_forward(transformer, [1, 2, 3], owner=owner)
|
|
|
|
assert torch.equal(output, expected)
|
|
assert len(transformer.calls) == call_count
|
|
|
|
|
|
def test_strict_prefix_of_cached_prompt_is_copied_without_transformer_call(prefix_cache):
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
cached_forward(transformer, [1, 2, 3, 4], owner=owner)
|
|
output, _ = cached_forward(transformer, [1, 2], owner=owner)
|
|
|
|
assert torch.equal(output, torch.tensor([[[1.0], [3.0]]]))
|
|
assert transformer.calls == [(4, True)]
|
|
cached = prefix_cache[owner]
|
|
assert cached[2] == (1, 2)
|
|
assert cached[3]._base is None
|
|
assert all(key.shape[2] == value.shape[2] == index == 2 for key, value, index in cached[4])
|
|
assert all(key._base is None and value._base is None for key, value, _ in cached[4])
|
|
|
|
|
|
def test_diverging_suffix_matches_full_causal_forward(prefix_cache):
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
cached_forward(transformer, [1, 2, 9], owner=owner)
|
|
output, _ = cached_forward(transformer, [1, 2, 4, 5], owner=owner)
|
|
|
|
assert torch.equal(output, torch.tensor([[[1.0], [3.0], [7.0], [12.0]]]))
|
|
assert transformer.calls == [(3, True), (2, True)]
|
|
assert transformer.num_tokens == [(3,), (2,)]
|
|
cached = prefix_cache[owner]
|
|
assert cached[2] == (1, 2, 4, 5)
|
|
assert torch.equal(cached[3], output)
|
|
key, value, length = cached[4][0]
|
|
assert length == 4
|
|
assert torch.equal(key.flatten(), torch.tensor([1.0, 2.0, 4.0, 5.0]))
|
|
assert torch.equal(value.flatten(), torch.tensor([2.0, 4.0, 8.0, 10.0]))
|
|
|
|
|
|
def test_llama_cached_multi_token_causal_mask_uses_absolute_positions(monkeypatch):
|
|
masks = []
|
|
|
|
class Layer:
|
|
def __call__(self, x, attention_mask, freqs_cis, optimized_attention, past_key_value):
|
|
masks.append(attention_mask.clone())
|
|
return x, None
|
|
|
|
class Model:
|
|
layers = (Layer(),)
|
|
norm = None
|
|
|
|
def get_past_len(self, past_key_values):
|
|
return past_key_values[0][2]
|
|
|
|
def compute_freqs_cis(self, position_ids, device):
|
|
return None
|
|
|
|
monkeypatch.setattr(llama, "optimized_attention_for_device", lambda *args, **kwargs: None)
|
|
x = torch.zeros((1, 2, 1))
|
|
past_key_values = [(torch.empty((1, 1, 3, 1)), torch.empty((1, 1, 3, 1)), 3)]
|
|
|
|
llama.Llama2_.forward(Model(), None, embeds=x, past_key_values=past_key_values)
|
|
|
|
blocked = torch.finfo(x.dtype).min / 4
|
|
expected = torch.tensor([
|
|
[0.0, 0.0, 0.0, 0.0, blocked],
|
|
[0.0, 0.0, 0.0, 0.0, 0.0],
|
|
])
|
|
assert masks[0].shape == (2, 5)
|
|
assert torch.equal(masks[0], expected)
|
|
|
|
|
|
def test_cache_is_isolated_by_owner_and_transformer_identity(prefix_cache):
|
|
shared_transformer = TinyCausalTransformer()
|
|
first_owner = CacheOwner("same-uuid")
|
|
second_owner = CacheOwner("same-uuid")
|
|
cached_forward(shared_transformer, [1, 2], owner=first_owner)
|
|
cached_forward(shared_transformer, [1, 2, 3], owner=second_owner)
|
|
assert shared_transformer.calls == [(2, True), (3, True)]
|
|
|
|
replacement = TinyCausalTransformer()
|
|
cached_forward(replacement, [1, 2, 3], owner=first_owner)
|
|
assert replacement.calls == [(3, True)]
|
|
|
|
|
|
def test_scope_end_clears_cached_prefixes_and_disables_reuse():
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
scope = anima_cache.begin_cache_scope()
|
|
cache = scope[0]
|
|
try:
|
|
cached_forward(transformer, [1, 2], owner=owner)
|
|
assert len(cache) == 1
|
|
finally:
|
|
anima_cache.end_cache_scope(scope)
|
|
|
|
assert len(cache) == 0
|
|
cached_forward(transformer, [1, 2, 3], owner=owner)
|
|
assert transformer.calls[-1] == (3, False)
|
|
assert len(cache) == 0
|
|
|
|
|
|
def test_child_tasks_share_the_same_cache_scope():
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
|
|
async def run():
|
|
scope = anima_cache.begin_cache_scope()
|
|
primed = asyncio.Event()
|
|
|
|
async def prime():
|
|
cached_forward(transformer, [1, 2], owner=owner)
|
|
primed.set()
|
|
|
|
async def reuse():
|
|
await primed.wait()
|
|
return cached_forward(transformer, [1, 2, 3], owner=owner)
|
|
|
|
try:
|
|
first = asyncio.create_task(prime())
|
|
second = asyncio.create_task(reuse())
|
|
await first
|
|
return await second
|
|
finally:
|
|
anima_cache.end_cache_scope(scope)
|
|
|
|
output, _ = asyncio.run(run())
|
|
|
|
assert torch.equal(output, torch.tensor([[[1.0], [3.0], [6.0]]]))
|
|
assert transformer.calls == [(2, True), (1, True)]
|
|
|
|
|
|
def test_nested_cache_scopes_do_not_share_prefixes():
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
outer_scope = anima_cache.begin_cache_scope()
|
|
outer_cache = outer_scope[0]
|
|
try:
|
|
cached_forward(transformer, [1, 2], owner=owner)
|
|
inner_scope = anima_cache.begin_cache_scope()
|
|
inner_cache = inner_scope[0]
|
|
try:
|
|
cached_forward(transformer, [1, 2, 3], owner=owner)
|
|
assert len(inner_cache) == 1
|
|
finally:
|
|
anima_cache.end_cache_scope(inner_scope)
|
|
|
|
assert len(inner_cache) == 0
|
|
cached_forward(transformer, [1, 2, 4], owner=owner)
|
|
assert len(outer_cache) == 1
|
|
finally:
|
|
anima_cache.end_cache_scope(outer_scope)
|
|
|
|
assert transformer.calls == [(2, True), (3, True), (1, True)]
|
|
|
|
|
|
def test_disabled_scope_hides_an_outer_cache_scope():
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
outer_scope = anima_cache.begin_cache_scope()
|
|
try:
|
|
cached_forward(transformer, [1, 2], owner=owner)
|
|
disabled_scope = anima_cache.begin_cache_scope(False)
|
|
try:
|
|
cached_forward(transformer, [1, 2], owner=owner)
|
|
finally:
|
|
anima_cache.end_cache_scope(disabled_scope)
|
|
|
|
call_count = len(transformer.calls)
|
|
cached_forward(transformer, [1, 2], owner=owner)
|
|
assert len(transformer.calls) == call_count
|
|
finally:
|
|
anima_cache.end_cache_scope(outer_scope)
|
|
|
|
assert transformer.calls == [(2, True), (2, False)]
|
|
|
|
|
|
def test_unsafe_inputs_and_changed_model_state_do_not_reuse_cache(prefix_cache):
|
|
transformer = TinyCausalTransformer()
|
|
owner = CacheOwner()
|
|
cached_forward(transformer, [1, 2], attention_mask=torch.tensor([[True, False]]), owner=owner)
|
|
cached_forward(transformer, [1, 2], intermediate_output=0, owner=owner)
|
|
cached_forward(transformer, [1, 2], embeds_info=[{"custom": True}], owner=owner)
|
|
assert transformer.calls == [(2, False), (2, False), (2, False)]
|
|
assert len(prefix_cache) == 0
|
|
|
|
cached_forward(transformer, [1, 2], dtype=torch.float32, owner=owner)
|
|
cached_forward(transformer, [1, 2, 3], dtype=torch.float64, owner=owner)
|
|
owner.current_weight_patches_uuid = "weights-b"
|
|
cached_forward(transformer, [1, 2, 3, 4], dtype=torch.float64, owner=owner)
|
|
assert transformer.calls[-3:] == [(2, True), (3, True), (4, True)]
|
|
|
|
|
|
def test_cache_owner_reference_is_not_registered_as_child_module():
|
|
owner = torch.nn.Module()
|
|
child = torch.nn.Module()
|
|
owner.add_module("child", child)
|
|
|
|
anima_cache.set_owner(child, owner)
|
|
|
|
assert anima_cache.get_owner(child) is owner
|
|
assert "_anima_cache_owner" not in child._modules
|
|
assert list(owner.state_dict()) == []
|