ComfyUI/comfy/transformers_compat.py

try:
    from transformers import T5TokenizerFast
except (ImportError, ModuleNotFoundError):
    from transformers import T5Tokenizer as T5TokenizerFast

try:
    from transformers import LlamaTokenizerFast
except (ImportError, ModuleNotFoundError):
    from transformers import LlamaTokenizer as LlamaTokenizerFast

try:
    from transformers import CLIPTokenizerFast
except (ImportError, ModuleNotFoundError):
    from transformers import CLIPTokenizer as CLIPTokenizerFast

try:
    from transformers import GPT2TokenizerFast
except (ImportError, ModuleNotFoundError):
    from transformers import GPT2Tokenizer as GPT2TokenizerFast

try:
    from transformers import BertTokenizerFast
except (ImportError, ModuleNotFoundError):
    from transformers import BertTokenizer as BertTokenizerFast

try:
    from transformers import Qwen2TokenizerFast
except (ImportError, ModuleNotFoundError):
    try:
        from transformers import Qwen2Tokenizer as Qwen2TokenizerFast
    except (ImportError, ModuleNotFoundError):
        # Fallback if neither exists, primarily for earlier versions or specific environments
        Qwen2TokenizerFast = None

# Alias Qwen2Tokenizer to the "Fast" version we found/aliased, as we might use either name
Qwen2Tokenizer = Qwen2TokenizerFast

try:
    from transformers import ByT5TokenizerFast
except ImportError:
    try:
        from transformers import ByT5Tokenizer as ByT5TokenizerFast
    except (ImportError, ModuleNotFoundError):
        ByT5TokenizerFast = None

ByT5Tokenizer = ByT5TokenizerFast

__all__ = [
    "T5TokenizerFast",
    "LlamaTokenizerFast",
    "CLIPTokenizerFast",
    "GPT2TokenizerFast",
    "BertTokenizerFast",
    "Qwen2Tokenizer",
    "ByT5Tokenizer",
]