""" Adapted from https://github.com/microsoft/unilm/blob/master/textdiffuser-2/inference_textdiffuser2_t2i_full.py#L334 The MIT License (MIT) Copyright (c) Microsoft Corporation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ from __future__ import annotations import string from typing import Optional, List from comfy.language.transformers_model_management import TransformersManagedModel from comfy.nodes.package_typing import CustomNode, InputTypes, ValidatedNodeResult from comfy.sd import CLIP from comfy.sd1_clip import SDTokenizer class TextDiffuserAddTokens(CustomNode): ALPHABET = string.digits + string.ascii_lowercase + string.ascii_uppercase + string.punctuation + ' ' # len(alphabet) = 95 TOKENS = [] @classmethod def INPUT_TYPES(cls) -> InputTypes: return { "required": { "clip": ("CLIP",) } } RETURN_TYPES = ("CLIP",) FUNCTION = "execute" def execute(self, clip: CLIP): clip = clip.clone() if len(TextDiffuserAddTokens.TOKENS) == 0: for i in range(520): TextDiffuserAddTokens.TOKENS.append(f'l{i}') TextDiffuserAddTokens.TOKENS.append(f't{i}') TextDiffuserAddTokens.TOKENS.append(f'r{i}') TextDiffuserAddTokens.TOKENS.append(f'b{i}') for c in TextDiffuserAddTokens.ALPHABET: TextDiffuserAddTokens.TOKENS.append(f'[{c}]') tokenizer: SDTokenizer = clip.tokenizer.sd_tokenizer existing_vocab = frozenset(tokenizer.tokenizer.get_vocab().keys()) tokens = [t for t in TextDiffuserAddTokens.TOKENS if t not in existing_vocab] if len(tokens) != 0: tokenizer.add_tokens(tokens) # todo: assert that the clip's vocab size is what we expect return clip, class TextDiffuserPrepareInstructPrompt(CustomNode): @classmethod def INPUT_TYPES(cls) -> InputTypes: return { "required": { "text": ("STRING", {"default": "", "multiline": True}), }, "optional": { "text_to_render": ("STRING", {"default": "", "multiline": True}) } } FUNCTION = "execute" RETURN_TYPES = "STRING", RETURN_NAMES = "INSTRUCT STRING", def execute(self, text: str, text_to_render: Optional[str] = None, *args, **kwargs) -> ValidatedNodeResult: keywords = text_to_render.split("\n") if len(keywords) > 0: # text diffusers does indeed format keywords as # ['some', 'word'] message = f'Given a prompt that will be used to generate an image, plan the layout of visual text for the image. The size of the image is 128x128. Therefore, all properties of the positions should not exceed 128, including the coordinates of top, left, right, and bottom. In addition, we also provide all keywords at random order for reference. You dont need to specify the details of font styles. At each line, the format should be keyword left, top, right, bottom. So let us begin. Prompt: {text}. Keywords: {str(keywords)}' else: message = f'Given a prompt that will be used to generate an image, plan the layout of visual text for the image. The size of the image is 128x128. Therefore, all properties of the positions should not exceed 128, including the coordinates of top, left, right, and bottom. All keywords are included in the caption. You dont need to specify the details of font styles. At each line, the format should be keyword left, top, right, bottom. So let us begin. Prompt: {text}' return message, class TextDiffuserDecodeLayoutString2ClipString(CustomNode): @classmethod def INPUT_TYPES(cls) -> InputTypes: return { "required": { "layout_model": ("MODEL", {}), "clip": ("CLIP", {}), "prompt": ("STRING", {"forceInput": True}), "instruct_response": ("STRING", {"forceInput": True}) } } FUNCTION = "execute" RETURN_TYPES = "STRING", RETURN_NAMES = "CLIP STRING", def execute(self, layout_model: TransformersManagedModel, clip: CLIP, prompt: str, instruct_response: str | List[str], *args, **kwargs) -> ValidatedNodeResult: # todo: better support for batching if isinstance(instruct_response, List): instruct_response = instruct_response[0] current_ocr = instruct_response.split('\n') words = [clip.tokenizer.sd_tokenizer.tokenizer.eos_token, clip.tokenizer.sd_tokenizer.tokenizer.bos_token] for ocr in current_ocr: ocr = ocr.strip() # .com ?? if len(ocr) == 0 or '###' in ocr or '.com' in ocr: continue items = ocr.split() pred = ' '.join(items[:-1]) box = items[-1] l, t, r, b = map(int, box.split(',')) words.extend([f'l{l}', f't{t}', f'r{r}', f'b{b}']) char_list = [f'[{i}]' for i in pred] words.extend(char_list) words.append(clip.tokenizer.sd_tokenizer.tokenizer.eos_token) return prompt + ' ' + ' '.join(words), NODE_CLASS_MAPPINGS = {} for cls in ( TextDiffuserDecodeLayoutString2ClipString, TextDiffuserPrepareInstructPrompt, TextDiffuserAddTokens, ): NODE_CLASS_MAPPINGS[cls.__name__] = cls