Remove single quote pattern to avoid wrong matches (#9842)

This commit is contained in:
Kimbing Ng 2025-09-14 04:59:19 +08:00 committed by GitHub
parent 29bf807b0e
commit e5e70636e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -22,17 +22,14 @@ class HunyuanImageTokenizer(QwenImageTokenizer):
# ByT5 processing for HunyuanImage # ByT5 processing for HunyuanImage
text_prompt_texts = [] text_prompt_texts = []
pattern_quote_single = r'\'(.*?)\''
pattern_quote_double = r'\"(.*?)\"' pattern_quote_double = r'\"(.*?)\"'
pattern_quote_chinese_single = r'(.*?)' pattern_quote_chinese_single = r'(.*?)'
pattern_quote_chinese_double = r'“(.*?)”' pattern_quote_chinese_double = r'“(.*?)”'
matches_quote_single = re.findall(pattern_quote_single, text)
matches_quote_double = re.findall(pattern_quote_double, text) matches_quote_double = re.findall(pattern_quote_double, text)
matches_quote_chinese_single = re.findall(pattern_quote_chinese_single, text) matches_quote_chinese_single = re.findall(pattern_quote_chinese_single, text)
matches_quote_chinese_double = re.findall(pattern_quote_chinese_double, text) matches_quote_chinese_double = re.findall(pattern_quote_chinese_double, text)
text_prompt_texts.extend(matches_quote_single)
text_prompt_texts.extend(matches_quote_double) text_prompt_texts.extend(matches_quote_double)
text_prompt_texts.extend(matches_quote_chinese_single) text_prompt_texts.extend(matches_quote_chinese_single)
text_prompt_texts.extend(matches_quote_chinese_double) text_prompt_texts.extend(matches_quote_chinese_double)