Merge branch 'master' into master

This commit is contained in:
samith 2026-07-04 21:48:45 +05:30 committed by GitHub
commit 6251a537d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -543,18 +543,24 @@ class SDTokenizer:
def _try_get_embedding(self, embedding_name:str): def _try_get_embedding(self, embedding_name:str):
''' '''
Takes a potential embedding name and tries to retrieve it. Takes a potential embedding name and tries to retrieve it.
Returns a Tuple consisting of the embedding and any leftover string, embedding can be None. Returns a Tuple consisting of the embedding, the cleaned embedding name, and any leftover string, embedding can be None.
''' '''
split_embed = embedding_name.split() split_embed = embedding_name.split()
embedding_name = split_embed[0] embedding_name = split_embed[0]
leftover = ' '.join(split_embed[1:]) leftover = ' '.join(split_embed[1:])
match = re.search(r'[<\[]', embedding_name)
if match is not None:
leftover = embedding_name[match.start():] + (" " + leftover if leftover else "")
embedding_name = embedding_name[:match.start()]
embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size, self.embedding_key) embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size, self.embedding_key)
if embed is None: if embed is None:
stripped = embedding_name.strip(',') stripped = embedding_name.strip(',')
if len(stripped) < len(embedding_name): if len(stripped) < len(embedding_name):
embed = load_embed(stripped, self.embedding_directory, self.embedding_size, self.embedding_key) embed = load_embed(stripped, self.embedding_directory, self.embedding_size, self.embedding_key)
return (embed, "{} {}".format(embedding_name[len(stripped):], leftover)) return (embed, embedding_name, "{} {}".format(embedding_name[len(stripped):], leftover))
return (embed, leftover) return (embed, embedding_name, leftover)
def pad_tokens(self, tokens, amount): def pad_tokens(self, tokens, amount):
if self.pad_left: if self.pad_left:
@ -585,7 +591,7 @@ class SDTokenizer:
tokens = [] tokens = []
for weighted_segment, weight in parsed_weights: for weighted_segment, weight in parsed_weights:
to_tokenize = unescape_important(weighted_segment) to_tokenize = unescape_important(weighted_segment)
split = re.split(' {0}|\n{0}'.format(self.embedding_identifier), to_tokenize) split = re.split(r'(?<=\s){}'.format(re.escape(self.embedding_identifier)), to_tokenize)
to_tokenize = [split[0]] to_tokenize = [split[0]]
for i in range(1, len(split)): for i in range(1, len(split)):
to_tokenize.append("{}{}".format(self.embedding_identifier, split[i])) to_tokenize.append("{}{}".format(self.embedding_identifier, split[i]))
@ -595,7 +601,7 @@ class SDTokenizer:
# if we find an embedding, deal with the embedding # if we find an embedding, deal with the embedding
if word.startswith(self.embedding_identifier) and self.embedding_directory is not None: if word.startswith(self.embedding_identifier) and self.embedding_directory is not None:
embedding_name = word[len(self.embedding_identifier):].strip('\n') embedding_name = word[len(self.embedding_identifier):].strip('\n')
embed, leftover = self._try_get_embedding(embedding_name) embed, embedding_name, leftover = self._try_get_embedding(embedding_name)
if embed is None: if embed is None:
logging.warning(f"warning, embedding:{embedding_name} does not exist, ignoring") logging.warning(f"warning, embedding:{embedding_name} does not exist, ignoring")
else: else: