Upstream the chat templates

comfy/language/chat_templates.py

@@ -11,7 +11,7 @@ KNOWN_CHAT_TEMPLATES = {}
 def _update_known_chat_templates():
     try:
         _chat_templates: Traversable
-        with files("huggingface_extra_chat_templates") / "chat_templates" as _chat_templates:
+        with files(__package__) / "chat_templates" as _chat_templates:
             _extra_jinja_templates = {Path(traversable.name).stem: traversable.read_text().replace('    ', '').replace('\n', '') for traversable in _chat_templates.iterdir() if traversable.is_file()}
             KNOWN_CHAT_TEMPLATES.update(_extra_jinja_templates)
     except ImportError as exc:

comfy/language/chat_templates/alpaca.jinja

@@ -0,0 +1,23 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] | trim + '\n\n' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if message['role'] == 'user' %}
+        {{ '### Instruction:\n' + message['content'] | trim + '\n\n' }}
+    {% elif message['role'] == 'assistant' %}
+        {{ '### Response:\n' + message['content'] | trim + eos_token + '\n\n' }}
+    {% endif %}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '### Instruction:\n' }}
+{% endif %}

comfy/language/chat_templates/amberchat.jinja

@@ -0,0 +1,23 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] | trim + '\n' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if message['role'] == 'user' %}
+        {{ '###Human: ' + message['content'] | trim + '\n' }}
+    {% elif message['role'] == 'assistant' %}
+        {{ '###Assistant: ' + message['content'] | trim + '\n' }}
+    {% endif %}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '###Assistant:' }}
+{% endif %}

comfy/language/chat_templates/chatml.jinja

@@ -0,0 +1,18 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set offset = 1 %}
+{% else %}
+    {% set offset = 0 %}
+{% endif %}
+
+{{ bos_token }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {{ '<|im_start|>' + message['role'] + '\n' + message['content'] | trim + '<|im_end|>\n' }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '<|im_start|>assistant\n' }}
+{% endif %}

comfy/language/chat_templates/chatqa.jinja

@@ -0,0 +1,30 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = 'System: ' + messages[0]['content'] | trim %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{% if messages[0]['role'] == 'context' %}
+    {% set context_message = '\n\n' + messages[0]['content'] | trim %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set context_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message + context_message}}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if message['role'] == 'user' %}
+        {{ '\n\nUser: ' + content | trim }}
+    {% elif message['role'] == 'assistant' %}
+        {{ '\n\nAssistant: '  + content | trim }}
+    {% endif %}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '\n\nAssistant:' }}
+{% endif %}

comfy/language/chat_templates/falcon-instruct.jinja

@@ -0,0 +1,20 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{{ system_message | trim }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% set content = message['content'].replace('\r\n', '\n').replace('\n\n', '\n') %}
+    {{ '\n\n' + message['role'] | capitalize + ': ' + content | trim }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '\n\nAssistant:' }}
+{% endif %}

comfy/language/chat_templates/gemma-it.jinja

@@ -0,0 +1,30 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] | trim + '\n\n' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if loop.index0 == 0 %}
+        {% set content = system_message + message['content'] %}
+    {% else %}
+        {% set content = message['content'] %}
+    {% endif %}
+
+    {% if (message['role'] == 'assistant') %}
+        {% set role = 'model' %}
+    {% else %}
+        {% set role = message['role'] %}
+    {% endif %}
+
+    {{ '<start_of_turn>' + role + '\n' + content | trim + '<end_of_turn>\n' }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{'<start_of_turn>model\n'}}
+{% endif %}

comfy/language/chat_templates/llama-2-chat.jinja

@@ -0,0 +1,24 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = '<<SYS>>\n' + messages[0]['content'] | trim + '\n<</SYS>>\n\n' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if loop.index0 == 0 %}
+        {% set content = system_message + message['content'] %}
+    {% else %}
+        {% set content = message['content'] %}
+    {% endif %}
+
+    {% if message['role'] == 'user' %}
+        {{ bos_token + '[INST] ' + content | trim + ' [/INST]' }}
+    {% elif message['role'] == 'assistant' %}
+        {{ ' ' + content | trim + ' ' + eos_token }}
+    {% endif %}
+{% endfor %}

comfy/language/chat_templates/llama-3-instruct.jinja

@@ -0,0 +1,18 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set offset = 1 %}
+{% else %}
+    {% set offset = 0 %}
+{% endif %}
+
+{{ bos_token }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '<|start_header_id|>' + 'assistant' + '<|end_header_id|>\n\n' }}
+{% endif %}

comfy/language/chat_templates/llava-v1.6-mistral-7b-hf.jinja

@@ -0,0 +1,19 @@
+{% if messages[0]['role'] == 'system' %}
+  {% set system_message = messages[0]['content'] | trim + '\n\n' %}
+  {% set messages = messages[1:] %}
+{% else %}
+  {% set system_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message }}
+{% for message in messages %}
+  {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+    {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+  {% endif %}
+
+  {% if message['role'] == 'user' %}
+    {{ '[INST] <image>' + '\n' + message['content'] | trim + ' [/INST]' }}
+  {% elif message['role'] == 'assistant' %}
+    {{ message['content'] | trim + eos_token + '\n' }}
+  {% endif %}
+{% endfor %}

comfy/language/chat_templates/mistral-instruct-v0.1.jinja

@@ -0,0 +1,19 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] | trim + '\n\n' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message}}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if message['role'] == 'user' %}
+        {{ '[INST] ' + message['content'] | trim + ' [/INST]' }}
+    {% elif message['role'] == 'assistant' %}
+        {{ message['content'] | trim + eos_token + ' ' }}
+    {% endif %}
+{% endfor %}

comfy/language/chat_templates/mistral-instruct.jinja

@@ -0,0 +1,19 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] | trim + '\n\n' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message}}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if message['role'] == 'user' %}
+        {{ '[INST] ' + message['content'] | trim + ' [/INST]' }}
+    {% elif message['role'] == 'assistant' %}
+        {{ message['content'] | trim + eos_token }}
+    {% endif %}
+{% endfor %}

comfy/language/chat_templates/openchat.jinja

@@ -0,0 +1,19 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] | trim + '<|end_of_turn|>' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {{ 'GPT4 Correct ' + message['role'] | capitalize + ': ' + message['content'] + '<|end_of_turn|>' }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ 'GPT4 Correct Assistant:' }}
+{% endif %}

comfy/language/chat_templates/phi-3.jinja

@@ -0,0 +1,18 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set offset = 1 %}
+{% else %}
+    {% set offset = 0 %}
+{% endif %}
+
+{{ bos_token }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {{ '<|' + message['role'] + '|>\n' + message['content'] | trim + '<|end|>' + '\n' }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '<|assistant|>\n' }}
+{% endif %}

comfy/language/chat_templates/saiga.jinja

@@ -0,0 +1,23 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set offset = 1 %}
+{% else %}
+    {% set offset = 0 %}
+{% endif %}
+
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %}
+        {{ raise_exception('Conversation roles must alternate user/bot/user/bot/...') }}
+    {% endif %}
+
+    {% if message['role'] == 'assistant' %}
+        {{ set role = 'bot' }}
+    {% else %}
+        {{ set role = message['role'] }}
+    {% endif %}
+
+    {{ bos_token + role + '\n' + message['content'] | trim + eos_token }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ bos_token + 'bot\n' }}
+{% endif %}

comfy/language/chat_templates/solar-instruct.jinja

@@ -0,0 +1,18 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set offset = 1 %}
+{% else %}
+    {% set offset = 0 %}
+{% endif %}
+
+{{ bos_token }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {{ '### ' + message['role'] | capitalize + ':\n' + message['content'] | trim + '\n\n' }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '### Assistant:\n' }}
+{% endif %}

comfy/language/chat_templates/vicuna.jinja

@@ -0,0 +1,23 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set system_message = messages[0]['content'] | trim + '\n\n' %}
+    {% set messages = messages[1:] %}
+{% else %}
+    {% set system_message = '' %}
+{% endif %}
+
+{{ bos_token + system_message }}
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {% if message['role'] == 'user' %}
+        {{ 'USER: ' + message['content'] | trim + '\n' }}
+    {% elif message['role'] == 'assistant' %}
+        {{ 'ASSISTANT: ' + message['content'] | trim + eos_token + '\n' }}
+    {% endif %}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ 'ASSISTANT:' }}
+{% endif %}

comfy/language/chat_templates/zephyr.jinja

@@ -0,0 +1,17 @@
+{% if messages[0]['role'] == 'system' %}
+    {% set offset = 1 %}
+{% else %}
+    {% set offset = 0 %}
+{% endif %}
+
+{% for message in messages %}
+    {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %}
+        {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {% endif %}
+
+    {{ '<|' + message['role'] + '|>\n' + message['content'] | trim + eos_token + '\n' }}
+{% endfor %}
+
+{% if add_generation_prompt %}
+    {{ '<|assistant|>\n' }}
+{% endif %}

comfy/language/generation_configs/alpaca.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/alpaca.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request."
+}

comfy/language/generation_configs/amberchat.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/amberchat.jinja",
+    "stop_str": "\n###Human",
+    "stop_token_ids": [2],
+    "system_prompt": null
+}

comfy/language/generation_configs/chatqa.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/chatqa.jinja",
+    "stop_str": null,
+    "stop_token_ids": [128001, 128009],
+    "system_prompt": "This is a chat between a user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions based on the context. The assistant should also indicate when the answer cannot be found in the context."
+}

comfy/language/generation_configs/gemma-it.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/gemma-it.jinja",
+    "stop_str": null,
+    "stop_token_ids": [1, 107],
+    "system_prompt": null
+}

comfy/language/generation_configs/llama-2-chat.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/llama-2-chat.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
+}

comfy/language/generation_configs/llama-3-instruct.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/llama-3-instruct.jinja",
+    "stop_str": null,
+    "stop_token_ids": [128001, 128009],
+    "system_prompt": null
+}

comfy/language/generation_configs/mistral-instruct-v0.1.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/mistral-instruct-v0.1.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": "Always assist with care, respect, and truth. Respond with utmost utility yet securely. Avoid harmful, unethical, prejudiced, or negative content. Ensure replies promote fairness and positivity."
+}

comfy/language/generation_configs/mistral-instruct.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/mistral-instruct.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": "Always assist with care, respect, and truth. Respond with utmost utility yet securely. Avoid harmful, unethical, prejudiced, or negative content. Ensure replies promote fairness and positivity."
+}

comfy/language/generation_configs/openchat.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/openchat.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2, 32000],
+    "system_prompt": null
+}

comfy/language/generation_configs/orca-2.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/chatml.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2, 32002],
+    "system_prompt": "You are Orca, an AI language model created by Microsoft. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."
+}

comfy/language/generation_configs/phi-3.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/phi-3.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2, 32000, 32007],
+    "system_prompt": null
+}

comfy/language/generation_configs/qwen2-chat.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/openchat.jinja",
+    "stop_str": null,
+    "stop_token_ids": [151643, 151645],
+    "system_prompt": null
+}

comfy/language/generation_configs/saiga.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/saiga.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": "Ты — Сайга, русскоязычный автоматический ассистент. Ты разговариваешь с людьми и помогаешь им."
+}

comfy/language/generation_configs/solar-instruct.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/solar-instruct.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": null
+}

comfy/language/generation_configs/vicuna.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/vicuna.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
+}

comfy/language/generation_configs/yi-chat.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/chatml.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2, 7],
+    "system_prompt": null
+}

comfy/language/generation_configs/zephyr.json

@@ -0,0 +1,6 @@
+{
+    "chat_template": "chat_templates/zephyr.jinja",
+    "stop_str": null,
+    "stop_token_ids": [2],
+    "system_prompt": null
+}

requirements.txt

@@ -50,7 +50,6 @@ opentelemetry-util-http
 opentelemetry-instrumentation-aio-pika
 opentelemetry-instrumentation-requests
 opentelemetry-semantic-conventions
-huggingface_extra_chat_templates @ git+https://github.com/AppMana/appmana-comfyui-chat-templates.git
 wrapt>=1.16.0
 certifi
 spandrel>=0.3.4