mirrors/ComfyUI
1
0
Fork 0
mirror of https://github.com/comfyanonymous/ComfyUI.git synced 2025-09-12 04:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better tokenizing code for AuraFlow.

Browse Source
This commit is contained in:
comfyanonymous
2024-07-12 01:08:45 -04:00
parent b6f09cf649
commit 29c2e26724
5 changed files with 25 additions and 1175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
comfy/text_encoders/aura_t5.py
View File

@@ -1,5 +1,5 @@
from comfy import sd1_clip
from transformers import LlamaTokenizerFast
from .llama_tokenizer import LLAMATokenizer
import comfy.t5
import os
@@ -10,8 +10,8 @@ class PT5XlModel(sd1_clip.SDClipModel):
class PT5XlTokenizer(sd1_clip.SDTokenizer):
def __init__(self, embedding_directory=None):
tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer")
super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LlamaTokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model")
super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LLAMATokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):
def __init__(self, embedding_directory=None):