o
    iCy                     @   s  d dl Z d dlmZ d dlZd dlZd dlZd dlZddlm	Z	 d dl
Zd dlZd dlZd dlZd dlZdd ZG dd dZG d	d
 d
ejjeZdd Zdd Zdd Zdd Zdd Zdd Zdd Zd#ddZG dd dZG dd dZG dd  d eZG d!d" d"ejjZdS )$    N)CLIPTokenizer   )model_managementc                 C   sd   |  dd }|  dd }|  d}g }|d ur|| |d ur%|| ||g|t|  7 }|S )Nstartendpad)getappendlen)special_tokenslengthstart_token	end_token	pad_tokenoutput r   ,/mnt/c/Users/fbmor/ComfyUI/comfy/sd1_clip.pygen_empty_tokens   s   


r   c                   @   s   e Zd Zdd ZdS )ClipTokenWeightEncoderc                 C   s:  t  }d}d}|D ]#}t tdd |}tt||}|p&ttdd | }|| q	t|}|s7|dkrPt| drG|| | j| n	|t| j| | 	|}|d d \}	}
|
d urn|
dd j
t d	}n|
}g }td|D ]I}|	||d  }|r|	d
 }tt|D ]-}tt|| D ]"}|| | d }|dkr|| | ||  | ||  || |< qq|| qwt|dkr|	d
d  j
t d	|f}ntj|ddj
t d	|f}t|dkri }|d D ]#}|d | }|dkr|d |  jddj
t d	}|||< q||f }|S )Nr   Fc                 S   s   | d S Nr   r   ar   r   r   <lambda>!       z=ClipTokenWeightEncoder.encode_token_weights.<locals>.<lambda>c                 S   s   | d dkS )Nr         ?r   r   r   r   r   r   #   s    r      r   devicer   dimattention_mask)listmapmaxr
   allr	   hasattrr   r   encodetor   intermediate_devicerangetorchcatflatten	unsqueeze)selftoken_weight_pairs	to_encodemax_token_lenhas_weightsxtokenssectionsooutpooledfirst_pooledr   kzz_emptyijweightrextravr   r   r   encode_token_weights   sV   

(
$

z+ClipTokenWeightEncoder.encode_token_weightsN)__name__
__module____qualname__rE   r   r   r   r   r      s    r   c                       s   e Zd Zg dZdddddddejjdddd	dd
d
dd
i f fdd	Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd ZdddZ  ZS )SDClipModel)lastr:   hiddenr&   cpuM   TrJ   Ni  i  )r   r   r   Fc                    s  t    |d u r$tjtjtjtd}d|vr$i |ddi}t|t	r,|}nt
|}t|}W d    n1 s@w   Y  |d|ddi }| D ]\}}|||< qV|dd }|dd }|d u r|d urtjj||dd	}td
 ntjj}|| _||||| j| _| jj| _|| _|r|   || _d | _|	| _tj t!d| _"|| _#|| _$|
| _%|| _&|| _'d | _(|dkr|d usJ t)|| jk sJ | *d|i | j| j| j&f| _+d S )Nzsd1_clip_config.json
model_nameclip_lz{}_model_config custom_operationsquantization_metadataT)full_precision_mmz(Using MixedPrecisionOps for text encodergy&1l@rK   layer),super__init__ospathjoindirnamerealpath__file__
isinstancedictopenjsonloadr   formatitemscomfyopsmixed_precision_opslogginginfomanual_cast
operationstransformer
num_layers
max_lengthfreezerT   	layer_idxr   r,   nn	Parametertensorlogit_scaleenable_attention_maskszero_out_maskedlayer_norm_hidden_statereturn_projected_pooledreturn_attention_masksexecution_deviceabsset_clip_optionsoptions_default)r0   r   rm   rn   rT   ro   textmodel_json_configdtypemodel_classr   rv   rt   ru   rw   rx   model_optionsconfigfte_model_optionsr<   rD   rj   quant_config	__class__r   r   rV   X   sR   




zSDClipModel.__init__c                 C   s$   | j  | _ |  D ]}d|_q
d S )NF)rk   eval
parametersrequires_grad)r0   paramr   r   r   rn      s   zSDClipModel.freezec                 C   s   | d| j}| d| j| _| d| j| _t| jts"| jdkr$d S t|tr.|| _d S |d u s9t|| jkr>d| _d S d| _|| _d S )NrT   projected_pooledry   r&   rJ   rK   )	r   ro   rw   ry   r]   rT   r#   rz   rl   )r0   optionsro   r   r   r   r{      s   



zSDClipModel.set_clip_optionsc                 C   s.   | j d | _| j d | _| j d | _d | _d S )Nr   r   r   )r|   rT   ro   rw   ry   r0   r   r   r   reset_clip_options   s   
zSDClipModel.reset_clip_optionsc              	   C   sF  | j dd }| j dd}|d u r|}n|}g }g }g }|D ]r}	g }
g }g }d}d}d}|	D ]N}t|tjrst|}|dkrF||krFd}|sN|rT||krT|
d n|
d d}||g7 }|sr||krr|sr|d u rpd|
d< d}n|||f |d7 }q0tj|g|tj	d}| j
 |tjd	}d}d}g }|D ]}|d }t|rd
|d}d }|dd }|d
kr|dd }nt| j
dr| j
j||d\}}nd }|d u r|d7 }q||d  }|dd|jd j|tjd}|jd }|jd |jd kr<tj|d d d |f ||d d |d f gdd}|
d | dg|  |
|d   }
||d 7 }|||||d q|d7 }||7 }td|jd |jd  q|dkr| j
 tj| j d g| g|tj	dtjd	}tj||gdd}|
dg|  }
|| ||
 |t|
 qt|tj||tj	d||fS )Nr   r   r   Fr   Tr   )r   r~   )	out_dtype	embedding)typedatar   r   preprocess_embedr   r    )r   indexsizerC   zZWARNING: shape mismatch when trying to apply embedding, embedding will be ignored {} != {})r   r   r]   numbersIntegralintr	   r,   rr   longrk   get_input_embeddingsfloat32	is_tensorr'   r   viewshaper)   r-   rg   warningrb   sum)r0   r6   r   r   r   	cmp_token
embeds_outattention_masks
num_tokensr5   r"   tokens_tempother_embedseosr   left_padytokentokens_embed	pad_extraembeds_infor8   embrC   emb_typeind	emb_shape
padd_embedr   r   r   process_tokens   s   





 
4" 
0

 zSDClipModel.process_tokensc              
   C   sJ  | j d u r| j jj}n| j }| ||\}}}}d }| jr!|}t| jt	r+| j}n| jdkr3d}n| j
}| jd ||||| jtj|d}	| jdkrQ|	d  }
n|	d  }
| jrc|
|d 9 }
d }t|	dkr| jst|	dkr|	d d ur|	d  }n|	d	 d ur|	d	  }i }| jr||d
< t|dkr|
||fS |
|fS )Nr&   )embedsr   intermediate_outputfinal_layer_norm_intermediater~   r   rJ   r   r   r         r   r"   )ry   rk   r   rA   r   r   rt   r]   rT   r#   ro   rv   r,   r   floatru   r/   r
   rw   rx   )r0   r6   r   r   r"   r   r   attention_mask_modelr   outputsr=   pooled_outputrC   r   r   r   forward  s<   



zSDClipModel.forwardc                 C   s   | |S Nr   )r0   r6   r   r   r   r(   1  s   zSDClipModel.encodec                 C   s   | j j|dt| dddS )NFcan_assign_sd)strictassign)rk   load_state_dictgetattrr0   sdr   r   r   load_sd4  s   zSDClipModel.load_sd        c                 C   sb   t |trtt| }n|}dd |D }| j|| jdd }| jj|||||||||	|
d
S )Nc                 S      g | ]	}d d |D qS )c                 S   s   g | ]}|d  qS )r   r   .0tr   r   r   
<listcomp><      z3SDClipModel.generate.<locals>.<listcomp>.<listcomp>r   )r   br   r   r   r   <      z(SDClipModel.generate.<locals>.<listcomp>r   r   )presence_penalty)	r]   r^   nextitervaluesr   ry   rk   generate)r0   r6   	do_samplerm   temperaturetop_ktop_pmin_prepetition_penaltyseedr   tokens_onlyr   r   r   r   r   7  s   
 zSDClipModel.generate)r   )rF   rG   rH   LAYERSrd   
clip_modelCLIPTextModelrV   rn   r{   r   r   r   r(   r   r   __classcell__r   r   r   r   rI   Q   s    :X-rI   c                 C   s   g }d}d}| D ]?}|dkr(|dkr|r| | d}nd}n||7 }|d7 }q|dkrC|d8 }|dkr>| |d  d}q||7 }q||7 }q|rO| | |S )NrP   r   (r   ))r	   )stringresultcurrent_itemnesting_levelcharr   r   r   parse_parentheses@  s,   




r   c                 C   s   t | }g }|D ]O}|}t|dkrP|d dkrP|d dkrP|dd }|d}|d9 }|dkrHzt||d d  }|d | }W n   Y |t||7 }q|||fg7 }q|S )	Nr   r   r   r   r   r   :g?)r   r
   rfindr   token_weights)r   current_weightr   r9   r5   rA   xxr   r   r   r   \  s"   $
r   c                 C      |  dd} |  dd} | S )Nz\) z\( replacetextr   r   r   escape_importantp     r   c                 C   r   )Nr   r   r   r   r   r   r   r   r   unescape_importantu  r   r   c              
   C   s  t | s}ttdd | }|  |D ]X}||I}| }t|d }d}|dk r7	 W d    q|d dkr?d}|| }t	j
|t	jd}	|	||f }
~	|
W  d      W  d    S 1 sjw   Y  qW d    d S 1 s{w   Y  d S )Nc                 S   s   d| v S )Nzdata/r   r   r   r   r   r   |  r   z%safe_load_embed_zip.<locals>.<lambda>r   i      r   )r~   )zipfileZipFiler#   filternamelistreverser_   readr
   r,   
frombufferr   reshapeclone)
embed_pathmyzipnamesnmyfiler   numberlength_embed
num_embedsembedr9   r   r   r   safe_load_embed_zipz  s.   "r  c                 C   sF   t  }| D ]}|| tj|ddD ]
\}}}|| qqt|S )NT)followlinks)setaddrW   walkr#   )directoriesdirsr5   rootsubdirfiler   r   r   expand_directory_list  s   
r  c                 C   sN   g }| D ]}| |r||r|| |  qt|dkr d S tj|ddS )Nr   r    )
startswithendswithr	   r
   r,   r-   )r
  prefixsuffixout_listr<   r   r   r   bundled_embed  s   r  c              	   C   sZ  t |tr|g}t|}d }|D ]M}tjtj|| }tj|}ztj||f|kr0W qW n   Y qtj|sUg d}|D ]}|| }	tj|	rS|	} nqCn|}|d ur] nq|d u rdd S |}d }
z)|	 
dr}dd l}|jj|dd}nz
tj|ddd}W n   t|}
Y W n ty   tdt |  Y d S w |
d u r+d	|v r|d	  }tt|}
|
S t |trg }tt|D ]#}|| D ]}|| | }	|	jd
 |krq||	d
|	jd
  qqtj|dd}
|
S |d ur||v r|| }
|
S t |dd}
|
d u rt |dd|}
|
d u r+| }tt|}
|
S )N).safetensorsz.ptz.binr  r   rL   r   T)weights_onlymap_locationz1{}

error loading embedding, skipping loading: {}string_to_paramr   r    zbundle_emb.z.string_to_param.*z.{})!r]   strr  rW   rX   abspathrY   
commonpathisfilelowerr  safetensors.torchr,   	load_filera   r  	Exceptionrg   r   rb   	traceback
format_excr   r   r   r#   r+   r
   r   r	   r   r-   r  )embedding_nameembedding_directoryembedding_size	embed_key
valid_file	embed_dirr  
extensionsr5   r   	embed_outsafetensorsr
  r   r  r<   r   r   r   
load_embed  s   






r3  c                   @   s   e Zd Zddddddeddddddddddi i fddZd	efd
dZdd Zdi fdefddZdd Z	dd Z
dddZdS )SDTokenizerNrM   Tr   rO   Fc                 C   s  |d u rt jt jt jtd}|j|fi || _|d	||| _
|d	||| _d | _|| _|| _| dd }|	| _|rtt|dkrUd| _|d | _nd| _|| _|d u rdtd |d url|| _n|	rs|d | _nd| _|| _|d ur|| _n|	r|d | _|d ur|| _n
|r| j| _nd| _|| _|
| _| j }d	d
 | D | _|| _d| _d| _|| _|| _|| _ d S )Nsd1_tokenizerz{}_max_length{}_min_lengthrP   	input_idsr   r   z7WARNING: There's something wrong with your tokenizers.'c                 S   s   i | ]\}}||qS r   r   )r   r<   rD   r   r   r   
<dictcomp>  s    z(SDTokenizer.__init__.<locals>.<dictcomp>   z
embedding:)!rW   rX   rY   rZ   r[   r\   from_pretrained	tokenizerr   rb   rm   
min_lengthr   min_paddingpad_lefttokenizer_adds_end_tokenr
   tokens_startr   rg   r   r   pad_with_endpad_to_max_length	get_vocabrc   	inv_vocabr+  max_word_lengthembedding_identifierr,  embedding_keydisable_weights)r0   tokenizer_pathrm   rA  r+  r,  rG  tokenizer_classhas_start_tokenhas_end_tokenrB  r<  r   r   r   r=  r>  rH  tokenizer_datatokenizer_argsemptyvocabr   r   r   rV     sX   





zSDTokenizer.__init__r*  c                 C   s   |  }|d }d|dd }t|| j| j| j}|du rD|d}t|t|k rDt|| j| j| j}|d|t|d |fS ||fS )z
        Takes a potential embedding name and tries to retrieve it.
        Returns a Tuple consisting of the embedding and any leftover string, embedding can be None.
        r    r   N,z{} {})	splitrY   r3  r+  r,  rG  stripr
   rb   )r0   r*  split_embedleftoverr
  strippedr   r   r   _try_get_embedding  s   
zSDTokenizer._try_get_embeddingc                 C   sH   | j rt|D ]}|d| jddf qd S || jddfg|  d S )Nr   r   )r>  r+   insertr   extend)r0   r6   amountr?   r   r   r   
pad_tokens/  s
   zSDTokenizer.pad_tokensr   c              	      s  | d| j| j}| d| j| j}| d|}t|}| d| jr-|dfg}nt|d}g }|D ]\}	t|	}
t	
d| j|
}|d g}
tdt|D ]|
d	| j|  qTd
d |
D }
|
D ]u}|| jr| jdur|t| jd d}| |\ } du rtd| d n"t jdkr| fg n| fddt jd D  |dkr|}nqld}| jrd}|fdd| |d | j| D  qlq6g }g }| jdur|| jddf || t|D ]\}t|| jk}| jdurd}nd}t|dkrt|t| | j| kr| jt| | }|r[|fdd|d| D  | jdurT|| jddf ||d }n| jdurj|| jddf | jrt|  || g }| jdur|| jddf || n|fdd|D  g }t|dksq| jdur|| jddf |dur|  || | jrt|| jk r|  || jt|  |durt||k r|  ||t|  |sdd |D }|S )ab  
        Takes a prompt and converts it to a list of (token, weight, word id) elements.
        Tokens can both be integer tokens and pre computed CLIP tensors.
        Word id values are unique per word and embedding, where the id 0 is reserved for non word tokens.
        Returned list has the dimensions NxM where M is the input size of CLIP
        r6  z{}_min_paddingr<  rH  r   z	 {0}|
{0}r   r   z{}{}c                 S   s   g | ]}|d kr|qS )rP   r   r   r5   r   r   r   r   Q      z5SDTokenizer.tokenize_with_weights.<locals>.<listcomp>N
zwarning, embedding:z does not exist, ignoringc                    s   g | ]} | fqS r   r   r]  )r
  rA   r   r   r   ]  r^  rP   l   J)r   c                    s   g | ]}| fqS r   r   r   )rA   r   r   r   g  r   r7  c                       g | ]\}}|| d  fqS r   r   r   r   wr?   r   r   r   |      c                    r`  ra  r   rb  rd  r   r   r     re  c                 S   r   )c                 S   s   g | ]	\}}}||fqS r   r   )r   r   rc  _r   r   r   r     r   z@SDTokenizer.tokenize_with_weights.<locals>.<listcomp>.<listcomp>r   r]  r   r   r   r     r   )!r   rb   rG  r<  r=  r   rH  r   r   rerS  rF  r+   r
   r	   r  r+  rT  rX  rg   r   r   r?  r;  r@  r   	enumeraterE  r   rm   rZ  rB  r\  )r0   r   return_word_idstokenizer_optionskwargsr<  r=  parsed_weightsr6   weighted_segmentto_tokenizerS  wordr*  rV  r   batched_tokensbatcht_groupis_largerL  remaining_lengthr   )r
  r?   rA   r   tokenize_with_weights6  s   

$.

 
z!SDTokenizer.tokenize_with_weightsc                    s   t t fdd|S )Nc                    s   |  j | d  fS r   )rD  r   r   r   r   r     s    z(SDTokenizer.untokenize.<locals>.<lambda>)r#   r$   r0   token_weight_pairr   r   r   
untokenize     zSDTokenizer.untokenizec                 C   s   i S r   r   r   r   r   r   
state_dict  s   zSDTokenizer.state_dictc                 C   s   | j j||dS N)skip_special_tokens)r;  decoder0   	token_idsr|  r   r   r   r}       zSDTokenizer.decodeT)rF   rG   rH   r   rV   r   rX  r\  ru  rx  rz  r}  r   r   r   r   r4    s    08ir4  c                   @   sJ   e Zd Zdi dedfddZddefddZd	d
 Zdd ZdddZ	dS )SD1TokenizerNlc                 C   s`   |d ur|| _ d| j | _n
|| _ d| j | _|d| j|}t| | j|||d d S )N{}clip_{}z{}_tokenizer_class)r+  rM  )	clip_namerb   clipr   setattr)r0   r+  rM  r  r;  namer   r   r   rV     s   zSD1Tokenizer.__init__Fr   c                 K   s*   i }t | | jj||fi ||| j< |S r   )r   r  ru  r  )r0   r   ri  rk  r9   r   r   r   ru    s   "z"SD1Tokenizer.tokenize_with_weightsc                 C      t | | j|S r   )r   r  rx  rv  r   r   r   rx       zSD1Tokenizer.untokenizec                 C   s   t | | j S r   )r   r  rz  r   r   r   r   rz    r  zSD1Tokenizer.state_dictTc                 C   s   t | | jj||dS r{  )r   r  r}  r~  r   r   r   r}    ry  zSD1Tokenizer.decode)Fr  )
rF   rG   rH   r4  rV   r   ru  rx  rz  r}  r   r   r   r   r    s    r  c                       s$   e Zd Zddi f fdd	Z  ZS )SD1CheckpointClipModelrL   Nc                    s   t  j|d||d d S )NF)r   rw   r~   r   )rU   rV   )r0   r   r~   r   r   r   r   rV     s   zSD1CheckpointClipModel.__init__)rF   rG   rH   rV   r   r   r   r   r   r    s    r  c                       sT   e Zd Zddi dedf fdd	Zdd Zdd	 Zd
d Zdd ZdddZ	  Z
S )SD1ClipModelrL   Nr  c              	      s   t    |d ur|| _d| j| _n
|| _d| j| _|d| j|}i |d| ji}t| | j|d|||d| t | _|d urQ| j	| d S d S )Nr  r  z{}_classrN   )r   r~   r   r   )
rU   rV   r  rb   r  r   r  r  dtypesr  )r0   r   r~   r   r  r   r  rk  r   r   r   rV     s   
 zSD1ClipModel.__init__c                 C   s   t | | j| d S r   )r   r  r{   )r0   r   r   r   r   r{     ry  zSD1ClipModel.set_clip_optionsc                 C   s   t | | j  d S r   )r   r  r   r   r   r   r   r     s   zSD1ClipModel.reset_clip_optionsc                 C   s    || j  }t| | j|}|S r   )r  r   r  rE   )r0   r1   r9   r   r   r   rE     s   
z!SD1ClipModel.encode_token_weightsc                 C   r  r   )r   r  r   r   r   r   r   r     r  zSD1ClipModel.load_sdT   r   2   ffffff?r   c                 C   s&   t | | jj|||||||||	|
d
S )N)	r   rm   r   r   r   r   r   r   r   )r   r  r   )r0   r6   r   rm   r   r   r   r   r   r   r   r   r   r   r     s   &zSD1ClipModel.generate)	Tr  r   r  r  r   r   Nr   )rF   rG   rH   r  rV   r{   r   rE   r   r   r   r   r   r   r   r    s    r  r   ) rW   transformersr   	comfy.opsrd   r,   r(  r   rP   r   comfy.clip_modelr`   rg   r   rg  r   r   rp   ModulerI   r   r   r   r   r  r  r  r3  r4  r  r  r  r   r   r   r   <module>   s:    6 p

G C