o
    iq                      @   s   d dl Z d dlmZ d dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZ dddddd	d
dddd
ZG dd dejZG dd dejZG dd dejjZG dd dejZG dd dejZG dd deZdefddZdS )    N)override)ComfyExtensionio      i               i   
quick_geluclip_vision_model)
hidden_size
image_sizeintermediate_sizenum_attention_headsnum_channelsnum_hidden_layers
patch_sizeprojection_dim
hidden_act
model_typec                       s,   e Zd Zdejf fdd	Zdd Z  ZS )MLPTc                    sV   t    |r||ksJ ||| _|||| _|||| _|| _t	 | _
d S N)super__init__	LayerNorm	layernormLinearfc1fc2use_residualnnGELUact_fn)selfin_dimout_dim
hidden_dimr    
operations	__class__ ;/mnt/c/Users/fbmor/ComfyUI/comfy_extras/nodes_photomaker.pyr      s   
zMLP.__init__c                 C   s>   |}|  |}| |}| |}| |}| jr|| }|S r   )r   r   r#   r   r    )r$   xresidualr+   r+   r,   forward#   s   



zMLP.forward)__name__
__module____qualname__comfyopsr   r/   __classcell__r+   r+   r)   r,   r      s    
r   c                       s4   e Zd Z fddZdd ZdejfddZ  ZS )
FuseModulec                    sF   t    t|d ||d|d| _t|||d|d| _||| _d S )N   F)r    r(   T)r   r   r   mlp1mlp2r   
layer_norm)r$   	embed_dimr(   r)   r+   r,   r   /   s   
zFuseModule.__init__c                 C   s8   t j||gdd}| || }| |}| |}|S )Ndim)torchcatr8   r9   r:   )r$   prompt_embeds	id_embedsstacked_id_embedsr+   r+   r,   fuse_fn5   s
   

zFuseModule.fuse_fnreturnc                 C   s*  | |j}| d}|jd d \}}|jd }|d|jd |jd }tj||jdd d d f |d d d f k }	||		  }
|d|jd }|d}|
d|
jd }
|| }| 
||
}| |jd ks|J |  d|jd  ||d d d f | |j |||d}|S )Nr   r7      r<   )devicez != )todtypesum	unsqueezeshapeviewr?   arangerH   flattenrD   masked_scatter_)r$   rA   rB   class_tokens_mask
num_inputs
batch_sizemax_num_inputs
seq_lengthflat_id_embedsvalid_id_maskvalid_id_embedsimage_token_embedsrC   updated_prompt_embedsr+   r+   r,   r/   <   s*   

. zFuseModule.forward)	r0   r1   r2   r   rD   r?   Tensorr/   r5   r+   r+   r)   r,   r6   .   s    r6   c                       s$   e Zd Z fddZdd Z  ZS )PhotoMakerIDEncoderc                    sd   t j | _t j }t j| j}t t||t j	j
 t j	j
jdddd| _tdt j	j
| _d S )Nr   i   F)biasi   )r3   model_managementtext_encoder_deviceload_devicetext_encoder_offload_devicetext_encoder_dtyper   r   VISION_CONFIG_DICTr4   manual_castr   visual_projection_2r6   fuse_module)r$   offload_devicerJ   r)   r+   r,   r   _   s   
zPhotoMakerIDEncoder.__init__c                 C   s   |j \}}}}}||| |||}| |d }	| |	}
| |	}|
||dd}
|||dd}tj|
|fdd}
| ||
|}|S )Nr7   rF   r<   r=   )rM   rN   vision_modelvisual_projectionrf   r?   r@   rg   )r$   id_pixel_valuesrA   rR   brS   chwshared_id_embedsrB   id_embeds_2r[   r+   r+   r,   r/   h   s   

zPhotoMakerIDEncoder.forward)r0   r1   r2   r   r/   r5   r+   r+   r)   r,   r]   ^   s    	r]   c                   @   $   e Zd Zedd Zedd ZdS )PhotoMakerLoaderc                 C   s0   t jddt jjdtddgt j gddS )Nrs   _for_testing/photomakerphotomaker_model_name
photomaker)optionsTnode_idcategoryinputsoutputsis_experimental)r   SchemaComboInputfolder_pathsget_filename_list
PhotomakerOutputclsr+   r+   r,   define_schemaz   s   zPhotoMakerLoader.define_schemac                 C   sF   t d|}t }tjj|dd}d|v r|d }|| t|S )Nrv   T)	safe_load
id_encoder)	r   get_full_path_or_raiser]   r3   utilsload_torch_fileload_state_dictr   
NodeOutput)r   ru   photomaker_model_pathphotomaker_modeldatar+   r+   r,   execute   s   

zPhotoMakerLoader.executeNr0   r1   r2   classmethodr   r   r+   r+   r+   r,   rs   y   s
    
rs   c                   @   rr   )PhotoMakerEncodec                 C   sL   t jddt jdt jdt jdt jjddddd	gt j gdd
S )Nr   rt   rv   imagecliptextTzphotograph of photomaker)	multilinedynamic_promptsdefaultrx   )	r   r~   r   r   ImageClipStringConditioningr   r   r+   r+   r,   r      s   


zPhotoMakerEncode.define_schemac                    sZ  d}t j||j }z|d|d  W n ty%   d Y nw |j	|dd}i }|D ]6}	g ||	< ||	 D ]+}
t
t fdd|
}t|t|
k r_||
d  t|t|
k sP||	 | q;q1|j|dd	\}} d
kr d dfddtdD }||d
||jtj|tj|jdd
d}n|}t|d|iggS )Nrv    rF   r<   T)return_word_idsc                    s   | d  kS )Nr7   r+   )r-   )indexr+   r,   <lambda>   s    z*PhotoMakerEncode.execute.<locals>.<lambda>)return_pooledr   c                    s0   g | ]}|  kr  k rn nd ndqS )TFr+   ).0i)num_id_imagestoken_indexr+   r,   
<listcomp>   s   0 z,PhotoMakerEncode.execute.<locals>.<listcomp>M   )rJ   rH   )rk   rA   rR   pooled_output)r3   clip_visionclip_preprocessrI   ra   floatsplitr   
ValueErrortokenizelistfilterlenappendencode_from_tokensrangerL   r?   tensorboolr   r   )r   rv   r   r   r   special_tokenpixel_valuestokens
out_tokensktfcondpooledrR   outr+   )r   r   r   r,   r      s8   zPhotoMakerEncode.executeNr   r+   r+   r+   r,   r      s
    
r   c                   @   s(   e Zd Zedeeej  fddZdS )PhotomakerExtensionrE   c                    s
   t tgS r   )rs   r   )r$   r+   r+   r,   get_node_list   s   z!PhotomakerExtension.get_node_listN)	r0   r1   r2   r   r   typer   	ComfyNoder   r+   r+   r+   r,   r      s    r   rE   c                      s   t  S r   )r   r+   r+   r+   r,   comfy_entrypoint   s   r   )r?   torch.nnr!   r   comfy.clip_modelr3   comfy.clip_vision	comfy.opstyping_extensionsr   comfy_api.latestr   r   rd   Moduler   r6   
clip_modelCLIPVisionModelProjectionr]   r   rs   r   r   r   r+   r+   r+   r,   <module>   s4    02