o
    i]                     @   s2  d dl Z d dlZd dlmZ d dlm  mZ d dlmZ d dl	Z
G dd dejZG dd dejZG dd dejjZG d	d
 d
ejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZdS )    N)optimized_attentionc                       sD   e Zd Zdedef fddZdejdejfddZd	d
 Z  Z	S )GELUdim_indim_outc                    s"   t    |j||||d| _d S Ndevicedtype)super__init__Linearproj)selfr   r   
operationsr   r	   	__class__ @/mnt/c/Users/fbmor/ComfyUI/comfy/ldm/hunyuan3dv2_1/hunyuandit.pyr   
   s   
zGELU.__init__gatereturnc                 C   s4   |j jdkrt|jtjdj|jdS t|S )Nmpsr	   )r   typeFgelutotorchfloat32r	   )r   r   r   r   r   r      s   
z	GELU.geluc                 C   s   |  |}| |}|S N)r   r   )r   hidden_statesr   r   r   forward   s   

zGELU.forward)
__name__
__module____qualname__intr   r   Tensorr   r    __classcell__r   r   r   r   r      s    r   c                       sF   e Zd Z		ddededef fddZd	ejd
ejfddZ  Z	S )FeedForwardN           dimmultdropoutc	           
         s   t    |d u rt|| }|d ur|n|}t|||||d}	tg | _| j|	 | jt| | j|j	||||d d S )Nr   r   r	   r   )
r
   r   r$   r   nn
ModuleListnetappendDropoutr   )
r   r*   r   r+   r,   	inner_dimr   r   r	   act_fnr   r   r   r      s   
zFeedForward.__init__r   r   c                 C   s   | j D ]}||}q|S r   )r0   )r   r   moduler   r   r   r    /   s   

zFeedForward.forward)Nr(   r)   NNNN
r!   r"   r#   r$   floatr   r   r%   r    r&   r   r   r   r   r'      s    r'   c                   @   s$   e Zd Zedd Zedd ZdS )
AddAuxLossc                 C   s   |j | _|j| _|S r   )requires_gradrequires_aux_lossr	   )ctxxlossr   r   r   r    6   s   zAddAuxLoss.forwardc                 C   s&   d }| j rtjd| j|jd}||fS )N   r	   r   )r:   r   onesr	   r   )r;   grad_output	grad_lossr   r   r   backward>   s   zAddAuxLoss.backwardN)r!   r"   r#   staticmethodr    rC   r   r   r   r   r8   4   s
    
r8   c                       s4   e Zd Zd fdd	Zdejdejfd	d
Z  ZS )MoEGate      {Gz?Nc                    sF   t    || _|| _|| _|| _ttj	| j| jf||d| _
d S r   )r
   r   top_kn_routed_expertsalpha
gating_dimr.   	Parameterr   emptyweight)r   	embed_dimnum_expertsnum_experts_per_tokaux_loss_alphar   r	   r   r   r   r   K   s   
$zMoEGate.__init__r   r   c                 C   s   | d|d}tj|tjj| j|j|j	dd d}|j
dd}tj|| jddd\}}| jr\| jdkr\|}tj| d| jd }||  }|d	}	|	| | j  | j }
nd }
|||
fS )
Nr?   )biasr*   F)kr*   sortedr)   )	minlengthr   )viewsizer   linearcomfymodel_managementcast_torO   r	   r   softmaxr   topkrI   trainingrK   bincountrJ   r7   numelmeansum)r   r   logitsscorestopk_weighttopk_idxscores_for_auxcountscePiaux_lossr   r   r   r    V   s   $

zMoEGate.forward)rF   rG   rH   NN)r!   r"   r#   r   r   r%   r    r&   r   r   r   r   rE   I   s    rE   c                	       sT   e Zd Z		ddedededef fd	d
ZdejfddZe	 dd Z
  ZS )MoEBlock   rG   r)   NrQ   	moe_top_kr,   ff_inner_dimc	           	         sl   t    || _|| _t fddt|D | _t|| d| _	t
 d| _d S )Nc              
      s    g | ]}t  d qS )r,   r3   r   r   r	   )r'   ).0_r   r*   r,   r	   rs   r   r   r   
<listcomp>z   s    z%MoEBlock.__init__.<locals>.<listcomp>)rQ   rR   r   r	   rt   )r
   r   rr   rQ   r.   r/   rangeexpertsrE   r   r'   shared_experts)	r   r*   rQ   rr   r,   rs   r   r   r	   r   rw   r   r   s   s   

zMoEBlock.__init__r   c                 C   s  |}|j }| |\}}}|d|j d }|d}| jrj|j| jdd}tj||jd}t	| j
D ]\}	}
|
|||	k }||j|||	k< q3|jg |j dR  |d jdd}|j| }t||}n| j|||dddj| }|| | }|S )NrT   r   rV   r   r>   )flat_expert_indicesflat_expert_weights)shaper   rZ   rb   repeat_interleaverr   r   
empty_liker	   	enumeraterz   r   	unsqueezerf   r8   apply	moe_inferr{   )r   r   identity
orig_shaperj   ri   ro   flat_topk_idxyiexperttmpr   r   r   r       s"   
(
zMoEBlock.forwardc                 C   s   t |}| }| d}|| j }t|D ]:\}}	|dkr#dn||d  }
|
|	kr.q| j| }||
|	 }|| }||}||||
|	   |	d|| q|S Nr   r>   )
r   
zeros_likeargsortrc   cumsumrr   r   rz   mul_
index_add_)r   r<   r|   r}   expert_cacheidxstokens_per_expert
token_idxsr   end_idx	start_idxr   exp_token_idxexpert_tokens
expert_outr   r   r   r      s   


zMoEBlock.moe_infer)rq   rG   r)   NNNN)r!   r"   r#   r$   r7   r   r   r%   r    no_gradr   r&   r   r   r   r   rp   r   s    rp   c                	       sD   e Zd Z		ddedededef fdd	Zd
ejfddZ  Z	S )	Timestepsr)         ?'  num_channelsdownscale_freq_shiftscale
max_periodc                    s   t    || _|d }t| tj|tjd ||  }t|}|d dkr3t	||
dg}| jd|dd || _d S )NrG   r   r>   inv_freqF)
persistent)r
   r   r   mathlogr   aranger   expcat	new_zerosregister_bufferr   )r   r   r   r   r   half_dimexponentr   r   r   r   r      s   


zTimesteps.__init__	timestepsc                 C   s   |  d| j|jd }| }| }tj||gdd}| j	dkr,|| j	 }|j
d | jkr?|d d d | jf }|S )Nr>   r   rV   r   )r7   r   r   r   r   sincosr   r   r   r~   r   )r   r   r<   sin_embcos_embembr   r   r   r       s   "

zTimesteps.forward)r)   r   r   r6   r   r   r   r   r      s    r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )TimestepEmbedder   Nc                    sp   t    t|j||d||dt |j||d||d| _|| _|d ur1|j||d||d| _t	|| _
d S )NTrU   r   r	   F)r
   r   r.   
Sequentialr   r   mlpfrequency_embedding_size	cond_projr   
time_embed)r   hidden_sizer   cond_proj_dimr   r   r	   r   r   r   r      s   
zTimestepEmbedder.__init__c                 C   sH   |  || jd jj}|d ur| |}|| }| |}|dS r   )r   r   r   rO   r	   r   r   )r   r   	conditiontimestep_embed
cond_embedtime_conditionedr   r   r   r       s   


zTimestepEmbedder.forward)r   NNNN)r!   r"   r#   r   r    r&   r   r   r   r   r      s    r   c                       s4   e Zd Zdddddef fddZdd Z  ZS )MLPNr-   widthc                   sN   t    || _|j||d ||d| _|j|d |||d| _t | _d S )Nr(   r   )	r
   r   r   r   fc1fc2r.   r   r   )r   r   r   r   r	   r   r   r   r   	  s
   
zMLP.__init__c                 C   s   |  | | |S r   )r   r   r   r   r<   r   r   r   r      s   zMLP.forward)r!   r"   r#   r$   r   r    r&   r   r   r   r   r     s    r   c                       <   e Zd Zddejddddfdef fddZdd Z  ZS )	CrossAttentionTFNuse_fp16c                    s   t    || _|| _|| _| j| | _| jd | _|j||||
|	d| _|j||||
|	d| _	|j||||
|	d| _
|r@d}nd}|tjkrK|j}n|j}|rZ|| jd||
|	dnt | _|rk|| jd||
|	dnt | _|j||d|
|	d| _d S )N      r    @ >ư>Telementwise_affineepsr   r	   )r
   r   qdimkdim	num_headshead_dimr   r   to_qto_kto_vr.   	LayerNormRMSNormIdentityq_normk_normout_proj)r   r   r   r   qkv_biasqk_norm
norm_layerr   r   r	   r   kwargsr   r   r   r   r     s$   

""zCrossAttention.__init__c                 C   s4  |j \}}}|j \}}}|t| j j}| |}| |}| |}	tj	||	fdd}
|
j d | j
 d }|
dd| j
|d }
tj|
|dd\}}	|||| j
| j}|||| j
| j}|	||| j
| j }	| |}| |}t|||| j
| j |||| j
| j |	| j
dd}| |}|S )NrT   rV   rG   r>   Fheadslow_precision_attention)r~   r   nextr   
parametersr	   r   r   r   r   r   rZ   splitr   reshaper   r   r   r   )r   r<   r   bs1rv   s2qrW   vkv
split_sizeoutr   r   r   r    =  s0   





zCrossAttention.forward	r!   r"   r#   r.   r   boolr   r    r&   r   r   r   r   r     s    )r   c                       r   )		AttentionTFNr   c
                    s   t    || _|| _| j| | _| jd | _|j|||||	d| _|j|||||	d| _|j|||||	d| _	|r=d}
nd}
|t
jkrH|j}n|j}|rW|| jd|
||	dnt
 | _|rh|| jd|
||	dnt
 | _|j||||	d| _d S )Nr   r   r   r   Tr   r   )r
   r   r*   r   r   r   r   r   r   r   r.   r   r   r   r   r   r   )r   r*   r   r   r   r   r   r   r   r	   r   r   r   r   r   c  s"   

""zAttention.__init__c                 C   s  |j \}}}| |}| |}| |}tj|||fdd}|j d | j d }	|dd| j|	d }
tj|
|	dd\}}}|	||| j| j
}|	||| j| j
}|	||| j| j
 }| |}| |}t|	||| j| j
 |	||| j| j
 || jdd}| |}|S )NrT   rV      r>   Fr   )r~   r   r   r   r   r   r   rZ   r   r   r   r   r   r   r   )r   r<   BNrv   querykeyvalueqkv_combinedr   qkvr   r   r   r      s,   





zAttention.forwardr   r   r   r   r   r   a  s    $r   c                       sX   e Zd Zddejdddddddddddfdeded	ed
ef fddZdddZ  Z	S )HunYuanDiTBlock   FT   rG   Nuse_moerQ   rr   r   c                    s<  t    |r
d}nd}||d|||d| _t|||||||||d	| _||d|||d| _|
| _| jrEtt	 |j
||d||d| _t||||||||||d
| _||d|||d| _|	rw||d|||d| _|j
d| |||d| _nd | _|| _| jrt|||d	t|d
 |||d| _d S t||||d| _d S )Nr   r   Tr   )r   r   r   r   r   r   r	   r   r   rG   r   r)   g      @)rQ   rr   r,   rs   r   r	   r   )r   r   r   r	   )r
   r   norm1r   attn1norm2timested_modulater.   r   SiLUr   default_modulationr   attn2norm3	skip_normskip_linearr   rp   r$   moer   r   )r   r   
c_emb_sizer   text_states_dimr   r   qk_norm_layerr   skip_connectionr  r   rQ   rr   r   r   r   r	   r   r   r   r   r     sH   





zHunYuanDiTBlock.__init__c           	      C   s   | j d urtj||gdd}|  |}| |}| jr(| |jdd}|| }| | |}|| }|| 	| 
|| }| |}| jrP|| | }|S || | }|S )NrT   rV   r>   )r  r   r   r  r  r  r   r   r   r  r  r  r   r	  r   )	r   r   conditioningtext_statesskip_tensorcombinedmodulation_shiftself_attn_out	mlp_inputr   r   r   r      s    



zHunYuanDiTBlock.forward)NNN)
r!   r"   r#   r.   r   r   r$   r   r    r&   r   r   r   r   r     s0    Dr   c                       s,   e Zd Zddef fddZdd Z  ZS )	
FinalLayerFNr   c                    sH   t    |r
d}nd}|j|d|||d| _|j||d||d| _d S )Nr   r   Tr   r   )r
   r   r   
norm_finalr   r\   )r   final_hidden_sizeout_channelsr   r   r   r	   r   r   r   r   r   	  s   
zFinalLayer.__init__c                 C   s,   |  |}|d d dd f }| |}|S )Nr>   )r  r\   r   r   r   r   r      s   

zFinalLayer.forward)FNN)r!   r"   r#   r   r   r    r&   r   r   r   r   r    s    r  c                       sx   e Zd Z												
					ddededededededededededef fddZi fddZ  ZS )HunYuanDiTPlain@      r      rF   TFrq   layerr   rG   Nin_channelsr   context_dimdepthr   r   r   num_moe_layersrQ   rr   r   c                    s   | _ t   | _|| _|| _| _| _|
dkr
jn
j	
j	 | _
|	| _
j|dd| _td |	
d| _t 	
fddtD | _| _t| j
d| _d S )	Nr  Tr   r(   )r   r   r	   r   c              	      s   g | ]C}t di d ddd dddd|d kd	d
| 	kr/dnddddddd
qS )r   r
  r   r  r   r   r  r  rG   r   r   TFrQ   rr   r   r   r	   r   r   )r   )ru   r  r  r   r   r	   r   rr   normrQ   r   r!  r   r   r   r   r   r   rx   I  sF    	
z,HunYuanDiTPlain.__init__.<locals>.<listcomp>)r   r   r   r	   )r	   r
   r   r   r  r  r   r   r   r   r  guidance_cond_proj_dimr   
x_embedderr   
t_embedderr.   r/   ry   blocksr  final_layer)r   r  r   r  r   r   r   r   r!  r$  	norm_typerQ   rr   r   r	   r   r   r   r   r"  r   r     s$   
(
zHunYuanDiTPlain.__init__c                    sl  | dd}|jddd\}}tj||gdd}|}d| }| j||dd}	|jt| j	 j
d	}| |}
tj|	|
gd
d} fdd}g }|di }|di }t| jD ]=\} || jd krid }n| }d|f|v r|d|f ||	||dd|i}n ||	||d}|| jd k r|| q[| |}| ddd }|jddd\}}t||gS )NrT   rG   r   rV   r   guidance_cond)r   r   r>   c                    s"    | d | d | d |  ddS )Nr<   tcondskipr  )get)argsblockr   r   
block_wrapp  s   z+HunYuanDiTPlain.forward.<locals>.block_wrappatches_replaceditr3  )r<   r,  r-  r.  original_blockr/  g      )movedimchunkr   r   r&  r0  r   r   r%  r   r	   r   r'  r   popr1   r(  )r   r<   r,  contexttransformer_optionsr   
uncond_embcond_embmain_conditiontime_embedded
x_embeddedr  r4  
skip_stackr5  blocks_replaceidx
skip_inputoutputr   r2  r   r    _  sD   




zHunYuanDiTPlain.forward)r  r  r   r  rF   TFrq   r  r  r   rG   FNNN)r!   r"   r#   r$   r   r   r    r&   r   r   r   r   r    sR    	Br  )r   r   torch.nnr.   torch.nn.functional
functionalr   comfy.ldm.modules.attentionr   comfy.model_managementr]   Moduler   r'   autogradFunctionr8   rE   rp   r   r   r   r   r   r   r  r  r   r   r   r   <module>   s&    )L-NEa