o
    ¶Ïi[+  ã                   @   s°   d dl Z d dlZd dlmZ ddlmZmZ d dlZejj	ZG dd„ dej
ƒZG dd„ dej
ƒZG d	d
„ d
ej
ƒZG dd„ dƒZG dd„ dej
ƒZG dd„ dej
ƒZdd„ ZdS )é    N)Únné   )ÚCrossAttentionÚFeedForwardc                       ó$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚGatedCrossAttentionDensec                    s~   t ƒ  ¡  t||||td| _t|dd| _t |¡| _t |¡| _	|  
dt t d¡¡¡ |  
dt t d¡¡¡ d| _d S ©N)Ú	query_dimÚcontext_dimÚheadsÚdim_headÚ
operationsT©ÚgluÚ
alpha_attnç        Úalpha_denser   )ÚsuperÚ__init__r   ÚopsÚattnr   ÚffÚ	LayerNormÚnorm1Únorm2Úregister_parameterr   Ú	ParameterÚtorchÚtensorÚscale©Úselfr	   r
   Ún_headsÚd_head©Ú	__class__© ú*/mnt/c/Users/fbmor/ComfyUI/comfy/gligen.pyr   
   s   
û
z!GatedCrossAttentionDense.__init__c                 C   sT   || j t | j¡ |  |  |¡||¡  }|| j t | j¡ |  |  |¡¡  }|S ©N)	r   r   Útanhr   r   r   r   r   r   )r!   ÚxÚobjsr&   r&   r'   Úforward    s   
ÿÿ
ÿÿz GatedCrossAttentionDense.forward©Ú__name__Ú
__module__Ú__qualname__r   r,   Ú__classcell__r&   r&   r$   r'   r   	   ó    r   c                       r   )ÚGatedSelfAttentionDensec                    sŒ   t ƒ  ¡  t ||¡| _t||||td| _t|dd| _t 	|¡| _
t 	|¡| _|  dt t d¡¡¡ |  dt t d¡¡¡ d| _d S r   ©r   r   r   ÚLinearÚlinearr   r   r   r   r   r   r   r   r   r   r   r   r   r    r$   r&   r'   r   +   s   
û
z GatedSelfAttentionDense.__init__c              
   C   sˆ   |j d }|  |¡}|| jt | j¡ |  |  tj||gdd¡¡d d …d|…d d …f   }|| jt | j	¡ |  
|  |¡¡  }|S )Nr   ©Údimr   )Úshaper6   r   r   r)   r   r   r   Úcatr   r   r   )r!   r*   r+   ÚN_visualr&   r&   r'   r,   E   s   

ÿÿ
ÿÿzGatedSelfAttentionDense.forwardr-   r&   r&   r$   r'   r3   *   s    r3   c                       r   )ÚGatedSelfAttentionDense2c                    sŠ   t ƒ  ¡  t ||¡| _t|||td| _t|dd| _t 	|¡| _
t 	|¡| _|  dt t d¡¡¡ |  dt t d¡¡¡ d| _d S )N)r	   r
   r   r   Tr   r   r   r   r   r4   r    r$   r&   r'   r   S   s   
ÿ
z!GatedSelfAttentionDense2.__init__c                 C   s0  |j \}}}|j \}}}|  |¡}t |¡}t |¡}t|ƒ|ks%J dƒ‚t|ƒ|ks/J dƒ‚t|ƒ}t|ƒ}|  |  tj||gdd¡¡d d …|d …d d …f }	|	 	ddd¡ 
|d||¡}	tjjj|	||fdd	}	|	 
|d|¡ 	ddd¡}
|| jt | j¡ |
  }|| jt | j¡ |  |  |¡¡  }|S )
Nz%Visual tokens must be square rootablez(Grounding tokens must be square rootabler   r7   r   é   éÿÿÿÿÚbicubic)Úmode)r9   r6   ÚmathÚsqrtÚintr   r   r   r:   ÚpermuteÚreshaper   Ú
functionalÚinterpolater   r)   r   r   r   r   )r!   r*   r+   ÚBr;   Ú_ÚN_groundÚsize_vÚsize_gÚoutÚresidualr&   r&   r'   r,   i   s0   


ÿ
ÿ
ÿÿz GatedSelfAttentionDense2.forwardr-   r&   r&   r$   r'   r<   R   r2   r<   c                   @   s(   e Zd Zd	dd„Ze ¡ d
dd„ƒZdS )ÚFourierEmbedderé@   éd   c                 C   s$   || _ || _|t |¡|  | _d S r(   )Ú	num_freqsÚtemperaturer   ÚarangeÚ
freq_bands)r!   rR   rS   r&   r&   r'   r   ‰   s   zFourierEmbedder.__init__r>   c                 C   sD   g }| j D ]}| t || ¡¡ | t || ¡¡ qt ||¡S )z*x: arbitrary shape of tensor. dim: cat dim)rU   Úappendr   ÚsinÚcosr:   )r!   r*   Úcat_dimrM   Úfreqr&   r&   r'   Ú__call__   s
   
zFourierEmbedder.__call__N)rP   rQ   )r>   )r.   r/   r0   r   r   Úno_gradr[   r&   r&   r&   r'   rO   ˆ   s    
rO   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚPositionNeté   c              
      s    t ƒ  ¡  || _|| _t|d| _|d d | _t t	 
| j| j d¡t ¡ t	 
dd¡t ¡ t	 
d|¡¡| _tj t | jg¡¡| _tj t | jg¡¡| _d S )N)rR   r=   é   i   )r   r   Úin_dimÚout_dimrO   Úfourier_embedderÚposition_dimr   Ú
Sequentialr   r5   ÚSiLUÚlinearsr   r   ÚzerosÚnull_positive_featureÚnull_position_feature)r!   r`   ra   Úfourier_freqsr$   r&   r'   r   š   s$   


ûÿ
ÿzPositionNet.__init__c                 C   sÀ   |j \}}}| d¡}|}|  |¡}| jj|j|jd ddd¡}| jj|j|jd ddd¡}	|| d| |  }|| d| |	  }|  	t
j||gdd¡}
|
j t
 ||| jg¡ks^J ‚|
S )Nr>   ©ÚdeviceÚdtyper   r7   )r9   Ú	unsqueezerb   rh   Útorl   rm   Úviewri   rf   r   r:   ÚSizera   )r!   ÚboxesÚmasksÚpositive_embeddingsrH   ÚNrI   Úxyxy_embeddingÚpositive_nullÚ	xyxy_nullr+   r&   r&   r'   r,   ¯   s"   

ÿ
ÿÿzPositionNet.forward)r^   r-   r&   r&   r$   r'   r]   ™   s    r]   c                       s4   e Zd Z‡ fdd„Zdd„ Zdd„ Zdd„ Z‡  ZS )	ÚGligenc                    s8   t ƒ  ¡  t |¡| _|| _|| _d| _t 	d¡| _
d S )Né   Úcpu)r   r   r   Ú
ModuleListÚmodule_listÚposition_netÚkey_dimÚmax_objsr   rl   Úcurrent_device)r!   Úmodulesr~   r   r$   r&   r'   r   Ç   s   
zGligen.__init__c                    s    ˆ  |||¡‰ ‡ ‡fdd„}|S )Nc                    s*   |d }ˆj | }|| ˆ j| j| jdƒS )NÚtransformer_indexrk   )r}   ro   rl   rm   )r*   Úextra_optionsÚkeyÚmodule©r+   r!   r&   r'   ÚfuncÑ   s   
z"Gligen._set_position.<locals>.func)r~   )r!   rr   rs   rt   rˆ   r&   r‡   r'   Ú_set_positionÏ   s   zGligen._set_positionc                 C   sj  |\}}}}t j| jgdd}g }	g }
|D ]>}|d | }|d | }|d |d  | }|d |d  | }d|t|	ƒ< |	t  ||||f¡ d¡g7 }	|
|d g7 }
qg }g }t|	ƒ| jk r~t j| jt|	ƒ dgddg}t j| jt|	ƒ | jgddg}t  |	| ¡ d¡ |dd¡}| d¡ |d¡}t  |
| ¡ d¡ |dd¡}|  	| 
|¡| 
|¡| 
|¡¡S )	Nr{   ©rl   r_   é   r=   r   g      ð?r   )r   rg   r€   Úlenr   rn   r   r:   Úrepeatr‰   ro   )r!   Úlatent_image_shapeÚposition_paramsrl   ÚbatchÚcÚhÚwrs   rr   rt   ÚpÚx1Úy1Úx2Úy2Úappend_boxesÚappend_condsÚbox_outÚcondsr&   r&   r'   Úset_position×   sJ   ÿÿÿÿÿÿýzGligen.set_positionc           
      C   s„   |\}}}}t j| jgdd |d¡}t j| jdgdd |dd¡}t j| j| jgdd |dd¡}	|  | |¡| |¡|	 |¡¡S )Nr{   rŠ   r   r_   )r   rg   r€   r   r   r‰   ro   )
r!   rŽ   rl   r   r‘   r’   r“   rs   r›   rœ   r&   r&   r'   Ú	set_emptyö   s"   ÿ
ÿÿ
ÿýzGligen.set_empty)r.   r/   r0   r   r‰   r   rž   r1   r&   r&   r$   r'   ry   Æ   s
    ry   c                    sF  |   ¡ }g }d}dD ]c‰ tdƒD ]\‰t‡ ‡fdd„|ƒ}tdd„ |ƒ}i }|D ]}| |d  ||d < q't|ƒdkrl|d	 jd }|d	 jd }|dkrSd
}|| }	nd}	||	 }t||||	ƒ}
|
j|dd | |
¡ qq
d|v rš| d jd }| d jd }G dd„ dt	j
jƒ}|ƒ }t||ƒ|_|j| dd t||j|ƒ}|S )Ni   )Úinput_blocksÚmiddle_blockÚoutput_blocksé   c                    s   d  ˆ ˆ¡| v od| v S )Nz{}.{}.ú.fuser.)Úformat©Úk©ÚaÚbr&   r'   Ú<lambda>	  s   
 ÿzload_gligen.<locals>.<lambda>c                 S   s   | |   d¡d fS )Nr£   r>   )Úsplitr¥   r&   r&   r'   rª     s    r   r   zlinear.weightr^   rP   F)Ústrictz"position_net.null_positive_featurezposition_net.linears.4.weightc                   @   s   e Zd ZdS )z"load_gligen.<locals>.WeightsLoaderN)r.   r/   r0   r&   r&   r&   r'   ÚWeightsLoader$  s    r­   )ÚkeysÚrangeÚfilterÚmaprŒ   r9   r3   Úload_state_dictrV   r   r   ÚModuler]   r~   ry   )ÚsdÚsd_kÚoutput_listr   Úk_tempÚn_sdr¦   r	   r"   r#   Úgatedr`   ra   r­   r“   Úgligenr&   r§   r'   Úload_gligen  sH   ÿ
ÿ
€êr»   )rA   r   r   Úldm.modules.attentionr   r   Ú	comfy.opsÚcomfyr   Úmanual_castr³   r   r3   r<   rO   r]   ry   r»   r&   r&   r&   r'   Ú<module>   s    !(6-=