o
    i                     @   s>   d dl Z d dlmZ d dlmZ d dlZG dd dejZdS )    N)optimized_attentionc                	       s8   e Zd Zd
dedededef fddZdd	 Z  ZS )AttentionPoolNspacial_dim	embed_dim	num_heads
output_dimc                    s   t    ttj|d |||d| _|j||||d| _|j||||d| _	|j||||d| _
|j||p8|||d| _|| _|| _d S )N   )dtypedevice)super__init__nn	Parametertorchemptypositional_embeddingLineark_projq_projv_projc_projr   r   )selfr   r   r   r   r	   r
   
operations	__class__ 5/mnt/c/Users/fbmor/ComfyUI/comfy/ldm/hydit/poolers.pyr      s   

zAttentionPool.__init__c                 C   sf  |d d d | j jd d f }|ddd}tj|jddd|gdd}|tj| j d d d d d f | }| 	|d d }| 
|}| |}|jd }| j| j }|d|| j |dd|| jd|}||jd || j |dd|| jd|}||jd || j |dd|| jd|}t|||| jdddd}| |}|dS )	Nr   r      T)dimkeepdim)r   )skip_reshape)r   shapepermuter   catmeancomfyopscast_to_inputr   r   r   r   r   view	transposer   r   squeeze)r   xqkv
batch_sizehead_dimattn_outputr   r   r   forward   s    &


*00

zAttentionPool.forward)NNNN)__name__
__module____qualname__intr   r3   __classcell__r   r   r   r   r      s     
r   )	r   torch.nnr   comfy.ldm.modules.attentionr   	comfy.opsr&   Moduler   r   r   r   r   <module>   s
    