o
i @ s d Z ddlZddlmZ ddlmZ ddlZG dd dejZ G dd dejZ
dd Z G d
d d ejZ G d d
d
ejZ
G dd dejZG dd dejZG dd dejZdS )a
This file is part of ComfyUI.
Copyright (C) 2024 Stability AI
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
N)optimized_attentionc s* e Zd Zd fdd Zi fddZ ZS )OptimizedAttention Nc sl t || _|j||d||d| _|j||d||d| _|j||d||d| _|j||d||d| _d S )NT)biasdtypedevice)super__init__headsLinearto_qto_kto_vout_projselfcnheaddropoutr r
operations __class__ 6/mnt/c/Users/fbmor/ComfyUI/comfy/ldm/cascade/common.pyr s
zOptimizedAttention.__init__c C s< | |}| |}| |}t|||| j|d}| |S )Ntransformer_options)r r
r r r
r )r qkvr outr r r forward# s
zOptimizedAttention.forwardr NNN__name__
__module__ __qualname__r r
__classcell__r r r r r
r c s, e Zd Zd fdd Zdi fddZ ZS ) Attention2Dr Nc s" t t|||||d| _d S )Nr r r )r r r attnr r r r r - s
zAttention2D.__init__Fc C sj |j }||d|ddddd}|r tj||gdd}| j||||d}|dddj| }|S )Nr dimr )shapeviewsizepermutetorchcatr* )r xkv self_attnr
orig_shaper r r r 2 s $zAttention2D.forwardr! r" r r r r r( , s r( c C s G dd d| j }|S )Nc s( e Zd Z fddZ fddZ ZS )z#LayerNorm2d_op..LayerNorm2dc s t j|i | d S N)r r )r argskwargsr r r r ? s z,LayerNorm2d_op..LayerNorm2d.__init__c s$ t |ddddddddS Nr r- r+ )r r r3 r r6 r r r r B s $z+LayerNorm2d_op..LayerNorm2d.forwardr" r r r r LayerNorm2d> s r@ ) LayerNorm)r r@ r r r LayerNorm2d_op= s rB c s* e Zd ZdZd fdd Zdd Z ZS )GlobalResponseNormzwfrom https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105Nc
sJ t ttjddd|||d| _ttjddd|||d| _d S )Nr+ r r )r r nn Parameterr4 emptygammabeta)r r/ r r r r r r H s
"zGlobalResponseNorm.__init__c C sT t j|dddd}||jdddd }tj| j||| tj| j| | S )Nr- )r+ r- T)pr/ keepdimr, )r/ rK ư>)r4 normmeancomfyops
cast_to_inputrH rI )r r6 GxNxr r r r M s ,zGlobalResponseNorm.forward)NN)r# r$ r% __doc__r r r& r r r r rC F s rC c s( e Zd Zd fdd Zd
ddZ ZS ) ResBlockr r> r Nc s t |j||||d |||d| _t||dd||d| _t|j|| |d ||dt t
|d ||dt ||j|d |||d| _ d S )Nr- ) kernel_sizepaddinggroupsr r FrL elementwise_affineepsr r rD )
r r Conv2d depthwiserB rM rE
Sequentialr GELUrC Dropout channelwise)r r c_skiprV r r r r r r r r T s
zResBlock.__init__c C sX |}| | |}|d urtj||gdd}| |dddddddd}|| S )Nr+ r. r r- r> )rM r^ r4 r5 rb r3 )r r6 x_skipx_resr r r r a s "zResBlock.forward)r r> r NNNr: r" r r r r rU S s
rU c s* e Zd Zd fdd Zi fddZ ZS ) AttnBlockTr Nc sb t || _t||dd||d| _t||||||d| _tt |j
||||d| _ d S )NFrL rY r) rD ) r r r8 rB rM r( attentionrE r_ SiLUr kv_mapper) r r c_condr r8 r r r r r r r r k s
zAttnBlock.__init__c C s, | |}|| j| ||| j|d }|S )N)r8 r )ri rg rM r8 )r r6 r7 r r r r r u s
zAttnBlock.forward)Tr NNNr" r r r r rf j r' rf c s& e Zd Zd fdd Zdd Z ZS )FeedForwardBlockr Nc sv t t||dd||d| _t|j||d ||dt t|d ||dt ||j|d |||d| _
d S )NFrL rY r\ rD ) r r rB rM rE r_ r r` rC ra rb )r r r r r r r r r r | s
zFeedForwardBlock.__init__c C s0 || | |dddddddd }|S r= )rb rM r3 r? r r r r s ,zFeedForwardBlock.forwardr! r" r r r r rk { s rk c s0 e Zd Zdgdddf fdd Zdd Z ZS )
TimestepBlockscaNc
sZ t |j||d ||d| _|| _|D ]}t| d| |j||d ||d qd S )Nr- rD mapper_)r r r mappercondssetattr)r r
c_timesteprp r r r cnamer r r r s
&zTimestepBlock.__init__c C s |j t| jd dd}| |d d d d d d d f j ddd\}}t| jD ],\}}t| d| ||d d d d d d d f j ddd\}}|| || }}q)|d| | S )Nr+ r. r r- rn )chunklenrp ro enumerategetattr) r r6 tabir acbcr r r r s 0>zTimestepBlock.forwardr" r r r r rl s rl )rT r4 torch.nnrE comfy.ldm.modules.attentionr comfy.opsrO Moduler r( rB rC rU rf rk rl r r r r s