o
    iu)                     @   s`   d dl Z d dlZddlmZ ddlmZ G dd deZG dd de jjZG d	d
 d
eZ	dS )    N   )QwenImageTransformer2DModel)QwenImageTransformerBlockc                       s   e Zd Zd fdd	Z  ZS )QwenImageFunControlBlockFNc                    sN   t  j||||||d || _|r|j||||d| _|j||||d| _d S )N)dimnum_attention_headsattention_head_dimdtypedevice
operationsr
   r	   )super__init__has_before_projLinearbefore_proj
after_proj)selfr   r   r   r   r	   r
   r   	__class__ =/mnt/c/Users/fbmor/ComfyUI/comfy/ldm/qwen_image/controlnet.pyr   	   s   z!QwenImageFunControlBlock.__init__)FNNN)__name__
__module____qualname__r   __classcell__r   r   r   r   r      s    r   c                       sV   e Zd Z										d fd	d
	Zdd Zdddi dfdejfddZ  ZS )QwenImageFunControlNetModel               <   r      r   $   0   Nc                    s|   t    || _|| _t|| _d| _|
j|||	|d| _t	j
g | _t|D ]}| jt||||dk||	|
d q(d S )Ng      ?r   r   )r   r   r   r   r	   r
   r   )r   r   r	   main_model_doubletupleinjection_layers
hint_scaler   control_img_intorchnn
ModuleListcontrol_blocksrangeappendr   )r   control_in_features	inner_dimr   r   num_control_blocksr'   r)   r	   r
   r   ir   r   r   r      s(   

z$QwenImageFunControlNetModel.__init__c              	   C   s  |d u rd S |j dkr|d}| jjjd d }|jd dkr@|dkr@t|d d d df }t|}tj|||gdd}|j\}}}}}	tjj	
|d|	d d|d f}
|
j}|
|d |d |d |d	 d d|d
 d d}
|
ddddddd}
|
|||d d  |	d d  |d }
| jjjd }|
jd
 }||k rtj|
jd |
jd || f|
j|
jd}tj|
|gd
d}
|
S ||kr|
d d d d d |f }
|
S )N      r      !   r   r      r!      r   )ndim	unsqueezer+   weightshaper,   
zeros_likecatr-   
functionalpadviewpermutereshapezerosr
   r	   )r   hint
expected_c
zeros_maskzeros_inpaintbscthwhidden_states
orig_shapeexpected_incur_inrG   r   r   r   _process_hint_tokens=   sP   


 

	
z0QwenImageFunControlNetModel._process_hint_tokensguidancec	           !   	   K   s  |d u rt d|}
d }
||\}}}| |}|d u r!t d|jd |jd krTt|jd |jd }|d d d |f }|d d d |f }|d d d |f }tt|jd |jd  |j d |jd |jd  |j d }tj	|||jd  |j
dddd|jd dd	}tj||fdd
}|||j }||}||}||}|d ur|d }|d u r|||n||||}| |}t| jD ]A\}}|dkr||| }g }nttj|dd
}|d}||||
|||d\}}||| j }|||g7 }tj|dd
}qtj|dd
d d }d g| j  }t| j!D ]\}} |t"|k rF| t"|k rF|| || < q.d|iS )Nz?Qwen Fun ControlNet requires a QwenImage base model at runtime.z2Qwen Fun ControlNet requires a control hint image.r   r=   r7   r<   r
   r   r>   r:     )rU   encoder_hidden_statesencoder_hidden_states_masktembimage_rotary_embtransformer_optionsinput)#RuntimeErrorprocess_imgrY   rC   minroundmax
patch_sizer,   aranger
   rJ   repeatrE   pe_embeddertor	   
contiguousimg_intxt_normtxt_intime_text_embedr+   	enumerater/   r   listunbindpopr   r*   stackr'   r)   len)!r   x	timestepscontextattention_maskrZ   rL   ra   
base_modelkwargsr^   rU   img_ids_hint_tokens
max_tokens	txt_starttxt_idsidsr`   r]   r_   rQ   r5   blockc_inall_cc_outc_skiphintscontrolnet_block_samples	local_idxbase_idxr   r   r   forwardn   sp   
6





	z#QwenImageFunControlNetModel.forward)
r   r   r   r    r!   r"   r#   NNN)	r   r   r   r   rY   r,   Tensorr   r   r   r   r   r   r      s*    $6r   c                       sB   e Zd Z				d fdd	Zddddi fdejfddZ  ZS )	QwenImageControlNetModelr   Nc              	      s   t  jdd|||d| d| _tjg | _tt| j	D ]}| j
|j| j| j||d q|j| j| | j||d| _d S )NF)final_layerr	   r
   r   r"   r   r   )r   r   r'   r,   r-   r.   controlnet_blocksr0   rw   transformer_blocksr1   r   r3   in_channelscontrolnet_x_embedder)r   extra_condition_channelsr	   r
   r   r}   r   r   r   r   r      s     z!QwenImageControlNetModel.__init__rZ   c	                 K   s  |}
|}|}|  |\}}}|  |\}}}tt|jd | jd  | j d |jd | jd  | j d }tj|||jd  |jdddd	|jd dd}tj
||fdd}| ||j }~~~| || | }| |}| |}|d ur|d	 }|d u r| |
|n| |
||}t| jt| j }d
}t| jD ]\}}||||||d\}}|| j| |f|  }qd|d | j iS )Nr=   r7   r<   r   r[   r   r>   r:   r\   r   )rU   r]   r^   r_   r`   rb   )rd   rf   rg   rC   rh   r,   ri   r
   rJ   rj   rE   rk   rl   r	   rm   rn   r   ro   rp   rq   mathceilr'   rw   r   rr   r   )r   rx   ry   rz   r{   rZ   ref_latentsrL   ra   r}   timestepr]   r^   rU   r~   rV   r   r   r   r   r`   r_   rj   r   r5   r   r   r   r   r      s>   B6


z QwenImageControlNetModel.forward)r   NNN)r   r   r   r   r,   r   r   r   r   r   r   r   r      s    r   )
r,   r   modelr   r   r   r-   Moduler   r   r   r   r   r   <module>   s     .