o
    i5                     @   sv   d dl Z d dlmZ d dlZd dlm  mZ d dlZ	ddl
mZmZmZmZmZ G dd deZG dd deZdS )	    N)Optional   )WeightAdapterBaseWeightAdapterTrainBaseweight_decomposepad_tensor_to_shapetucker_weight_from_convc                       sH   e Zd Z fddZdd Zdejdejdejfdd	Zd
d Z  Z	S )LoraDiffc                    s  t    |\}}}}}}|jd |jd }}	|jd |jd }	}
|d ur:|jd }tjjtjjtjjf| }ntjj	}||	|dd| _
||
|	dd| _| j
jj| | jjj| |d urs|||	dd| _| jjj| nd | _|	| _tjjt|dd| _d S )Nr   r      F)bias)requires_grad)super__init__shapendimtorchnnConv1dConv2dConv3dLinearlora_up	lora_downweightdatacopy_lora_midrank	Parametertensoralpha)selfweightsmat1mat2r    mid
dora_scalereshapeout_dimr   in_dimconvdimlayer	__class__ 7/mnt/c/Users/fbmor/ComfyUI/comfy/weight_adapter/lora.pyr      s$   

zLoraDiff.__init__c                 C   sd   |j }| jd u r| jj| jj }nt| jj| jj| jj}| j| j }||||j	  }|
|S N)dtyper   r   r   r   r   r    r   r'   r   to)r!   w	org_dtypediffscaler   r.   r.   r/   __call__'   s   

zLoraDiff.__call__xbase_outreturnc                 C   s  | j | j t| dd }t| dd}t| dd}t| di }| jj}| jj}|rtjtjtj	f|d  }	|
 d	krht| d
d| }
t| dd}|durZ|j|jd |g|
R  }n|jg |jdg| R  }|
 d	kr||jg |jdg| R  }|	||fi |}| jdur| jj}|
 d	kr|jg |jdg| R  }|	||}|	||}|| S t||}| jdur| jj}t||}t||}|| S )a_  
        Additive bypass component for LoRA training: h(x) = up(down(x)) * scale

        Simple implementation using the nn.Module weights directly.
        No mid/dora/reshape branches (create_train doesn't create them).

        Args:
            x: Input tensor
            base_out: Output from base forward (unused, for API consistency)
        
multiplier      ?is_convFconv_dimr   kw_dictr   r
   kernel_sizer   in_channelsN)r    r   getattrr   r   r   Fconv1dconv2dconv3ddimviewr   r   linear)r!   r8   r9   r6   r=   r>   r?   down_weight	up_weightconv_fnr@   rB   hidden
mid_weightoutr.   r.   r/   h3   sL   





z
LoraDiff.hc                 C   s   t dd |  D S )Nc                 s   s     | ]}|  |  V  qd S r0   )numelelement_size).0paramr.   r.   r/   	<genexpr>|   s    z0LoraDiff.passive_memory_usage.<locals>.<genexpr>)sum
parametersr!   r.   r.   r/   passive_memory_usage{   s   zLoraDiff.passive_memory_usage)
__name__
__module____qualname__r   r7   r   TensorrQ   rZ   __classcell__r.   r.   r,   r/   r	      s
    Hr	   c                   @   s   e Zd ZdZdd ZedddZdd	 Ze	
ddede	ee
jf dede
jdee ded  fddZdd Ze
jd
fddZde
jde
jde
jfddZd
S )LoRAAdapterlorac                 C   s   || _ || _d S r0   )loaded_keysr"   )r!   rb   r"   r.   r.   r/   r      s   
zLoRAAdapter.__init__r   r<   c                 C   s~   |j d }|j dd   }tj|||jtjd}tj|||jtjd}tjjj|dd tjj	|d t
|||d d d fS )Nr   r   )devicer1   gw@)ag        )r   rR   r   emptyrc   float32r   initkaiming_uniform_	constant_r	   )clsr   r   r    r(   r)   r#   r$   r.   r.   r/   create_train   s   
zLoRAAdapter.create_trainc                 C   s
   t | jS r0   )r	   r"   rY   r.   r.   r/   to_train   s   
zLoRAAdapter.to_trainNr8   r    r&   rb   r:   c                 C   s  |d u rt  }d|}d|}d|}d|}	d|}
d|}d|}d|}d }|| v rD|}d	|}d
|}n_|| v rT|}d|}d }nO|	| v rd|	}d|}d }n?|
| v rt|
}d|}d }n/|| v r|}d|}d }n|| v r|}d|}d }n|| v r|}d|}d }|d urd }|d ur|| v r|| }|| d }|| v rz||  }|| W n   Y || || ||||f}|| || | ||S d S )Nz{}.reshape_weightz{}.lora_up.weightz{}_lora.up.weightz{}.lora_B.weightz{}.lora.up.weightz	{}.lora_Bz{}.lora_linear_layer.up.weightz{}.lora_B.default.weightz{}.lora_down.weightz{}.lora_mid.weightz{}_lora.down.weightz{}.lora_A.weightz{}.lora.down.weightz	{}.lora_Az {}.lora_linear_layer.down.weightz{}.lora_A.default.weight)setformatkeysaddtolist)rj   r8   ra   r    r&   rb   reshape_nameregular_loradiffusers_loradiffusers2_loradiffusers3_lora
mochi_loratransformers_loraqwen_default_loraA_nameB_namemid_namer%   r'   r"   r.   r.   r/   load   sp   	


















zLoRAAdapter.loadc                 C   s   | j d }|d urt|S d S )N   )r"   tuple)r!   keyr'   r.   r.   r/   calculate_shape   s   
zLoRAAdapter.calculate_shapec	              
   C   s  | j }	tj|	d |j|}
tj|	d |j|}|	d }|	d }|d ur*t||}|	d d ur:|	d |jd  }nd}|	d d ur|tj|	d |j|}|jd |jd |jd |jd g}t|	ddj
dd|	ddj
dd|	dd}z6t|
j
dd|j
dd|j}~
~|d urt|||||||}W |S |||| | |j7 }W |S  ty } ztd	| j|| W Y d }~|S d }~ww )
Nr   r      r~   r
   r<      )	start_dimzERROR {} {} {})r"   comfymodel_managementcast_to_devicerc   r   r   r   mm	transposeflattenr'   r   typer1   	Exceptionloggingerrorrn   name)r!   r   r   strengthstrength_modeloffsetfunctionintermediate_dtypeoriginal_weightvr#   r$   r&   r'   r    mat3final_shape	lora_differ.   r.   r/   calculate_weight   sf   
$ zLoRAAdapter.calculate_weightr9   c                 C   s  ddt jt jt jt jg}| j}|d }|d }|d }|d }|jd }	|dur-||	 }
nd}
|
t| dd }
|j|j	d}|j|j	d}t| d	d
}t| dd}t| di }|r||d  }t| dd| }t| dd}|
 dkr|dur|j|jd |g|R  }n|jg |jdg| R  }|
 dkr|jg |jdg| R  }|dur|j|j	d}|
 dkr|jg |jdg| R  }nt j}i }|dur|s|j|j	d}|||}|||fi |}|||}||
 S |||fi |}|||}||
 S )a  
        Additive bypass component for LoRA: h(x) = up(down(x)) * scale

        Note:
            Does not access original model weights - bypass mode is designed
            for quantized models where weights may not be accessible.

        Args:
            x: Input tensor
            base_out: Output from base forward (unused, for API consistency)

        Reference: LyCORIS functional/locon.py bypass_forward_diff
        Nr   r   r
   r   r<   r;   )r1   r=   Fr>   r?   r@   rA   rB   )rD   rJ   rE   rF   rG   r"   r   rC   r2   r1   rH   rI   )r!   r8   r9   	FUNC_LISTr   updownr    r%   r   r6   r=   r>   r?   opr@   rB   rN   rP   r.   r.   r/   rQ      sZ   




zLoRAAdapter.h)r   r<   r0   )r[   r\   r]   r   r   classmethodrk   rl   strdictr   r^   floatrm   r   r}   r   rf   r   rQ   r.   r.   r.   r/   r`      s4    	E
 @r`   )r   typingr   r   torch.nn.functionalr   
functionalrD   comfy.model_managementr   baser   r   r   r   r   r	   r`   r.   r.   r.   r/   <module>   s    	o