o
    i>?                     @   sr   d dl Z d dlmZ d dlZd dlm  mZ d dlZ	ddl
mZmZmZmZ G dd deZG dd deZdS )	    N)Optional   )WeightAdapterBaseWeightAdapterTrainBaseweight_decomposefactorizationc                       s`   e Zd Z fddZedd Zedd Zdd Zd	ej	d
ej	dej	fddZ
dd Z  ZS )LokrDiffc              	      sN  t    |\	}}}}}}}}	}
d| _|d urA|jd |jd }}|jd |jd }}tj|| _tj|| _d| _	|| _
|d ur}|jd |jd }}|jd |jd }}tj|| _tj|| _|	d urwd| _tj|	| _d| _|| _|d urtj|| _d| _	|d urtj|| _d| _tjjt|dd| _d S )NFr   r   T)requires_grad)super__init__
use_tuckershapetorchnn	Parameter	lokr_w1_a	lokr_w1_b
w1_rebuildranka	lokr_w2_a	lokr_w2_blokr_t2
w2_rebuildrankblokr_w1lokr_w2tensoralpha)selfweightsr   r   r   r   r   r   r   r   
dora_scale_rank_arank_b	__class__ 7/mnt/c/Users/fbmor/ComfyUI/comfy/weight_adapter/lokr.pyr      sJ   
zLokrDiff.__init__c                 C   s$   | j r| j| j | j| j  S | jS N)r   r   r   r   r   r   r   r&   r&   r'   w1;   s   zLokrDiff.w1c                 C   sF   | j r | jrtd| j| j| j}n| j| j }|| j| j  S | j	S )Ni j k l, j r, i p -> p r k l)
r   r   r   einsumr   r   r   r   r   r   )r   w2r&   r&   r'   r-   B   s   zLokrDiff.w2c                 C   sR   | j }| j}t| |  D ]}|d}qt||}|||j	| S )N)
r*   r-   rangedim	unsqueezer   kronreshaper   to)r   wr*   r-   r!   diffr&   r&   r'   __call__R   s   zLokrDiff.__call__xbase_outreturnc                 C   s  | j }| j}t| dd}t| dd}t| dd}t| di }|d}	|rtjtjtjf|d  }
|j^}}}|j	||	 d	g|R  }|
 d
krW|jg |jdg| R  }|
||fi |}|j|d	g|jdd R  }|dd	}t||}|dd	}|j	|d	g|jdd R  }|| S |j	g |jdd	 |	d	R  }t||}|d	d}t||}|d	d}|j	g |jdd d	R  }|| S )a  
        Additive bypass component for LoKr training: efficient Kronecker product.

        Uses w1/w2 properties which handle both direct and decomposed cases.
        For create_train (direct w1/w2), no alpha scaling in properties.
        For to_train (decomposed), alpha/rank scaling is in properties.

        Args:
            x: Input tensor
            base_out: Output from base forward (unused, for API consistency)
        
multiplier      ?is_convFconv_dimr   kw_dictr   r.      N   )r*   r-   getattrsizeFconv1dconv2dconv3dr   r3   r0   view	transposelinear)r   r8   r9   r*   r-   r;   r=   r>   r?   uqconv_fnBC_inspatial
h_in_grouphbh_crosshcoutr&   r&   r'   h[   s6   
"z
LokrDiff.hc                 C   s   t dd |  D S )Nc                 s   s     | ]}|  |  V  qd S r(   )numelelement_size).0paramr&   r&   r'   	<genexpr>   s    z0LokrDiff.passive_memory_usage.<locals>.<genexpr>)sum
parametersr)   r&   r&   r'   passive_memory_usage   s   zLokrDiff.passive_memory_usage)__name__
__module____qualname__r   propertyr*   r-   r7   r   TensorrV   r^   __classcell__r&   r&   r$   r'   r      s    +

	Kr   c                   @   s   e Zd ZdZdd ZedddZdd	 Ze	
ddede	ee
jf dede
jdee ded  fddZe
jd
fddZde
jde
jde
jfddZd
S )LoKrAdapterlokrc                 C   s   || _ || _d S r(   )loaded_keysr   )r   rg   r   r&   r&   r'   r      s   
zLoKrAdapter.__init__r   r<   c              
   C   s   |j d }|j d }| dkr|j dd  nd}t||\}}t||\}	}
tj||	|jtjd}tj||
g|R |jtjd}tjjj	|dd tjj
|d t|||d d d d d d f	S )	Nr   r   r@   r&   )devicedtypegw@)ag        )r   r0   r   r   emptyrh   float32r   initkaiming_uniform_	constant_r   )clsweightrankr   out_dimin_dimk_sizeout_lout_kin_min_nmat1mat2r&   r&   r'   create_train   s    

zLoKrAdapter.create_trainc                 C   s
   t | jS r(   )r   r   r)   r&   r&   r'   to_train   s   
zLoKrAdapter.to_trainNr8   lorar   r    rg   r:   c              	   C   s  |d u rt  }d|}d|}d|}d|}	d|}
d|}d|}d }|| v r;|| }|| d }|| v rL|| }|| d }|| v r]|| }|| d }|	| v rn||	 }||	 d }|| v r|| }|| d }|| v r|| }|| d }|
| v r||
 }||
 |d us|d us|d us|d ur|||||||||f	}| ||S d S )Nz
{}.lokr_w1z
{}.lokr_w2z{}.lokr_w1_az{}.lokr_w1_bz
{}.lokr_t2z{}.lokr_w2_az{}.lokr_w2_b)setformatkeysadd)rp   r8   r~   r   r    rg   lokr_w1_namelokr_w2_namelokr_w1_a_namelokr_w1_b_namelokr_t2_namelokr_w2_a_namelokr_w2_b_namer   r   r   r   r   r   r   r   r&   r&   r'   load   sj   	














zLoKrAdapter.loadc	              
   C   s  | j }	|	d }
|	d }|	d }|	d }|	d }|	d }|	d }|	d }d }|
d u rC|jd }ttj||j|tj||j|}
n	tj|
|j|}
|d u r|jd }|d u rnttj||j|tj||j|}n'td	tj||j|tj||j|tj||j|}n	tj||j|}t	|jdkr|

d

d
}
|	d
 d ur|d ur|	d
 | }nd}z,t|
||j}|d urt|||||||}W |S |||| | |j7 }W |S  ty } ztd| j|| W Y d }~|S d }~ww )Nr   r   rA                  r+   r@   r<   zERROR {} {} {})r   r   r   mmcomfymodel_managementcast_to_devicerh   r,   lenr1   r2   r3   r   typeri   	Exceptionloggingerrorr   name)r   rq   keystrengthstrength_modeloffsetfunctionintermediate_dtypeoriginal_weightvr*   r-   w1_aw1_bw2_aw2_bt2r    r0   r   	lora_differ&   r&   r'   calculate_weight  s   
	
	 zLoKrAdapter.calculate_weightr9   c           &      C   s  ddt jt jt jt jg}| j}|d }|d }|d }|d }|d }	|d }
|d }|d	 }|du}|du}|du}t| d
d}t| dd}|rOt| di ni }|rZ||d  }nt j}|sd|	dn|sk|dn|}|duru|| ndt| dd }|r|j|j	d}n|j|j	d|	j|j	d }|d}|r|j|j	d}nO|j|j	d}|
j|j	d}|r|r|
 dkr|jg |jdg| R  }|
 dkr|jg |jdg| R  }n|
 dkr|jg |jdg| R  }|r	|j^}}}|j|| dg|R  }n|jg |jdd |dR  }|r'|||fi |}nO|rl|r]|j|j	d} | 
 dkrI| jg | jdg| R  } |||}!||!| fi |}"||"|}n|||fi |}!||!|}n
|||}!||!|}|r|j|dg|jdd R  }|dd}#n|dd}#t |#|}$|r|$dd}$|$j|dg|$jdd R  }%|%| S |$dd}$|$jg |$jdd dR  }%|%| S )a  
        Additive bypass component for LoKr: efficient Kronecker product application.

        Note:
            Does not access original model weights - bypass mode is designed
            for quantized models where weights may not be accessible.

        Args:
            x: Input tensor
            base_out: Output from base forward (unused, for API consistency)

        Reference: LyCORIS functional/lokr.py bypass_forward_diff
        Nr   r   r@   rA   r   r   r   r   r=   Fr>   r?   r<   r;   )ri   r.   rB   )rE   rK   rF   rG   rH   r   rC   rD   r4   ri   r0   rI   r   r3   rJ   )&r   r8   r9   	FUNC_LISTr   r*   r-   r   r   r   r   r   r   use_w1use_w2tuckerr=   r>   r?   oprr   scalecrL   barj   brN   r!   restrQ   rR   thahth_cross_grouprT   rU   r&   r&   r'   rV   n  s    
"


zLoKrAdapter.h)r   r<   r(   )r_   r`   ra   r   r   classmethodr|   r}   strdictr   rc   floatr   r   r   rl   r   rV   r&   r&   r&   r'   re      s2    R
 [re   )r   typingr   r   torch.nn.functionalr   
functionalrE   comfy.model_managementr   baser   r   r   r   r   re   r&   r&   r&   r'   <module>   s     