o
i7 @ s d Z ddlmZ ddlZddlZddlZddlZddlZddlm Z m
Z
ddl Z ddl Z
ddlZ
ddlZ
ddlZ
ddlZ
ddlZ
ddlmZ ddlmZ ddlmZmZmZ ddlZd2dd Z d3d d Z d3d
dZ d4ddZ dd Z dd Z!dd Z"dd Z#G dd d Z$d Z%d d Z&d d! Z'd"d# Z(G d$d% d%Z)G d&d' d'Z*e+d(d)d*gd+dZ,G d,d- d-e j-j.Z/G d.d/ d/Z0G d0d1 d1e0Z1e0Z2dS )5a
This file is part of ComfyUI.
Copyright (C) 2024 Comfy
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
) annotationsN)CallableOptional)UnetWrapperFunction)QuantizedTensor) CallbacksMPPatcherInjection
WrappersMPc C s | d }d|vri |d< n|d |d< ||d vr$i |d |< n |d | |d |< |d ur:|||f}n||f}||d | |< || d< | S )Ntransformer_optionspatches_replacecopy)
model_optionspatchname
block_namenumbertransformer_indextoblock r 1/mnt/c/Users/fbmor/ComfyUI/comfy/model_patcher.py set_model_options_patch_replace* s
r Fc C & | dg |g | d< |rd| d< | S )Nsampler_post_cfg_functionTdisable_cfg1_optimizationget)r post_cfg_functionr r r r #set_model_options_post_cfg_function? r c C r )Nsampler_pre_cfg_functionTr r )r pre_cfg_functionr r r r "set_model_options_pre_cfg_functionE r r# orig_model_optionsdictc C s t j| S N)comfypatcher_extensioncopy_nested_dicts)r$ r r r create_model_options_cloneK r* c C sD i }| D ]}i ||< | | D ]}| | | d d || |< qq|S r& r )orig_hook_patchesnew_hook_patcheshook_refkr r r create_hook_patches_cloneN s r0 c C s> t | dr | j| _| `t | drg | _t | dr g | _d S d S )Nprev_comfy_cast_weightsweight_function
bias_function)hasattrr1 comfy_cast_weightsr2 r3 )mr r r wipe_lowvram_weightV s
r7 c C st |d u rdS d}t | dr | jD ]}t |dr ||j|d7 }qt | dr8| jD ]}t |dr7||j|d7 }q(|S )Nr r2 move_todevicer3 )r4 r2 r8 r3 )r6 r: memoryfr r r move_weight_functionsa s
r= c C s t d tj| S )NzIWARNING: string_to_seed has moved from comfy.model_patcher to comfy.utils)loggingwarningr' utilsstring_to_seed)datar r r rA q s
rA c @ s e Zd ZdddZdd ZdS ) LowVramPatchNc C s || _ || _|| _|| _d S r& )keypatches convert_funcset_func)selfrD rE rF rG r r r __init__v
zLowVramPatch.__init__c C s t jj| j| j || j|jdS )Nintermediate_dtype)r' loracalculate_weightrE rD dtype)rH weightr r r __call__| s zLowVramPatch.__call__NN)__name__
__module__ __qualname__rI rQ r r r r rC u s
rC c C sJ t | |\}}}|d u rdS t| dtj}|d u r |j}| |j t S )Nr manual_cast_dtype)get_key_weightgetattrtorchfloat32rO numelitemsize"LOWVRAM_PATCH_ESTIMATE_MATH_FACTOR)modelrD rP rG rF model_dtyper r r low_vram_patch_estimate_vram s ra c C s d }d }| dd}t|dk rtj| |}nGtj| |d }z t|d|d }W n ty6 Y nw z t|d|d }W n tyL Y nw t||d }|d ur_tj| |}|||fS )N. rV r zset_{}z
convert_{})rsplitlenr' r@ get_attrrY formatAttributeError)r_ rD rG rF op_keysrP opr r r rX s(
rX c C s t | dkr|S d| |S )Nr z{}.{})re rg )rD paramr r r key_param_name_to_key s rl c @ s( e Zd Zd d ddZdd Zdd Zd
S )
AutoPatcherEjectorFr_ 'ModelPatcher'c C s || _ d| _d| _|| _d S NF)r_ was_injectedprev_skip_injection skip_and_inject_on_exit_only)rH r_ rr r r r rI rJ zAutoPatcherEjector.__init__c C s> d| _ | jj| _| jrd| j_| jjr | j d| _ d S d S )NFT)rp r_ skip_injectionrq rr is_injected eject_modelrH r r r __enter__ s
z AutoPatcherEjector.__enter__c G s@ | j r
| j| j_| j | jr| jjs| j | j| j_d S r& )rr rq r_ rs inject_modelrp )rH argsr r r __exit__ s
zAutoPatcherEjector.__exit__NF)r_ rn )rS rT rU rI rw rz r r r r rm s rm c @ s6 e Zd ZddddZddd Zdd d Zdd
dZdS )
MemoryCounterr initialintc C || _ || _d S r& valueminimum)rH r} r r r r rI
zMemoryCounter.__init__rP torch.Tensorc C s, | | }| |r| | dS dS )NTF)nelement element_size
is_useable decrement)rH rP weight_sizer r r use s
zMemoryCounter.useusedc C s | j | | jkS r& r rH r r r r r zMemoryCounter.is_useablec C s | j |8 _ d S r& )r r r r r r zMemoryCounter.decrementN)r )r} r~ )rP r )r r~ )rS rT rU rI r r r r r r r r| s
r|
FakeDevicetypeindexzcomfy-lazy-casterc s8 e Zd Z fddZdd Zedd Zdd Z ZS ) LazyCastingParamc s t | |S r& )super__new__)clsr_ rD tensor __class__r r r zLazyCastingParam.__new__c C r r& )r_ rD )rH r_ rD r r r r rI r zLazyCastingParam.__init__c C s t S r& )CustomTorchDevicerv r r r r: zLazyCastingParam.devicec O s | j j| j| j jdddS )NT) device_to
return_weightcpu)r_ patch_weight_to_devicerD load_devicer )rH ry kwargsr r r r s zLazyCastingParam.to) rS rT rU r rI propertyr: r
__classcell__r r r r r s
r c @ sD e Zd Zd ddZdd Zdd Zdd d
Zd d Zd
d Zdd Z dd Z
dddZ dd Z dddZ
d d Zd d Zdd d!Zdd"d#Zdd$d%Zd&d' Zdd*d+Zd,d- Zd.d/ Zdd0d1Zd2d3 Zd4d5 Zdd6d7Zdd8d9Zd:d; Z dd? Z d@dA Z dBdC Z dDdE Z!dFdG Z"dHdI Z#dJdK Z$dLdM Z%dNdO Z&dPdQ Z'dRdS Z(dTdU Z)ddZd[Z*d\d] Z+d^d_ Z,d`i fdadbZ-dcdd Z.ddfdgZ/ddhdiZ0ddjdkZ1ddldmZ2dndo Z3dpdq Z4drds Z5ddtduZ6d dvdwZ7d
dydzZ8d d{d|Z9d d}d~Z:d ddZ;dd Zdd Z?e@jAfddZBdd ZCd
ddZDdddZEdddZFdddZGdddZHdddZIdddZJdddZKdddZLdddZMdddZNdddZOdddZPdddZQdddZRdddZSdddZTdddZUdddZVdd ZWdd ZXdddZYddĄ ZZddƄ Z[ddȄ Z\ddʄ Z]dd̄ Z^dddЄZ_dddZ` ddddބZadd ddZbd ddZcd d ddZdd ddZed ddZfdd Zgd!ddZhdd"ddZidd Zjd#ddZkdd ZldS ($ ModelPatcherr Fc C sx || _ || _t| jdstd || j_n
| jjd u r || j_i | _i | _i | _i | _ i | _
i | _ di i| _ || _
|| _|| _d| _t | _d | _t | _i | _i | _t | _t | _ d| _ d| _ i | _!i | _"d | _#i | _$i | _%d | _&d | _'d| _(t)j*j+j,| _-d | _.t| jdsd| j_/t| jdsd| j_0t| jdsd| j_1t| jd sd | j_2t| jd
sd| j_3d S d S ) Nr: z&Model doesn't have a device attribute.r
Fmodel_loaded_weight_memoryr lowvram_patch_counter
model_lowvramcurrent_weight_patches_uuidmodel_offload_buffer_memory)4sizer_ r4 r> debugr: rE backupbackup_buffersobject_patchesobject_patches_backupweight_wrapper_patchesr r offload_deviceweight_inplace_updateforce_cast_weightsuuiduuid4 patches_uuidparentsetpinned attachmentsadditional_modelsr init_callbacks callbacksr
init_wrapperswrappersrt rs
injections hook_patcheshook_patches_backup hook_backupcached_hook_patches
current_hooks forced_hooksis_clipr' hooks EnumHookModeMaxSpeed hook_modecached_patcher_initr r r r r rH r_ r r r r r r r rI s`
zModelPatcher.__init__c C dS ro r rv r r r
is_dynamic# zModelPatcher.is_dynamicc C s& | j dkr| j S tj| j| _ | j S Nr )r r' model_management module_sizer_ rv r r r
model_size& s
zModelPatcher.model_sizec C s t jj| j|dS )N)free)r' r module_mmap_residencyr_ )rH r r r r model_mmap_residency, r z!ModelPatcher.model_mmap_residencyc C | j jS r& )r_ r rv r r r loaded_size/ zModelPatcher.loaded_sizec C r r& )r_ r rv r r r r 2 r z"ModelPatcher.lowvram_patch_counterc C s& d}t jjr tj }t j|| S r )r' memory_management
aimdo_enabled comfy_aimdo
model_vbar
vbars_analyzer get_free_memory)rH r: aimdo_memr r r r 5 s
z ModelPatcher.get_free_memoryc C s | j | j| j| j| jffS r& )r_ r r r r rv r r r get_clone_model_override> s z%ModelPatcher.get_clone_model_overrideNc C s | j }| r)|r)t}|d u r)| jd u rtd| jd | jd ddi}| }|d u r1| }||d | j| j| | j d}i |_
| j
D ]}| j
| d d |j
|< qG| j |_ | j
|_ | j
|_tj| j|_| |_| j|_|d \|_|_|_|_i |_| jD ] }t| j| dr| j| |j|< q| j| |j|< q| j D ]\}}dd |D |j |< q| j D ]\}}i |j |< | D ]
\}} |
|j | |< qq| j D ]\}}
i |j |< |
D ]
\}} |
|j | |< qq| j |_ | j!|_!| j" D ] \}} |
|j"|< qt#| j$|_$| j%r$t#| j%n| j%|_%| j&D ] }
i |j&|
< | j&|
D ]}| j&|
| |j&|
|< q7q+| j'|_'| j(rV| j() n| j(|_(| j*rc| j*) n| j*|_*| j+|_+| j,|_,| j|_| -t.j/D ]}|| | qy|S )
NzKCannot create non-dynamic delegate: cached_patcher_init is not initialized.r rc disable_dynamicT)r on_model_patcher_clonec S s g | ]}| qS r )clone).0xr r r
e z&ModelPatcher.clone..)0r r r r RuntimeErrorr r r r r rE r r r
r r' r@ deepcopy_list_dictr r r r r r r r r4 r r itemsr r rt rs r r0 r r r r r r r r r get_all_callbacksr ON_CLONE)rH r model_overrideclass_temp_model_patchernr/ ck1c1ww1igroupcallbackr r r r A st
zModelPatcher.clonec C s t |dr
| j|ju r
dS dS )Nr_ TF)r4 r_ )rH otherr r r is_clone s zModelPatcher.is_cloner rn c C s@ | |sdS | j|jkrdS | j|jkrdS | j |j kr#dS | j |j kr/dS | j |j kr;dS | jD ]}t| j| t|j| krQ dS q>| j D ]}t| j | t|j | krh dS qU| j
|j
krudS t| j dkrt|j dkrdS | j |j krt| j t|j krt
d d S dS d S )NFr TzOWARNING: something went wrong, same patch uuid but different length of patches.)r r r r keysr r r re r r rE r r> r? )rH r rD r r r clone_has_same_weights s:
z#ModelPatcher.clone_has_same_weightsc C s | j j|dS )N input_shape)r_ memory_requiredrH r r r r r r z ModelPatcher.memory_requiredc C s d| j d< d S )NTr r rv r r r disable_model_cfg1_optimization r z,ModelPatcher.disable_model_cfg1_optimizationc sF t t jdkr fdd| jd< n | jd< |r!| d S d S )N c s | d | d | d S )Nconduncond
cond_scaler )ry sampler_cfg_functionr r s z=ModelPatcher.set_model_sampler_cfg_function..r
)re inspect signature
parametersr r )rH r
r r r r set_model_sampler_cfg_function s
z+ModelPatcher.set_model_sampler_cfg_functionc C t | j||| _d S r& )r r )rH r r r r r #set_model_sampler_post_cfg_function z0ModelPatcher.set_model_sampler_post_cfg_functionc C r r& )r# r )rH r" r r r r "set_model_sampler_pre_cfg_function r z/ModelPatcher.set_model_sampler_pre_cfg_functionc C || j d< d S )N sampler_calc_cond_batch_functionr )rH r r r r *set_model_sampler_calc_cond_batch_function r z7ModelPatcher.set_model_sampler_calc_cond_batch_functionunet_wrapper_functionr c C r )Nmodel_function_wrapperr )rH r r r r set_model_unet_function_wrapper r z,ModelPatcher.set_model_unet_function_wrapperc C r )Ndenoise_mask_functionr )rH r r r r set_model_denoise_mask_function r z,ModelPatcher.set_model_denoise_mask_functionc C s< | j d }d|vr
i |d< |d |g |g |d |< d S )Nr
rE r r )rH r r r r r r set_model_patch s
"z ModelPatcher.set_model_patchc C s t | j|||||d| _d S )N)r )r r )rH r r r r r r r r set_model_patch_replace s z$ModelPatcher.set_model_patch_replacec C | |d d S )N attn1_patchr rH r r r r set_model_attn1_patch r z"ModelPatcher.set_model_attn1_patchc C r )N attn2_patchr! r" r r r set_model_attn2_patch r z"ModelPatcher.set_model_attn2_patchc C | |d||| d S )Nattn1r rH r r r r r r r set_model_attn1_replace z$ModelPatcher.set_model_attn1_replacec C r& )Nattn2r( r) r r r set_model_attn2_replace r+ z$ModelPatcher.set_model_attn2_replacec C r )Nattn1_output_patchr! r" r r r set_model_attn1_output_patch r z)ModelPatcher.set_model_attn1_output_patchc C r )Nattn2_output_patchr! r" r r r set_model_attn2_output_patch r z)ModelPatcher.set_model_attn2_output_patchc C r )Ninput_block_patchr! r" r r r set_model_input_block_patch r z(ModelPatcher.set_model_input_block_patchc C r )N input_block_patch_after_skipr! r" r r r &set_model_input_block_patch_after_skip r z3ModelPatcher.set_model_input_block_patch_after_skipc C r )Noutput_block_patchr! r" r r r set_model_output_block_patch r z)ModelPatcher.set_model_output_block_patchc C r )N emb_patchr! r" r r r set_model_emb_patch r z ModelPatcher.set_model_emb_patchc C r )N forward_timestep_embed_patchr! r" r r r &set_model_forward_timestep_embed_patch r z3ModelPatcher.set_model_forward_timestep_embed_patchc C r )N double_blockr! r" r r r set_model_double_block_patch r z)ModelPatcher.set_model_double_block_patchc C r )N
post_inputr! r" r r r set_model_post_input_patch r z'ModelPatcher.set_model_post_input_patchc C r )N
noise_refinerr! r" r r r set_model_noise_refiner_patch r z*ModelPatcher.set_model_noise_refiner_patchc K sT | j d di }||d< ||d< ||d< ||d< ||d< ||d< || j d d< d S ) Nr
rope_optionsscale_xscale_yscale_tshift_xshift_yshift_tr ) rH rC rF rD rG rE rH r rB r r r set_model_rope_options s z#ModelPatcher.set_model_rope_optionsc C || j |< d S r& )r )rH r objr r r add_object_patch
r z ModelPatcher.add_object_patchc C s( | d| |d ur
d| _t | _d S )NrW T)rL r r r r )rH rO r r r set_model_compute_dtype
s z$ModelPatcher.set_model_compute_dtypec C s( | j |g |g | j |< t | _d S r& )r r r r r )rH r functionr r r add_weight_wrapper s z ModelPatcher.add_weight_wrapperr strreturntorch.nn.Modulec C s8 || j v r
| j | S || jv r| j| S tj| j|S )a Retrieves a nested attribute from an object using dot notation considering
object patches.
Args:
name (str): The attribute path using dot notation (e.g. "model.layer.weight")
Returns:
The value of the requested attribute
Example:
patcher = ModelPatcher()
weight = patcher.get_model_object("layer1.conv.weight")
)r r r' r@ rf r_ )rH r r r r get_model_object s
z ModelPatcher.get_model_objectc C s | j d }d|v r/|d }|D ] }|| }tt|D ]}t|| dr-|| |||< qqd|v rU|d }|D ]}|| }|D ]}t|| drS|| |||< qAq9d| j v rn| j d }t|drp||| j d< d S d S d S )Nr
rE r r r )r rangere r4 r ) rH r: r rE r
patch_listr r/ wrap_funcr r r model_patches_to- s2
z ModelPatcher.model_patches_toc C s | j d }g }d|v r0|d }|D ] }|| }tt|D ]}t|| dr.||| 7 }q qd|v rU|d }|D ]}|| }|D ]}t|| drS||| 7 }qBq:d| j v rj| j d }t|drj|| 7 }|S )Nr
rE modelsr r )r rT re r4 rX ) rH r rX rE r rU r r/ rV r r r model_patches_modelsB s2
z!ModelPatcher.model_patches_modelscleanupc
C s | j d }d|v r2|d }|D ]"}|| }tt|D ]}t|| |r0t|| |di | qqd|v r[|d }|D ] }|| }|D ]}t|| |rYt|| |di | qDq.) model_state_dict
startswithr r r rX r_ rP rE )
rH
filter_prefixrj ri r/ bkhbkrP rG rF r r r get_key_patches s&
z ModelPatcher.get_key_patchesc C sl | ( | j }t| }|d ur#|D ] }||s"|| q|W d S 1 s/w Y d S r& )ra r_ rb rf r rp pop)rH rq sdr r/ r r r ro s
$z ModelPatcher.model_state_dictc C s6 t | j|\}}}|| jvr|S | jp|}|| jvr0|s0tdddg|j| j|d|| j|< t j
|}|d urEt j
j |||dd} n|j|dd} |d urV|| dd} t j
| j| | |}
|d u rt jj|
|jt j|d}
|rw|
S |rt j| j||
d S t j| j||
d S ||
|t j||d S )
N DimensionrP inplace_updater: r
Tr inplaceseed)rx r} r )rX r_ rE r r collections
namedtupler r r' r lora_compute_dtypecast_to_devicerM rN floatstochastic_roundingrO r@ rA
copy_to_paramset_attr_param) rH rD r rx r rP rG rF
temp_dtype temp_weight
out_weightr r r r s*
( z#ModelPatcher.patch_weight_to_devicec C s2 t | j|\}}}tj|r| j| d S d S r& )rX r_ r' r
pin_memoryr rd rH rD rP rG rF r r r pin_weight_to_device s z!ModelPatcher.pin_weight_to_devicec C s< || j v r t| j|\}}}tj| | j | d S d S r& )r rX r_ r' r unpin_memoryremover r r r unpin_weight s
zModelPatcher.unpin_weightc C s t | jD ]}| | qd S r& )rf r r rH rD r r r unpin_all_weights s z ModelPatcher.unpin_all_weightsc s, g } j D ]\}}d}dd |jddD }|jddD ] \}} ||vr*d} nq |rI|d urI| D ]\}
} | jj|t||
d d d| _q5|st|dsVt|d krt j
|} | } t|dr{ fd
d }
| |
d |7 } | |
d
|7 } |r| dk| f}n| f}|
|| |||f q|S )NFc S s i | ]\}}||qS r r )r r rk r r r
s z+ModelPatcher._load_list..recurseT_comfy_model_dtype)r: rO r5 r c sp | j v r t j| S t jdd }t j| \}}}|d u s#|d u r%dS |j|ks/t|tr6| |j S dS )NrW r )
rE ra r_ rY rX rO rc r r\ r] )rD r` rP _rv r r check_module_offload_mem s
z9ModelPatcher._load_list..check_module_offload_mem {}.weight{}.biasi )r_
named_modulesnamed_parametersr rB r rY r4 re r' r r rg re )rH for_dynamicdefault_deviceloadingr r6 defaultparamsr rk
param_name
module_memmodule_offload_memr
sort_criteriar rv r
_load_list s2
zModelPatcher._load_listc ! C s | 1 | d}d}d}d}| } g }
g } d} | jdd t| D ]\}
}|\}}}}}d}t| |tdd | |
d |
d tjj D }|| | |k }d
|}d
|}|s{t |d
r{|s{|} d}|d7 }||7 }t |d r{q&| j }| j |_
|rt |d
rg |_g |_|| jv r|r| | nt| j|\}}} t|| j| |g|_|d7 }|| jv r|r| | nt| j|\}}} t|| j| |g|_|d7 }d}| ||||f n t |d
rt| |s|r||7 }|
||||f n|} |rt |d
r|j|_d|_|| jv r |j| j| || jv r.|j| j| |t||7 }q&|
jdd |
D ]L}|d }|d }|d
}t |drZ|j dkrZq>|D ]} t || } | | | j| |d q\tj |r}t j!" t#$d
|| d|_ q>|
D ]
}|d %| q| D ]}|d }|d
}|D ] } | &t || qq|dk rd
|d nd} |dkrt#'d
| |d |d | d | d| j_(n t#'d
| |d | d| j_(|r| j%| | ) }| j j*|7 _*|| j_+|| j_,| | j_-| j.| j_/| 0t1j2D ] } | | |||| q| j3| j4dd W d d S 1 s:w Y d S )Nr T)reverseFc S s g | ]}|d qS )rc r )r x1r r r r r z%ModelPatcher.load..rc r r r5 r1 rV r comfy_patched_weightsr z&lowvram: loaded module regularly {} {}nFz{:.2f} MB usable, zjloaded partially; {} {:.2f} MB loaded, {:.2f} MB offloaded, {:.2f} MB buffer reserved, lowvram patches: {}z5loaded completely; {} {:.2f} MB loaded, full load: {} force_apply)5ra
unpatch_hooksr sort enumeratemaxsumr' r NUM_STREAMSrg r4 r comfy_force_cast_weightsr2 r3 rE r rX r_ rC re r7 r5 r1 r extendr= r rl r is_device_cudarZ cuda synchronizer> r r r infor r r r: r r r r r r ON_LOAD apply_hooksr )!rH r lowvram_model_memoryforce_patch_weights full_load mem_counter
patch_counterlowvram_counterlowvram_mem_counterr load_completely offloadedoffload_bufferr r r r r r6 r lowvram_weightpotential_offload lowvram_fits
weight_keybias_key cast_weightr rG rF rk rD usable_statr r r r load s 4
$
$zModelPatcher.loadTc C s | 8 | jD ]}tj| j|| j| }|| jvr || j|< q|dkr(d}nd}|r5| j||||d W d n1 s?w Y | | jS )Nr TFr r r ) ra r r' r@ set_attrr_ r r rx )rH r r load_weightsr r/ oldr r r r patch_model s
zModelPatcher.patch_modelc C sB | |r| | | jjr+| j D ] }t|| t| qd| j_d| j_t | j
}|D ] }| j
| }|j rIt
j| j||j q4t
j| j||j q4d | j_| j
|d urk| j| || j_d| j_d| j_| j D ] }t|dr|`qxt | j }|D ]}t
j| j|| j| q| j d S )NFr r ) ru r r r_ r modulesr= r7 r rf r r rx r' r@ r rP r r clearr r: r r r4 r r r )rH r unpatch_weightsr6 r r/ rr r r r
unpatch_model s>
zModelPatcher.unpatch_modelc C s" | d}d}d}| }| | jj}t|dkr2tjj} t || d |d d g| }
|D ]} || | jj |k rD n | \} }
}}}| t
|
}t |d}t |drO|j dkrOd}|D ]>}t
||}| j|d }|d ur|s~d} n'|s| d}|jrtj| j||j n
tj| j||j | j| qfd|}d|}|rO| j}|| |
t||7 }
|r|| jv r|r| | nt | j|\}}} |j t || j| | |d7 }|| jv r|r| | nt | j|\}}} |j! t || j| | |d7 }d}|r#t |dr#|j"|_#d|_"d|_ ||
7 }t$||}|
|
|
d t%&d | |D ] }| 't
|| qCq4d| j_(| j j)|7 _)| j j*|8 _*|| j_t%+d
|d | jj*d |d | jj) |W d S 1 sw Y d S ) NFr rc r5 r Tr r zfreed {}zmUnloaded partially: {:.2f} MB freed, {:.2f} MB remains loaded, {:.2f} MB buffer reserved, lowvram patches: {}r ),ra r r r_ r re r' r r minr r4 r rl r r r rx r@ r rP r ru rg r r r= rE r rX r2 re rC r3 r5 r1 r r> r r r r r r ) rH r memory_to_freer hooks_unpatched memory_freedr unload_listr NSoffload_weight_factorunloadr r r r6 r r lowvram_possible move_weightrk rD rr r r r r rG rF r r r partially_unload s
*&z ModelPatcher.partially_unloadc C sb | j dd | jjd uo| jj| jkp|}| jj}| j| j|d |r,||| jj 7 }| jdd |dk rK|sK| j| j| |d W d dS d}| jj dkrj| jjdkrj| j
| j dd W d dS | jj| | krvd}| jj}z
| j
||| ||d W n ty } z| |d }~ww | jj| W d S 1 sw Y d S )
NTrr r Fr r )r r r )ra r_ r r r r r r r r r r r r Exceptiondetach) rH r extra_memoryr r r r current_useder r r partially_load s8
$zModelPatcher.partially_loadc C r r r rv r r r pinned_memory_size* r z ModelPatcher.pinned_memory_sizec C d S r& r )rH
ram_to_unloadr r r partially_unload_ram. r z!ModelPatcher.partially_unload_ramc C sJ | | | j |r| j| j|d | tjD ]}|| | q| jS )Nr )ru rW r r r r ON_DETACHr_ )rH unpatch_allr r r r r 1 s zModelPatcher.detachc C r r& )r_ r: rv r r r current_loaded_device: r z"ModelPatcher.current_loaded_devicec C s t d tjj||||dS )NziThe ModelPatcher.calculate_weight function is deprecated, please use: comfy.lora.calculate_weight insteadrK )r> r? r' rM rN )rH rE rP rD rL r r r rN = s
z ModelPatcher.calculate_weightc C sF | j dd | t| jdrd | j_| tjD ]}|| qd S )NrZ )r[ current_patcher)r] clean_hooksr4 r_ r r r
ON_CLEANUPrH r r r r rZ A s
zModelPatcher.cleanup call_typer r c C | |d | d S r& )add_callback_with_key)rH r r r r r add_callbackI r zModelPatcher.add_callbackrD c C $ | j |i |g }|| d S r& )r
setdefaultre )rH r rD r r r r r r L z"ModelPatcher.add_callback_with_keyc C ( | j |i }||v r|| d S d S r& )r r ru )rH r rD r r r r remove_callbacks_with_keyP z&ModelPatcher.remove_callbacks_with_keyc C | j |i |g S r& )r r )rH r rD r r r
get_callbacksU r+ zModelPatcher.get_callbacksc C * g }| j |i D ]}|| q |S r& )r r valuesr )rH r c_listr r r r r X z ModelPatcher.get_all_callbacks wrapper_typewrapperc C r r& )add_wrapper_with_key)rH r r r r r add_wrapper^ r zModelPatcher.add_wrapperc C r r& )r r re )rH r rD r r r r r r a r z!ModelPatcher.add_wrapper_with_keyc C r r& )r r ru )rH r rD r r r r remove_wrappers_with_keye r z%ModelPatcher.remove_wrappers_with_keyc C r r& )r r )rH r rD r r r get_wrappersj r+ zModelPatcher.get_wrappersc C r r& )r r r r )rH r w_listr r r r get_all_wrappersm r z ModelPatcher.get_all_wrappersc C rJ r& )r )rH rD
attachmentr r r set_attachmentss r z ModelPatcher.set_attachmentsc C || j v r
| j | d S d S r& )r ru r r r r remove_attachmentsv
z ModelPatcher.remove_attachmentsc C | j |d S r& )r r r r r r get_attachmentz r zModelPatcher.get_attachmentr list[PatcherInjection]c C rJ r& )r )rH rD r r r r set_injections} r zModelPatcher.set_injectionsc C r r& )r ru r r r r remove_injections r z ModelPatcher.remove_injectionsc C r r& )r r r r r r get_injections r zModelPatcher.get_injectionsrX list['ModelPatcher']c C rJ r& )r )rH rD rX r r r set_additional_models r z"ModelPatcher.set_additional_modelsc C r r& )r ru r r r r remove_additional_models r z%ModelPatcher.remove_additional_modelsc C s | j |g S r& )r r r r r r get_additional_models_with_key r z+ModelPatcher.get_additional_models_with_keyc C s" g }| j D ]}|| q|S r& )r r r )rH
all_modelsrX r r r get_additional_models s z"ModelPatcher.get_additional_modelsc s. d fdd | }t|} ||d}|S ) N prev_modelslist[ModelPatcher] cache_setset[ModelPatcher]c sZ g }| D ]}| }|D ]}||vr || || q qt|dkr&| S | || S )z>Make sure circular references do not cause infinite recursion.r )r re rd re )r r! next_modelsr_
candidatesr _evaluate_sub_additional_modelsr r r& s
zRModelPatcher.get_nested_additional_models.._evaluate_sub_additional_models)r r! )r r r! r" )r r )rH r
models_setreal_all_modelsr r% r get_nested_additional_models s
z)ModelPatcher.get_nested_additional_modelsc C s t | |dS )Nr )rm )rH rr r r r ra r+ zModelPatcher.use_ejectedc C sb | j s| jrd S | j D ]}|D ]
}|| d| _ qq
| j r-| tjD ]}|| q&d S d S NT)rt rs r r injectr r ON_INJECT_MODELrH r injr r r r rx s
zModelPatcher.inject_modelc C sR | j sd S | j D ] }|D ]}|| qq
d| _ | tjD ]}|| q d S ro )rt r r ejectr r ON_EJECT_MODELr- r r r ru s
zModelPatcher.eject_modelc C s2 t | jdr
| | j_| tjD ]}|| qd S )Nr )r4 r_ r r r
ON_PRE_RUNr r r r pre_run s
zModelPatcher.pre_runc C s | tjD ]}|| | qd S r& )r r ON_PREPARE_STATE)rH timestepr r r r
prepare_state s zModelPatcher.prepare_statec C s | j d ur| j | _d | _ d S d S r& )r r rv r r r restore_hook_patches s
z!ModelPatcher.restore_hook_patchesr comfy.hooks.EnumHookModec C s
|| _ d S r& )r )rH r r r r
set_hook_mode s
zModelPatcher.set_hook_modetr
hook_groupcomfy.hooks.HookGroupr dict[str]c C s |d }d}| di }|jD ]5}|jj||d}|rD| jd ur/| jjD ]
} | |kr.d} nq$t| j D ]
}
|
|rC| j |
q6q|rN|
d d S d S )Nr Fr
)curr_tr
T) r r
hook_keyframeprepare_current_keyframer rf r r containsru patch_hooks) rH r9 r: r r= reset_current_hooksr
hookchanged current_hook cached_groupr r r %prepare_hook_patches_current_keyframe s&
z2ModelPatcher.prepare_hook_patches_current_keyframer target_dictr%
registeredc C s | |d u r
tj }g }|tjjjD ]}|j| jvr%| | q|
| qt |dkrDt | j| _
|D ]
}|| ||| q9| tjD ]
}|| |||| qJ|S r )r6 r' r HookGroupget_type EnumHookTypeWeightr. r re rd re r0 r add_hook_patchesr r ON_REGISTER_ALL_HOOK_PATCHES)rH r rH r rI weight_hooks_to_registerrC r r r r register_all_hook_patches s
z&ModelPatcher.register_all_hook_patchesrC comfy.hooks.WeightHookc
C s | i | j|ji }t }| j }|D ]?}d } d }
t|tr%|} n|d } |d } t |dkr7|d }
| |v rV|
| || g } | ||| || |
f | || < q|| j|j< t
| _t|W d S 1 spw Y d S r` )ra r r r. r r_ rb rc rP re rd re r r r rf )
rH rC rE rg rh current_hook_patchesri rj r/ rk rN rD rl r r r rN s.
$z ModelPatcher.add_hook_patchesc C s i }|d urO|j D ]E}| j|ji }| D ]6}||g }t|jdr.||| n|| D ]}t |}|d |j9 < |
t | q2|||< qq |S )Nr_ r ) r r r r. r mathisclosestrengthr rf re tuple) rH r combined_patchesrC r rD rl r new_patchr r r get_combined_hook_patches s
z&ModelPatcher.get_combined_hook_patchesr
c C sd | j |kr|r| js|d u rtj| ||S | j|d | tjD ]}|| | q"tj| ||S )Nr ) r r r' r %create_transformer_options_from_hooksrA r r ON_APPLY_HOOKS)rH r r
r r r r r r , s zModelPatcher.apply_hooksc C sV | |d urt| }d }| jtjjjkr+t tj
| j tj
d d}| j|d }|d ur_t|}|D ] }||vrKtd| q<| j|||d || q<| | n7| | j|d}d }t|dkru| }|D ]}||vrtd| qw| j|||||d qwn| || _W d d S 1 sw Y d S ) NrV )r} r z:Cached hook could not patch. Key does not exist in model: )cached_weightsrD memory_counterr[ r z:Cached hook would not patch. Key does not exist in model: )r rX rD original_weightsr_ )ra rf ro r r r' r r r r| r r r minimum_inference_memoryr r r r> r? patch_cached_hook_weightsr r rZ re rt patch_hook_weight_to_devicer ) rH r
model_sd_keysr_ r^ model_sd_keys_setrD relevant_patchesr` r r r rA 5 sD
"zModelPatcher.patch_hooksr^ r_ r| c C s || j vr/tj| j|}| j}| jtjjj kr"|
|}|r"|j }|j |dd|j f| j |< tj
| j||| d j || d d d S )NTry r rc r9 )r r' r@ rf r_ r r r r r r r: r r )rH r^ rD r_ rP
target_devicer r r r rb Y s
.z&ModelPatcher.patch_cached_hook_weightsc C s | j | d d S r& )r r rA rv r r r clear_cached_hook_weightsd s
z&ModelPatcher.clear_cached_hook_weightsrX r` c
C sd ||vrd S t | j|\}}}|| jvr6| j} | jtjjjkr)| |}
|
r)|j
} |j | dd|j
f| j|< tj j
||j
tjdd} |d urL|| dd} tjj|| | ||d} ||= |d u rwtjj| |jtj|d} tj| j|| n || dtj|d | jtjjjkr| j} | |}
|
r|j
} | j|i | j | dd|j
f| j| |< ~ ~ ~d S ) NTry r rz )r` r| )rx r} F)rX r_ r r r r' r r r r r: r r r rZ r[ rM rN r r rO r@ rA r r r )
rH r rX rD r` r_ rP rG rF rg r r r r r r rc h s@
z(ModelPatcher.patch_hook_weight_to_devicewhitelist_keys_setset[str]Nonec C s | x t| jdkrd | _ W d d S t| j }|rH|D ]#}||v rFtj| j || j| d j
| j| d d | j | q#n,|D ]}tj| j || j| d j
| j| d d qJ| j d | _W d d S W d d S 1 sw Y d S )Nr rc r9 )
ra re r r rf r r' r@ r r_ r ru r )rH ri r r/ r r r r s(
. 0
"zModelPatcher.unpatch_hooksc C s | | d S r& )r rh rv r r r r s zModelPatcher.clean_hooksc
C s | j j }| D ]N\}}|dd}t|dk s |d dvr!q
z
tj| j j|d }W n Y q
|rEt |drEt |drF|j
dkrFq
d | } t | | tj| j | ||< q
| j j ||||d
S ) Nrb rc rV )rP biasr r5 r Tzdiffusion_model.)clip_state_dictvae_state_dictclip_vision_state_dict)
r_ diffusion_modelrb r rd re r' r@ rf r4 r r state_dict_for_saving)
rH rm rn ro unet_state_dictr/ vri rj rD r r r rq s"
z"ModelPatcher.state_dict_for_savingc C s | | jdd d S )NF)r )r r rv r r r __del__ s zModelPatcher.__del__r Fr{ )FN)r rn )r r r& )r rP rQ rR )r_ r_ )NFF)Nr FFNr TFr* )T)r rP r r )r rP rD rP r r )r rP rD rP )r rP )r rP r r )r rP rD rP r r )r rP rD rP )r rP )rD rP )rD rP r r )rD rP rX r )r r7 )r9 r r: r; r r< rR )r r; rH r< r r% rI r; )rC rR )r r; ro )r r; r
r% r^ r% rD rP r_ r|
r r; rX r% rD rP r` r% r_ r| ri rj rQ rk )NNN)mrS rT rU rI r r r r r r r r r r r r r r r r r r r r r# r% r* r- r/ r1 r3 r5 r7 r9 r; r= r? rA rI rL rM rO rS rW rY r] r` rm rt ro r r r r r r r r r r r r r r rZ r[ rN rZ r r r r r r r r
r r
r r r r r r r r r r r) ra rx ru r2 r5 r6 r8 rG rQ rN rZ r rA rb rh rc r r rq rt r r r r r s : G
% ) U
$ ' r c s e Zd Zd; fdd Zd< fdd Zdd Zd=d
d Zd d
Zdd Zdd Z dd Z
fddZ d>ddZ d t ||||| t| jdsi | j_d | _|d us J d S )N
dynamic_vbars)r rI r4 r_ r| non_dynamic_delegate_modelr r r r rI s
z ModelPatcherDynamic.__init__c C r r* r rv r r r r r z ModelPatcherDynamic.is_dynamicc C s^ | j tdkr
d S | jj| j d }|r-|d u r-tj| d | j j
}|| jj| j < |S )Nr
) r rZ r: r_ r| r r r ModelVBARr r )rH createvbarr r r _vbar_get s z ModelPatcherDynamic._vbar_getc C s$ | }|d ur | nd| jj S r )r r r_ r )rH r r r r r s z ModelPatcherDynamic.loaded_sizec C t d)Nz7pin_weight_to_device invalid for dymamic weight loadingr r r r r r r z(ModelPatcherDynamic.pin_weight_to_devicec C r )Nz/unpin_weight invalid for dymamic weight loadingr r r r r r r z ModelPatcherDynamic.unpin_weightc C s | d d S )Nr )r rv r r r r r z%ModelPatcherDynamic.unpin_all_weightsc s t j|dd d S )Nr g?i @)r r r r r r r s z#ModelPatcherDynamic.memory_requiredc s^ |rJ |rJ || j ksJ dd}d| j_| | | jdd}|d ur/| | jd|d}| |D ]} | ^ }
} } }
dd }fdd} fd d
}t | d rd| _
d | _ | _ || | || | d
\}}|| | d\}}|p~|}||7 }|rt
d d || d
| || d| n`|d urt | ds||| _||7 }nL|
D ]I}t |}t| j|\}}
}
|| jvrtdd
dg|d | j|< t| |d d }|j||d}tj| j|| | j j| | 7 _qt | | q<| jj ddD ]@\}}|| j vr|| j |< tj | j|\}} t|| d d }|j||d} tj!| j|| | j j| | 7 _qt"| jdkradt"| j d| jjd dnd} t
d | jj#j$ d |d d d | || j_%| j&| j_'| (t)j*D ] } | | |||| q| j+| j,dd! W d d S 1 sw Y d S )"Nr T)r r r c S s |st | ds d | _d S d S )N _v_signature)r4 r )itemdirtyr r r set_dirty s
z+ModelPatcherDynamic.load..set_dirtyc
s t ||}g }t| j|\}}}|d u rdS || jv r.setup_paramc sz t |}|| jv rtj| j|| j| j | j||d t| j|\}}}|d ur;| j j |
| 7 _ d S d S )Nr ) rl r r' r@ r r_ rP r rX r r\ r )rH r r rD rP r )r r r force_load_param+ s
z2ModelPatcherDynamic.load..force_load_paramr5 FrP rl zModule z" has resizing Lora - force loading_vrw rx r )rO r: r z Force pre-loaded z
weights: i z KB.r zModel z$ prepared for dynamic VRAM loading. r z MB Staged. z patches attached.r )-r r_ r ra r r
prioritizer r r4 r5
pin_failedseed_keyr> r allocr rl rX r r~ r rY r r' r@ r r\ r r=
named_buffersr resolve_attrset_attr_bufferre r rS r: r r r r r r r ) rH r r r r r allocated_sizer r r r r r6 r r r r
force_load
v_weight_sizeforce_load_bias
v_weight_biasrk rD rP r`
casted_weightbufmodulebuf_name
casted_bufforce_load_statr r )r r r r sx
4,
$zModelPatcherDynamic.loadc C s |rJ | j tdksJ | }|d u rdn||}||k r\t| j D ]}| j|}t j
| j ||j
q(t| j D ]}t j
| j || j| qB|| j j7 }d| j _|S )Nr r )r rZ r: r free_memoryrf r r ru r' r@ r r_ rP r r r )rH r r r r freedrD rr r r r r j s z$ModelPatcherDynamic.partially_unloadc C sV d}| j dd}|D ] }|\}}}}}}tj|}|d ur(|| | 7 }q
|S )Nr T)r )r r'
pinned_memoryget_pinr\ r )rH totalr r r r6 pinr r r r | s z&ModelPatcherDynamic.pinned_memory_sizec C sH | j d| jd}|D ]}|^ }}}|tj|8 }|dkr! d S q
d S )NTr r )r r r' r r )rH r r r r r6 r r r r s z(ModelPatcherDynamic.partially_unload_ramTc s |rJ t j||dS )N)r r )r r )rH r r r r r r r r s z ModelPatcherDynamic.patch_modelc sL t jd dd |r"| d | d d | j D ] }t|| qd S d S )NF)r r r )r r r r r_ r r= )rH r r r6 r r r r s
z!ModelPatcherDynamic.unpatch_modelc C s |rJ | j ddA | jjd uo| jj| jk}| j| jdd | jdd z | j||d W n tyA } z| |d }~ww W d d S 1 sNw Y d S )NTr Fr r )r )
ra r_ r r r r r r r r )rH r r r r r r r r r s $z"ModelPatcherDynamic.partially_loadr^ r% rD rP r_ r| c C s J r& r )rH r^ rD r_ r r r rb r z-ModelPatcherDynamic.patch_cached_hook_weightsr r; rX r` c C s ||vrd S t d)NzaHooks not implemented in ModelPatcherDynamic. Please remove --fast arguments form ComfyUI startupr )rH r rX rD r` r_ r r r rc s z/ModelPatcherDynamic.patch_hook_weight_to_deviceri rj rQ rk c C r r& r )rH ri r r r r r z!ModelPatcherDynamic.unpatch_hooksc C s | j d| jd}| | _|S )NT)r r )r r} r )rH
model_patcherr r r get_non_dynamic_delegate s
z,ModelPatcherDynamic.get_non_dynamic_delegate)NNNr Fru r{ )Nr FFFrv r* rw rx r& ry )rS rT rU r rI r r r r r r r r r r r r r r rb rc r r r r r r r rz s*
|
rz r& r{ )r$ r% )3__doc__
__future__r r~ r r> rT r typingr r rZ comfy.floatr' comfy.hooks
comfy.loracomfy.model_managementcomfy.patcher_extension comfy.utilscomfy.comfy_typesr comfy.quant_opsr r r r comfy_aimdo.model_vbarr r r r# r* r0 r7 r= rA rC r^ ra rX rl rm r| r r nn Parameterr r rz CoreModelPatcherr r r r sd
[