a
    Ad7T                     @   s4  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZmZmZmZmZmZ e e e e e	 e
 gZedZedZi Zi dddd	d
dddZ dd Z!dd Z"dd Z#dd Z$d;ddZ%eej&j'ej&j(ej&j)ej&j*ej&j+f dddZ,eej&j'ej&j(ej&j)ej&j*ej&j+f dddZ-dd Z.eej&j'ej&j(f ddd Z/d!d" Z0d#d$ Z1d%d& Z2d'd( Z3d)d* Z4d+d, Z5d-d. Z6d/d0 Z7d1d2 Z8d3d4 Z9d5d6 Z:ed7Z;d8d9 Z<dZ=ej>e?d:< dZ@i ZAi ZBg ZCi ZDi ZEi ZFe:  dS )<    N)Union)shareddevices	sd_modelserrorsscripts	sd_hijackz\d+z(.*)_([qkv]_proj)$Zin_layers_2Zout_layers_3Zin_layers_0Zout_layers_0Zemb_layers_1skip_connection)conv1conv2norm1norm2Ztime_emb_projconv_shortcut)
attentionsZresnetsc                    s   fdd}g }||dr(d|d  S ||dr@d|d  S ||drhd	|d d
 d
  |d  S ||drt |d i |d |d }dd|d d  |d
   d|d dkrdnd d| S ||dr"t |d i |d
 |d
 }d|d dkrdn
|d d
  d| S ||drt |d i |d |d }d|d d |d
   d|d dkrxdnd d| S ||drdd|d d   dS ||drdd
|d d   d|d dkrd
nd dS ||dr|rtd|d v r&d|d  d|d dd S d|d v rTd|d  d|d dd S d|d  d|d dd  S d!|d  d|d  S ||d"rd|d v rd#|d  d|d dd S d|d v rd#|d  d|d dd S d#|d  d|d dd  S  S )$Nc                    s\   t |}|d u r$t|}|t |< t| }|s8dS |   | dd | D  dS )NFc                 S   s$   g | ]}t t|rt|n|qS  )rematch	re_digitsint.0xr   r   H/var/www/html/stable-diffusion-webui/extensions-builtin/Lora/networks.py
<listcomp>:       zDconvert_diffusers_name_to_compvis.<locals>.match.<locals>.<listcomp>T)re_compiledgetr   compiler   clearextendgroups)
match_listZ
regex_textregexrkeyr   r   r   /   s    

z0convert_diffusers_name_to_compvis.<locals>.matchzlora_unet_conv_in(.*)Z diffusion_model_input_blocks_0_0r   zlora_unet_conv_out(.*)Zdiffusion_model_out_2z)lora_unet_time_embedding_linear_(\d+)(.*)Zdiffusion_model_time_embed_      z;lora_unet_down_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)   Zdiffusion_model_input_blocks__r   z3lora_unet_mid_block_(attentions|resnets)_(\d+)_(.+)Zdiffusion_model_middle_block_z9lora_unet_up_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)Zdiffusion_model_output_blocks_z/lora_unet_down_blocks_(\d+)_downsamplers_0_convZ_0_opz+lora_unet_up_blocks_(\d+)_upsamplers_0_conv_convz,lora_te_text_model_encoder_layers_(\d+)_(.+)Zmlp_fc1Zmodel_transformer_resblocks_Zmlp_c_fcZmlp_fc2Z
mlp_c_proj	self_attnattnZ&transformer_text_model_encoder_layers_z-lora_te2_text_model_encoder_layers_(\d+)_(.+)Z1_model_transformer_resblocks_)suffix_conversionr   replace)r%   is_sd2r   msuffixr   r$   r   !convert_diffusers_name_to_compvis.   sH    



 : * 80      r2   c                 C   s   i }t jjrjtt jjjD ]L\}}t|ds.q|j D ],\}}| d|	dd }|||< ||_
q8qn2t jjj D ]"\}}|	dd}|||< ||_
qxt jj D ]"\}}|	dd}|||< ||_
q|| _d S )Nwrappedr)   .)r   sd_modelis_sdxl	enumerateconditioner	embeddershasattrr3   named_modulesr.   network_layer_namecond_stage_modelmodelnetwork_layer_mapping)r5   r?   iembeddernamemodulenetwork_namer   r   r   'assign_network_names_to_compvis_modulesp   s$    
rE   c                 C   s$  t | |}tj|j|_t|j}t	t
jds>tt
j i }dt
jjv }i }| D ]$\}}|dd\}	}
t|	|}t
jj|d }|d u rt|}|rt
jj|dd }|d u rd|	v r|	dd}t
jj|d }nV|d u r>d|	v r>|	dd}t
jj|d }|d u r>|	dd	}t
jj|d }|d u rR|||< qZ||vrrt j||i |d
||< ||| j|
< qZ| D ]v\}}d }tD ]"}|||}|d ur qq|d u rtdddd tD  dd|j ||j|< q|r td|j d|  |S )Nr?   Zmodel_transformer_resblocksr4   r'   Z	lora_unetdiffusion_modelZlora_te1_text_modelZ0_transformer_text_modelZtransformer_text_model)network_keysd_keyw	sd_modulez%Could not find a module type (out of , c                 S   s   g | ]}|j jqS r   )	__class____name__r   r   r   r   r      r   z load_network.<locals>.<listcomp>z ) that would accept those keys: Network z didn't match keys: )networkNetworkospathgetmtimefilenamemtimer   read_state_dictr:   r   r5   rE   r?   itemssplitr2   r   	re_x_projr   groupr.   NetworkWeightsrI   module_typescreate_moduleAssertionErrorjoinmodulesloggingdebug)rB   network_on_disknetsdZkeys_failed_to_matchr/   Zmatched_networksZkey_networkweightZ!key_network_without_network_partsZnetwork_partr%   rJ   r0   weightsZ
net_moduleZnettyper   r   r   load_network   sV    








,rh   c                  C   sB   t ttjjkr6t tdkr6ttt} t| d  q t	  d S )Nr   )
lennetworks_in_memoryr   optslora_in_memory_limitnextiterpopr   torch_gc)rB   r   r   r   purge_networks_from_memory   s    rq   c                 C   s  i }t D ]}|j| v r|||j< qt   dd | D }tdd |D r^t  dd | D }g }tt|| D ]6\}\}	}
||
d }|	d ur<|d u rt|
}|d u st	j
|	j|jkr.z"t|
|	}t|
d  |t|
< W nF ty, } z,t|d|	j  W Y d }~qpW Y d }~n
d }~0 0 |
|_|	  |d u rb||
 td|
  qp|rp|| nd|_|r|| nd|_|r|| nd|_t | qp|rtjjd	d
|  t  d S )Nc                 S   s   g | ]}t |d qS Navailable_network_aliasesr   r   rB   r   r   r   r      r   z!load_networks.<locals>.<listcomp>c                 s   s   | ]}|d u V  qd S rr   r   r   r   r   r   	<genexpr>   r   z load_networks.<locals>.<genexpr>c                 S   s   g | ]}t |d qS rr   rs   ru   r   r   r   r      r   zloading network z Couldn't find network with name g      ?zNetworks not found: rK   ) loaded_networksrB   r   anylist_available_networksr7   zipr   rj   rQ   rR   rS   rT   rU   rh   ro   	Exceptionr   displaymentioned_name	read_hashappendra   infote_multiplierunet_multiplierdyn_dimr   model_hijackcommentsr_   rq   )namesZte_multipliersZunet_multipliersZdyn_dimsalready_loadedrd   Znetworks_on_diskZfailed_to_load_networksr@   rc   rB   er   r   r   load_networks   sH    



"

r   selfc                 C   s   t | dd }t | dd }|d u r,|d u r,d S |d urrt| tjjrf| j|d  | jj|d  n| j| |d urt| tjjr| jj	| q| j	| nt| tjjrd | j_	nd | _	d S )Nnetwork_weights_backupnetwork_bias_backupr   r'   )
getattr
isinstancetorchnnMultiheadAttentionin_proj_weightcopy_out_projrf   bias)r   weights_backupbias_backupr   r   r   #network_restore_weights_from_backup
  s     
r   c                 C   s  t | dd}|du rdS t | dd}tdd tD }t | dd}|du r|dkr|dkrbtdt| tjjr| jj	t
jd	d
| jjj	t
jd	d
f}n| jj	t
jd	d
}|| _t | dd}|du r&t| tjjr| jjdur| jjj	t
jd	d
}n*t | dddur| jj	t
jd	d
}nd}|| _||krt|  tD ]B}|j|d}|durt| drzt  || j\}}	t| jjdkr| jjd dkrtjj|d}|  j|7  _|	durt| dr| jdu rtj|	| _n|  j|	7  _W d   n1 s0    Y  W n` ty }
 zFtd|j d| d|
  tj |jdd tj |j< W Y d}
~
n
d}
~
0 0 q<|j|d d}|j|d d}|j|d d}|j|d d}t| tjjr:|r:|r:|r:|r:zt  || j\}}|| j\}}|| j\}}t!|||g}|| jj\}}	|  j|7  _| j j|7  _W d   n1 s0    Y  |	dur| jjdu rtj|	| j_n| j j|	7  _W n` ty4 }
 zFtd|j d| d|
  tj |jdd tj |j< W Y d}
~
n
d}
~
0 0 q<|du rHq<td|j d| d tj |jdd tj |j< q<|| _"dS )a
  
    Applies the currently selected set of networks to the weights of torch layer self.
    If weights already have this particular set of networks applied, does nothing.
    If not, restores orginal weights from backup and alters weights according to networks.
    r<   Nnetwork_current_namesr   c                 s   s"   | ]}|j |j|j|jfV  qd S rr   )rB   r   r   r   r   r   r   r   rv   0  r   z(network_apply_weights.<locals>.<genexpr>r   z=no backup weights found and current weights are not unchangedT)copyr   r   rf      r'   	   )r   r   r   r   r      rN   z layer z: r   Z_q_projZ_k_projZ_v_projZ	_out_projz#: couldn't find supported operation)#r   tuplerw   RuntimeErrorr   r   r   r   r   tor   cpur   rf   r   r   r   r   r`   r   r:   no_gradcalc_updownri   shape
functionalpad	Parameterra   rb   rB   extra_network_lorar   vstackr   )r   r<   Zcurrent_namesZwanted_namesr   r   rd   rC   updownex_biasr   Zmodule_qZmodule_kZmodule_vZ
module_outZupdown_qr)   Zupdown_kZupdown_vZ
updown_qkvZ
updown_outr   r   r   network_apply_weights$  s    (



$22(
0
2
 r   c                 C   sx   t tdkr|| |S t|}t|  t|  || |}t| dd}tD ](}|j|d} | du rfqJ| 	||}qJ|S )z
    Old way of applying Lora by executing operations during layer's forward.
    Stacking many loras this way results in big performance degradation.
    r   r<   N)
ri   rw   r   cond_cast_unetr   network_reset_cached_weightr   r`   r   forward)rC   inputoriginal_forwardyr<   lorar   r   r   network_forward  s    


r   c                 C   s   d| _ d | _d S )Nr   )r   r   r   r   r   r   r     s    r   c                 C   s*   t jjrt| |tjS t|  t| |S rr   )r   rk   lora_functionalr   	originalsZLinear_forwardr   r   r   r   r   r   network_Linear_forward  s    r   c                 O   s    t |  tj| g|R i |S rr   )r   r   ZLinear_load_state_dictr   argskwargsr   r   r   network_Linear_load_state_dict  s    r   c                 C   s*   t jjrt| |tjS t|  t| |S rr   )r   rk   r   r   r   ZConv2d_forwardr   r   r   r   r   network_Conv2d_forward  s    r   c                 O   s    t |  tj| g|R i |S rr   )r   r   ZConv2d_load_state_dictr   r   r   r   network_Conv2d_load_state_dict  s    r   c                 C   s*   t jjrt| |tjS t|  t| |S rr   )r   rk   r   r   r   ZGroupNorm_forwardr   r   r   r   r   network_GroupNorm_forward  s    r   c                 O   s    t |  tj| g|R i |S rr   )r   r   ZGroupNorm_load_state_dictr   r   r   r   !network_GroupNorm_load_state_dict  s    r   c                 C   s*   t jjrt| |tjS t|  t| |S rr   )r   rk   r   r   r   ZLayerNorm_forwardr   r   r   r   r   network_LayerNorm_forward  s    r   c                 O   s    t |  tj| g|R i |S rr   )r   r   ZLayerNorm_load_state_dictr   r   r   r   !network_LayerNorm_load_state_dict  s    r   c                 O   s    t |  tj| g|R i |S rr   )r   r   ZMultiheadAttention_forwardr   r   r   r   "network_MultiheadAttention_forward  s    r   c                 O   s    t |  tj| g|R i |S rr   )r   r   Z"MultiheadAttention_load_state_dictr   r   r   r   *network_MultiheadAttention_load_state_dict  s    r   c               	   C   s$  t   t  t  t  tddd tjtj	j
dd ttjtj	j
g dd} | ttjtj	jg dd7 } | D ]}tj|rq~tjtj|d }zt||}W n0 ty   tjd| d	| dd
 Y q~Y n0 |t |< |jtv rdt|j < |t|< |t|j< q~d S )Nr'   )noneZAddamsT)exist_ok)z.ptz.ckptz.safetensors)allowed_extensionsr   zFailed to load network z from )exc_info)available_networksr   rt   forbidden_network_aliasesavailable_network_hash_lookupupdaterQ   makedirsr   cmd_optslora_dirlist
walk_fileslyco_dir_backcompatrR   isdirsplitextbasenamerO   NetworkOnDiskOSErrorr   reportaliaslower)
candidatesrT   rB   entryr   r   r   ry     s,    
ry   z(.*)\s*\([0-9a-fA-F]+\)c                 C   s   ddd t jjD v rd S g }|D ]}|ds2q"|dd  }|d| dkrRq"|d| }|d u rjq"t|}|r|d}|d	| d
}|d| d| d q"|r|d  dd	| 7  < d S )NzAddNet Module 1c                 S   s   g | ]}|d  qS )r'   r   r   r   r   r   r     r   z#infotext_pasted.<locals>.<listcomp>zAddNet Model    zAddNet Module ZLoRAr'   zAddNet Weight A z1.0z<lora::>Prompt
 )
r   scripts_txt2imginfotext_fields
startswithr   re_network_namer   rZ   r   r_   )infotextparamsaddedknumrB   r0   
multiplierr   r   r   infotext_pasted  s&    


r   r   )NNN)Gra   rQ   r   lora_patchesrO   Znetwork_loraZnetwork_hadaZnetwork_ia3Znetwork_lokrZnetwork_fullZnetwork_normr   typingr   r`   r   r   r   r   r   r   ZModuleTypeLoraZModuleTypeHadaZModuleTypeIa3ZModuleTypeLokrZModuleTypeFullZModuleTypeNormr\   r   r   rY   r   r-   r2   rE   rh   rq   r   r   Conv2dLinear	GroupNorm	LayerNormr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ry   r   r   r   LoraPatches__annotations__r   r   rt   rw   rj   r   r   r   r   r   r   <module>   s~   
 


BB
600c				
