a
    Ad                     @   sL   d dl Z d dlZd dlZd dlmZ G dd dejZG dd dejZdS )    N)devicesc                   @   s    e Zd ZejejdddZdS )ModuleTypeLoranetweightsc                    s$   t  fdddD r t| S d S )Nc                 3   s   | ]}| j v V  qd S )N)w).0xr    L/var/www/html/stable-diffusion-webui/extensions-builtin/Lora/network_lora.py	<genexpr>
       z/ModuleTypeLora.create_module.<locals>.<genexpr>)lora_up.weightlora_down.weight)allNetworkModuleLoraselfr   r   r   r
   r   create_module	   s    
zModuleTypeLora.create_moduleN)__name__
__module____qualname__networkNetworkNetworkWeightsr   r   r   r   r   r      s   r   c                       sB   e Zd Zejejd fddZdddZdd Zd	d
 Z	  Z
S )r   r   c                    sX   t  || | |jd| _| |jd| _| j|jddd| _|jd jd | _d S )Nr   r   lora_mid.weightT)none_okr   )	super__init__r   r   up_model
down_model	mid_modelshapedimr   	__class__r   r   r      s
    zNetworkModuleLora.__init__Fc                 C   sH  | |}|d u r|rd S t| jtjjtjjjjtjj	fv }t| jtjj
fv }|r||jd d}tjj|jd |jd dd}nH|r|dks|dkr6t|jdkr||jd ddd}|jd dks|jd	 dkrtjj
|jd |jd | jj| jj| jjdd}n tjj
|jd |jd d
dd}n|rx|dkrxtjj
|jd |jd | jj| jj| jjdd}nZ|r|dks|dkrtjj
|jd |jd d
dd}ntd| j dt| jj t : |j|jjkr||jj}|j| W d    n1 s0    Y  |jtjtjd |jd |S )Nr      F)biasr   Zdyn_up      )r(   r(   r   r   Zdyn_downzLora layer z( matched a layer with unsupported type: )devicedtype)gettype	sd_moduletorchnnLinearmoduleslinearNonDynamicallyQuantizableLinearMultiheadAttentionConv2dreshaper#   lenkernel_sizestridepaddingAssertionErrornetwork_keyr   no_gradweightcopy_tor   cpur-   requires_grad_)r   r   keyr   rA   	is_linearis_convmoduler   r   r   r      s4    
$"2"2"
,zNetworkModuleLora.create_modulec                 C   s   | j jj|j|jd}| jjj|j|jd}|d|dg}| jd ur| jjj|j|jd}t	|||}||j
dd  7 }n4t|j
dkr||j
dd  7 }t|||| jj}| |||S )N)r-   r   r(   r*      )r    rA   rC   r,   r-   r!   sizer"   lyco_helpersZrebuild_cp_decompositionr#   r:   Zrebuild_conventionalr   dyn_dimfinalize_updown)r   orig_weightupdownoutput_shapemidupdownr   r   r   calc_updown?   s    
zNetworkModuleLora.calc_updownc                 C   sD   | j jtjd | jjtjd ||  | ||   |    S )N)r,   )r    rC   r   r,   r!   
multiplier
calc_scale)r   r	   yr   r   r   forwardP   s    zNetworkModuleLora.forward)F)r   r   r   r   r   r   r   r   rU   rY   __classcell__r   r   r%   r   r      s   	
%r   )	r1   rL   r   r4   r   
ModuleTyper   NetworkModuler   r   r   r   r   <module>   s
   