a
    Ad,                     @   s   d dl Zd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
mZ G dd dejZG dd dejZG d	d
 d
ejZG dd dejZdS )    N)	rearrange)	Rearrange)trunc_normal_DropPathc                       s8   e Zd ZdZ fddZdd Zdd Zdd	 Z  ZS )
WMSAz/ Self-attention module in Swin Transformer
    c                    s   t t|   || _|| _|| _| jd | _|| | _|| _|| _	t
j| jd| j dd| _t
td| d d| d  | j| _t
| j| j| _t| jdd tj
| jd| d d| d | jddd	d| _d S )
Ng         Tbias      {Gz?stdr   )superr   __init__	input_dim
output_dimhead_dimscalen_headswindow_sizetypennLinearembedding_layer	Parametertorchzerosrelative_position_paramslinearr   view	transpose)selfr   r   r   r   r   	__class__ S/var/www/html/stable-diffusion-webui/extensions-builtin/ScuNET/scunet_model_arch.pyr      s*    
 $zWMSA.__init__c              
   C   s   t j||||||t j| jjd}| jdkr.|S || }d|dddd|dd|dddf< d|ddd|dddd|ddf< d|dddddd|dd|df< d|ddddd|dddd|f< t|d}|S )z generating the mask of SW-MSA
        Args:
            shift: shift parameters in CyclicShift.
        Returns:
            attn_mask: should be (1 1 w p p),
        )dtypedeviceWTNz0w1 w2 p1 p2 p3 p4 -> 1 1 (w1 w2) (p1 p2) (p3 p4))r   r   boolr   r(   r   r   )r"   hwpshift	attn_masksr%   r%   r&   generate_mask$   s     
((((
zWMSA.generate_maskc                 C   sb  | j dkr.tj|| jd  | jd  fdd}t|d| j| jd}|d}|d}t|d| j| jd}| |}t|d	| jd
jddd\}}}t	d||| j
 }|t|  d }| j dkr| j||| j| jd d}	||	td}tjj|dd}
t	d|
|}t|d}| |}t|d|| jd}| j dkr^tj|| jd | jd fdd}|S )a   Forward pass of Window Multi-head Self-attention module.
        Args:
            x: input tensor with shape of [b h w c];
            attn_mask: attention mask, fill -inf where the value is True;
        Returns:
            output: tensor shape [b h w c]
        r)   r
   )r   r
   )shiftsdimsz&b (w1 p1) (w2 p2) c -> b w1 w2 p1 p2 c)p1p2r   z&b w1 w2 p1 p2 c -> b (w1 w2) (p1 p2) cz&b nw np (threeh c) -> threeh b nw np c)cr   r   dimzhbwpc,hbwqc->hbwpqzh p q -> h 1 1 p q)r/   z-infr*   zhbwij,hbwjc->hbwiczh b w p c -> b w p (h c)z*b (w1 w2) (p1 p2) c -> b (w1 p1) (w2 p2) c)w1r5   )r   r   rollr   r   sizer   r   chunkeinsumr   relative_embeddingr2   masked_fill_floatr   
functionalsoftmaxr   )r"   xZ	h_windowsZ	w_windowsqkvqkvsimr0   probsoutputr%   r%   r&   forward8   s*    
$


 


 zWMSA.forwardc                    s   t t fddt jD }|d d d d d f |d d d d d f   j d } jd d |d d d d df  |d d d d df  f S )Nc                    s$   g | ]}t  jD ]}||gqqS r%   )ranger   ).0ijr"   r%   r&   
<listcomp>`       z+WMSA.relative_embedding.<locals>.<listcomp>r   r   )r   tensornparrayrM   r   r   long)r"   Zcordrelationr%   rQ   r&   r?   _   s    $6zWMSA.relative_embedding)	__name__
__module____qualname____doc__r   r2   rL   r?   __classcell__r%   r%   r#   r&   r   
   s
   'r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )Blockr)   Nc                    s   t t|   || _|| _|dv s&J || _||kr:d| _t|| _t	||||| j| _
|dkrjt|nt | _t|| _tt|d| t td| || _dS )z SwinTransformer Block
        r)   SWr)              N)r   r^   r   r   r   r   r   	LayerNormln1r   msar   Identity	drop_pathln2
Sequentialr   GELUmlp)r"   r   r   r   r   rg   r   input_resolutionr#   r%   r&   r   g   s     zBlock.__init__c                 C   s8   ||  | | | }||  | | | }|S )N)rg   re   rd   rk   rh   )r"   rD   r%   r%   r&   rL   |   s    zBlock.forward)r)   NrY   rZ   r[   r   rL   r]   r%   r%   r#   r&   r^   f   s   r^   c                       s&   e Zd Zd fdd	Zdd Z  ZS )ConvTransBlockr)   Nc                    s  t t|   || _|| _|| _|| _|| _|| _|| _	| jdv sFJ | j	| jkrXd| _t
| j| j| j| j| j| j| j	| _tj| j| j | j| j ddddd| _tj| j| j | j| j ddddd| _ttj| j| jdddddtdtj| j| jddddd| _d	S )
z( SwinTransformer and Conv Block
        r_   r)   r   r   Tr   r   FN)r   rn   r   conv_dim	trans_dimr   r   rg   r   rl   r^   trans_blockr   Conv2dconv1_1conv1_2ri   ReLU
conv_block)r"   ro   rp   r   r   rg   r   rl   r#   r%   r&   r      s*    ((zConvTransBlock.__init__c                 C   sv   t j| || j| jfdd\}}| || }td|}| |}td|}| t j	||fdd}|| }|S )Nr   r8   zb c h w -> b h w czb h w c -> b c h w)
r   splitrs   ro   rp   rv   r   rq   rt   cat)r"   rD   Zconv_xZtrans_xresr%   r%   r&   rL      s    "
zConvTransBlock.forward)r)   Nrm   r%   r%   r#   r&   rn      s   rn   c                       s.   e Zd Zd fdd	Zdd	 Zd
d Z  ZS )SCUNetr   N@   ra      c              	      s  t t  |d u rg d}|__d_d_dd td|t	|D t
j|dddd	d
g_d  fddt|d D t
jd dddd	d
g _ |d 7   fddt|d D t
jd d dddd	d
g _ |d 7   fddt|d D t
jd d dddd	d
g _ |d 7   fddt|d D _ |d 7  t
jd d dddd	d
g fddt|d D  _ |d 7  t
jd d dddd	d
g fddt|d D  _ |d 7  t
jd dddd	d
g fddt|d D  _t
j|dddd	d
g_t
jj _t
jj _t
jj _t
jj _t
jj _t
jj _t
jj _t
jj _t
jj _d S )N)r
   r
   r
   r
   r
   r
   r
          c                 S   s   g | ]}|  qS r%   )item)rN   rD   r%   r%   r&   rR      rS   z#SCUNet.__init__.<locals>.<listcomp>r   r   r   Fr   c              
      s@   g | ]8}t d  d  jj|   |d  s4dndqS r
   r)   r`   rn   r   r   rN   rO   beginr9   dprrl   r"   r%   r&   rR      s    r
   c                    s<   g | ]4}t jj|   |d  s,dndd  qS r   r   r   r   r%   r&   rR      s   rb   c                    sD   g | ]<}t d  d  jj|   |d  s4dndd qS r
   r)   r`   rb   r   r   r   r%   r&   rR      s    c                    sD   g | ]<}t d  d  jj|   |d s4dndd qS )rb   r
   r)   r`   r~   r   r   r   r%   r&   rR      s    c                    sD   g | ]<}t d  d  jj|   |d  s4dndd qS r   r   r   r   r%   r&   rR      s    c                    s<   g | ]4}t jj|   |d  s,dndd  qS r   r   r   r   r%   r&   rR      s      c              
      s@   g | ]8}t d  d  jj|   |d  s4dndqS r   r   r   r   r%   r&   rR      s       )r   rz   r   configr9   r   r   r   linspacesumr   rr   m_headrM   m_down1m_down2m_down3m_bodyConvTranspose2dm_up3m_up2m_up1m_tailri   )r"   in_ncr   r9   drop_path_raterl   r#   r   r&   r      sv    






zSCUNet.__init__c                 C   s   |  dd  \}}tt|d d | }tt|d d | }td|d|f|}| |}| |}| |}| 	|}	| 
|	}
| |
|	 }
| |
| }
| |
| }
| |
| }
|
dd |d |f }
|
S )Nr{   r   .)r<   intrU   ceilr   ReplicationPad2dr   r   r   r   r   r   r   r   r   )r"   x0r,   r-   ZpaddingBottomZpaddingRightx1x2x3x4rD   r%   r%   r&   rL      s    




zSCUNet.forwardc                 C   sf   t |tjr6t|jdd |jd urbtj|jd n,t |tjrbtj|jd tj|jd d S )Nr   r   r   g      ?)	
isinstancer   r   r   weightr	   init	constant_rc   )r"   mr%   r%   r&   _init_weights  s    
zSCUNet._init_weights)r   Nr{   ra   r|   )rY   rZ   r[   r   rL   r   r]   r%   r%   r#   r&   rz      s   Drz   )numpyrU   r   torch.nnr   einopsr   Zeinops.layers.torchr   timm.models.layersr   r   Moduler   r^   rn   rz   r%   r%   r%   r&   <module>   s   \(