a
    þdJ  ã                   @   s‚   d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ G d	d
„ d
ejƒZG dd„ dejƒZG dd„ dejƒZdS )aw   Bilinear-Attention-Transform and Non-Local Attention

Paper: `Non-Local Neural Networks With Grouped Bilinear Attentional Transforms`
    - https://openaccess.thecvf.com/content_CVPR_2020/html/Chi_Non-Local_Neural_Networks_With_Grouped_Bilinear_Attentional_Transforms_CVPR_2020_paper.html
Adapted from original code: https://github.com/BA-Transform/BAT-Image-Classification
é    N)Únn)Ú
functionalé   )ÚConvNormAct)Úmake_divisible)Ú_assertc                       s2   e Zd ZdZd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )ÚNonLocalAttnzßSpatial NL block for image classification.

    This was adapted from https://github.com/BA-Transform/BAT-Image-Classification
    Their NonLocal impl inspired by https://github.com/facebookresearch/video-nonlocal-net.
    Tç      À?Né   c                    s¨   t t| ƒ ¡  |d u r&t|| |d}|r2|d nd| _tj||dddd| _tj||dddd| _tj||dddd| _	tj||dddd| _
t |¡| _|  ¡  d S )N©Zdivisorg      à¿g      ð?r   T)Úkernel_sizeZstrideÚbias)Úsuperr   Ú__init__r   Úscaler   ÚConv2dÚtÚpÚgÚzÚBatchNorm2dÚnormÚreset_parameters)ÚselfÚin_channelsZ	use_scaleÚrd_ratioÚrd_channelsÚ
rd_divisorÚkwargs©Ú	__class__© úc/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/non_local_attn.pyr      s    zNonLocalAttn.__init__c                 C   sÒ   |}|   |¡}|  |¡}|  |¡}| ¡ \}}}}	| ||d¡ ddd¡}| ||d¡}| ||d¡ ddd¡}t ||¡| j }
t	j
|
dd}
t |
|¡}| ddd¡ ||||	¡}|  |¡}|  |¡| }|S )Néÿÿÿÿr   é   r   ©Údim)r   r   r   ÚsizeÚviewZpermuteÚtorchZbmmr   ÚFZsoftmaxZreshaper   r   )r   ÚxZshortcutr   r   r   ÚBÚCÚHÚWZattr!   r!   r"   Úforward#   s    



zNonLocalAttn.forwardc                 C   s¶   |   ¡ D ]¨\}}t|tjƒrVtjj|jddd tt| 	¡ ƒƒdkr°tj 
|jd¡ qt|tjƒr„tj 
|jd¡ tj 
|jd¡ qt|tjƒrtj 
|jd¡ tj 
|jd¡ qd S )NZfan_outZrelu)ÚmodeZnonlinearityr   g        r   )Znamed_modulesÚ
isinstancer   r   ÚinitZkaiming_normal_ZweightÚlenÚlistÚ
parametersZ	constant_r   r   Z	GroupNorm)r   ÚnameÚmr!   r!   r"   r   9   s    ÿzNonLocalAttn.reset_parameters)Tr	   Nr
   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r0   r   Ú__classcell__r!   r!   r   r"   r      s   r   c                       s<   e Zd Zejejf‡ fdd„	Zedœdd„Zdd„ Z	‡  Z
S )ÚBilinearAttnTransformc                    sˆ   t t| ƒ ¡  t||d||d| _tj||| | |dfd| _tj||| | d|fd| _t||d||d| _	|| _
|| _|| _d S )Nr   ©Ú	act_layerÚ
norm_layer)r   )r   r>   r   r   Úconv1r   r   Úconv_pÚconv_qÚconv2Ú
block_sizeÚgroupsr   )r   r   rF   rG   r@   rA   r   r!   r"   r   J   s    zBilinearAttnTransform.__init__)r   c                 C   s¼   |j \}}}}t||kdƒ |dkr(|S | || ddd¡}|tj|||j|jd }| || ||||¡}tjtj|ddddd}tjtj|ddddd}| |||| || ¡}|S )	NÚ r   r#   )ÚdtypeÚdevicer%   é   r$   é   )	Úshaper   r(   r)   ZeyerI   rJ   ÚcatÚsplit)r   r+   r   r,   r-   rF   Zblock_size1r!   r!   r"   Ú
resize_matU   s    z BilinearAttnTransform.resize_matc                 C   sº  t |jd | j dkdƒ t |jd | j dkdƒ |j\}}}}|  |¡}t || jdf¡}t |d| jf¡}|  |¡ || j| j| j¡ 	¡ }	|  
|¡ || j| j| j¡ 	¡ }
|	|	jddd }	|
|
jd	dd }
|	 || jd| j| j¡ | d¡| j|| j | j| j¡ ¡ }	|	 ||| j| j¡}	|
 || jd| j| j¡ | d¡| j|| j | j| j¡ ¡ }
|
 ||| j| j¡}
|  |	|| j ¡}	|  |
|| j ¡}
|	 |¡}| |
¡}|  |¡}|S )
Nr#   r   rH   éþÿÿÿr   rK   T)r&   Zkeepdimr$   )r   rM   rF   rB   r*   Zadaptive_max_pool2drC   r(   rG   ZsigmoidrD   ÚsumÚexpandr'   Ú
contiguousrP   ÚmatmulrE   )r   r+   r,   r-   r.   r/   ÚoutÚrpÚcpr   ÚqÚyr!   r!   r"   r0   b   s8    
  ÿÿÿÿ


zBilinearAttnTransform.forward)r9   r:   r;   r   ÚReLUr   r   ÚintrP   r0   r=   r!   r!   r   r"   r>   H   s   r>   c                       s>   e Zd ZdZddddddejejf‡ fdd	„	Zd
d„ Z‡  Z	S )ÚBatNonLocalAttnzT BAT
    Adapted from: https://github.com/BA-Transform/BAT-Image-Classification
    é   r$   g      Ð?Nr
   gš™™™™™É?c
                    sp   t ƒ  ¡  |d u r"t|| |d}t||d||	d| _t|||||	d| _t||d||	d| _tj	|d| _
d S )Nr   r   r?   )r   )r   r   r   r   rB   r>   ÚbarE   r   Z	Dropout2dÚdropout)r   r   rF   rG   r   r   r   Z	drop_rater@   rA   Ú_r   r!   r"   r      s    
zBatNonLocalAttn.__init__c                 C   s0   |   |¡}|  |¡}|  |¡}|  |¡}|| S )N)rB   r_   rE   r`   )r   r+   ZxlrZ   r!   r!   r"   r0   Œ   s
    



zBatNonLocalAttn.forward)
r9   r:   r;   r<   r   r[   r   r   r0   r=   r!   r!   r   r"   r]   |   s
   

þr]   )r<   r)   r   Ztorch.nnr   r*   Zconv_bn_actr   Zhelpersr   Ztrace_utilsr   ÚModuler   r>   r]   r!   r!   r!   r"   Ú<module>   s   84