a
    þd	  ã                   @   sl   d Z ddlmZ ddlm  mZ ddlmZmZ ddl	m
Z
 ddlmZ ddlmZ G d	d
„ d
ejƒZdS )a   Global Context Attention Block

Paper: `GCNet: Non-local Networks Meet Squeeze-Excitation Networks and Beyond`
    - https://arxiv.org/abs/1904.11492

Official code consulted as reference: https://github.com/xvjiarui/GCNet

Hacked together by / Copyright 2021 Ross Wightman
é    )ÚnnNé   )Úcreate_act_layerÚget_act_layer)Úmake_divisible)ÚConvMlp)ÚLayerNorm2dc                	       sB   e Zd Zdddddddejdf	‡ fdd„	Zd	d
„ Zdd„ Z‡  ZS )ÚGlobalContextTFg      À?Nr   Zsigmoidc                    s¤   t t| ƒ ¡  t|	ƒ}	|r,tj|ddddnd | _|d u rLt|| |dd}|rdt|||	t	d| _
nd | _
|r‚t|||	t	d| _nd | _t|
ƒ| _|| _|  ¡  d S )Nr   T)Zkernel_sizeZbiasg        )Zround_limit)Ú	act_layerZ
norm_layer)Úsuperr	   Ú__init__r   r   ZConv2dÚ	conv_attnr   r   r   Úmlp_addÚ	mlp_scaler   ÚgateÚinit_last_zeroÚreset_parameters)ÚselfZchannelsZuse_attnZfuse_addZ
fuse_scaler   Zrd_ratioZrd_channelsZ
rd_divisorr
   Z
gate_layer©Ú	__class__© úc/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/global_context.pyr      s    
zGlobalContext.__init__c                 C   s@   | j d ur tjj| j jddd | jd ur<tj | jjj¡ d S )NZfan_inZrelu)ÚmodeZnonlinearity)r   r   ÚinitZkaiming_normal_Zweightr   Zzeros_Zfc2)r   r   r   r   r   +   s    

zGlobalContext.reset_parametersc           	      C   sÂ   |j \}}}}| jd urr|  |¡ |d|| ¡}tj|dd d¡}| |||| ¡ d¡| }| ||dd¡}n|jddd}| jd ur¢|  |¡}||  	|¡ }| j
d ur¾|  
|¡}|| }|S )Nr   éÿÿÿÿ)Údimé   )é   r   T)r   Zkeepdim)Úshaper   ZreshapeÚFZsoftmaxZ	unsqueezeÚviewÚmeanr   r   r   )	r   ÚxÚBÚCÚHÚWZattnÚcontextZmlp_xr   r   r   Úforward1   s    




zGlobalContext.forward)	Ú__name__Ú
__module__Ú__qualname__r   ZReLUr   r   r(   Ú__classcell__r   r   r   r   r	      s
   ÿr	   )Ú__doc__Ztorchr   Ztorch.nn.functionalZ
functionalr   Z
create_actr   r   Zhelpersr   Zmlpr   Znormr   ÚModuler	   r   r   r   r   Ú<module>   s   	