a
    d                     @   st   d Z ddlZddlmZ ddlm  mZ ddlmZm	Z	 ddl
m
Z
 ddlmZ ddlmZ G d	d
 d
ejZdS )a   Gather-Excite Attention Block

Paper: `Gather-Excite: Exploiting Feature Context in CNNs` - https://arxiv.org/abs/1810.12348

Official code here, but it's only partial impl in Caffe: https://github.com/hujie-frank/GENet

I've tried to support all of the extent both w/ and w/o params. I don't believe I've seen another
impl that covers all of the cases.

NOTE: extent=0 + extra_params=False is equivalent to Squeeze-and-Excitation

Hacked together by / Copyright 2021 Ross Wightman
    N)nn   )create_act_layerget_act_layer)create_conv2d)make_divisible)ConvMlpc                       sD   e Zd ZdZddddddddejejdf fd	d
	Zdd Z  Z	S )GatherExcitez$ Gather-Excite Attention Module
    NFr   Tg      ?r   Zsigmoidc                    s  t t|   |	| _t|
}
|| _|r t | _|dkr|d usJJ d| j	dt
|||ddd |r~| j	dt| n|d dksJ tt|}t|D ]v}| j	d	|d  t
||d
ddd |r| j	d|d  t| ||d kr| j	d|d  |
dd qnJd | _| jdkr@d| _d| _n*|d dksRJ | jd d | _| j| _|st|| |dd}|rt|||
dnt | _t|| _d S )Nr   zBspatial feature size must be specified for global extent w/ paramsZconv1r   T)kernel_sizestrideZ	depthwiseZnorm1   conv   ZnormZact)Zinplaceg        )Zround_limit)	act_layer)superr	   __init__add_maxpoolr   extentr   Z
SequentialgatherZ
add_moduler   BatchNorm2dintmathlog2rangegkgsr   r   ZIdentitymlpr   gate)selfZchannelsZ	feat_sizeZextra_paramsr   Zuse_mlpZrd_ratioZrd_channelsZ
rd_divisorr   r   Z
norm_layerZ
gate_layerZnum_convi	__class__ b/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/gather_excite.pyr      sF    
"zGatherExcite.__init__c                 C   s   |j dd  }| jd ur$| |}n| jdkr^|jddd}| jrd| d|jddd  }nLtj|| j| j	| jd d	d
}| jrd| dtj
|| j| j	| jd d  }| |}|j d dks|j d dkrtj||d}|| | S )Nr   )r   r   T)ZdimZkeepdimsg      ?)Zkeepdimr   F)r
   r   paddingZcount_include_pad)r
   r   r%   r   )size)shaper   r   meanr   ZamaxFZ
avg_pool2dr   r   Z
max_pool2dr   Zinterpolater   )r   xr'   Zx_ger"   r"   r#   forwardF   s     

(
zGatherExcite.forward)
__name__
__module____qualname____doc__r   ZReLUr   r   r,   __classcell__r"   r"   r    r#   r	      s   
*r	   )r0   r   Ztorchr   Ztorch.nn.functionalZ
functionalr*   Z
create_actr   r   r   Zhelpersr   r   r   Moduler	   r"   r"   r"   r#   <module>   s   