a
    d
D                     @   s$  d Z ddlmZmZmZmZ ddlZddlmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ d%ddZG dd dejZG dd dejZd&ddZG dd dejjZdd Zdd Zdd ZG dd dej Z!G dd  d ej Z"G d!d" d"ej#Z$G d#d$ d$ej#Z%dS )'aw   Normalization + Activation Layers

Provides Norm+Act fns for standard PyTorch norm layers such as
* BatchNorm
* GroupNorm
* LayerNorm

This allows swapping with alternative layers that are natively both norm + act such as
* EvoNorm (evo_norm.py)
* FilterResponseNorm (filter_response_norm.py)
* InplaceABN (inplace_abn.py)

Hacked together by / Copyright 2022 Ross Wightman
    )UnionListOptionalAnyN)nn)
functional)FrozenBatchNorm2d   )get_act_layer)is_fast_normfast_group_normfast_layer_norm)_assertFTc                 C   sD   t | } |pi }| d ur8|r8|r(||d< | f i |}nt }|S )Ninplace)r
   r   Identity)	act_layer
act_kwargsr   	apply_actact r   ]/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/norm_act.py_create_act   s    r   c                       sB   e Zd ZdZdddddejdddddf fdd	Zdd	 Z  ZS )
BatchNormAct2dzBatchNorm + Activation

    This module performs BatchNorm + Activation in a manner that will remain backwards
    compatible with weights trained with separate bn, act. This is why we inherit from BN
    instead of composing it as a .bn member.
    h㈵>g?TNc              
      s   z0||d}t t| j|f||||d| W n, ty\   t t| j|||||d Y n0 |
d url|
 nt | _t|||	|d| _d S )N)devicedtype)epsmomentumaffinetrack_running_statsr   r   r   )	superr   __init__	TypeErrorr   r   dropr   r   )selfnum_featuresr   r   r   r   r   r   r   r   
drop_layerr   r   Zfactory_kwargs	__class__r   r   r"   .   s,    



zBatchNormAct2d.__init__c              
   C   s   t |jdkd|j d | jd u r*d}n| j}| jrr| jrr| jd urr| jd | jd u rldt| j }n| j}| jr~d}n| jd u o| j	d u }t
|| jr| jr| jnd | jr| jr| j	nd | j| j||| j}| |}| |}|S )N   zexpected 4D input (got zD input)g        r	   g      ?T)r   ndimr   Ztrainingr   num_batches_trackedZadd_floatrunning_meanrunning_varFZ
batch_normweightbiasr   r$   r   )r%   xZexponential_average_factorZbn_trainingr   r   r   forwardS   s4    




zBatchNormAct2d.forward)	__name__
__module____qualname____doc__r   ReLUr"   r4   __classcell__r   r   r(   r   r   '   s   	%r   c                       s(   e Zd Zejejd fddZ  ZS )SyncBatchNormActr3   returnc                    s8   t  |}t| dr | |}t| dr4| |}|S )Nr$   r   )r!   r4   hasattrr$   r   r%   r3   r(   r   r   r4      s    



zSyncBatchNormAct.forward)r5   r6   r7   torchTensorr4   r:   r   r   r(   r   r;      s   r;   c                 C   s  | }t | tjjjjrt | trPt| j| j	| j
| j| j|d}| j|_| j|_n tj| j| j	| j
| j| j|}| jrt   | j|_| j|_W d    n1 s0    Y  | j|_| j|_| j|_t| dr| j|_|  D ]\}}||t|| q~ |S )N)process_groupqconfig)
isinstancer@   r   modules	batchnormZ
_BatchNormr   r;   r&   r   r   r   r   r   r$   SyncBatchNormZno_gradr1   r2   r.   r/   r,   r>   rC   named_children
add_moduleconvert_sync_batchnorm)modulerB   Zmodule_outputnamechildr   r   r   rJ      sD    
	

&
rJ   c                       s   e Zd ZdZddejdddfeed fddZe	e
e	eee
 ee
 ee
 d fd	d
ZejejdddZe
dddZ  ZS )FrozenBatchNormAct2da$  
    BatchNormAct2d where the batch statistics and the affine parameters are fixed

    Args:
        num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)``
        eps (float): a value added to the denominator for numerical stability. Default: 1e-5
    r   TN)r&   r   c                    s   t    || _| dt| | dt| | dt| | dt| |d urf| nt | _	t
||||d| _d S )Nr1   r2   r.   r/   r    )r!   r"   r   Zregister_bufferr@   Zoneszerosr   r   r$   r   r   )r%   r&   r   r   r   r   r   r'   r(   r   r   r"      s    

zFrozenBatchNormAct2d.__init__)
state_dictprefixlocal_metadatastrictmissing_keysunexpected_keys
error_msgsc           	   	      s2   |d }||v r||= t  ||||||| d S )Nr,   )r!   _load_from_state_dict)	r%   rP   rQ   rR   rS   rT   rU   rV   Znum_batches_tracked_keyr(   r   r   rW      s    
z*FrozenBatchNormAct2d._load_from_state_dictr<   c                 C   s   | j dddd}| jdddd}| jdddd}| jdddd}||| j   }|||  }|| | }| | |}|S )Nr	   )	r1   Zreshaper2   r/   r.   r   Zrsqrtr   r$   )r%   r3   wbrvZrmZscaler2   r   r   r   r4      s    zFrozenBatchNormAct2d.forward)r=   c                 C   s,   | j j d| jjd  d| j d| j dS )N(r   z, eps=z, act=))r)   r5   r1   shaper   r   )r%   r   r   r   __repr__   s    zFrozenBatchNormAct2d.__repr__)r5   r6   r7   r8   r   r9   intr-   r"   dictstrboolr   rW   r@   rA   r4   r_   r:   r   r   r(   r   rN      s*   rN   c                 C   sL  | }t | ttfrt| j}| j|_| j|_| jrZ| jj 	 |j_| j
j 	 |j
_| jj|j_| jj|j_| j|_| j|_| j|_nt | tjjjjtjjjjfrt| j}| j|_| j|_| jr| jj 	 |j_| j
j 	 |j
_| jj|j_| jj|j_| j|_n2|  D ](\}}t|}||ur||| q|S )a  
    Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers
    of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )rD   r   r;   rN   r&   r   r1   dataclonedetachr2   r.   r/   r   r$   r   r@   r   rE   rF   BatchNorm2drG   r   rH   freeze_batch_norm_2drI   rK   resrL   rM   	new_childr   r   r   rh      s8    

 


rh   c                 C   s  | }t | trxt| j}| jrF| jj  |j_| j	j  |j	_| j
j|j
_| jj|j_| j|_| j|_| j|_nt | trtj| j}| jr| jj  |j_| j	j  |j	_| j
j|j
_| jj|j_| j|_n.|  D ]$\}}t|}||ur||| q|S )a  
    Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance
    of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked
    recursively and submodules are converted in place.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )rD   rN   r   r&   r   r1   rd   re   rf   r2   r.   r/   r   r$   r   r   r@   r   rg   rH   unfreeze_batch_norm_2drI   ri   r   r   r   rl   *  s0    




rl   c                 C   s    |r| | dksJ | | S |S )Nr   r   )num_channels
num_groups
group_sizer   r   r   _num_groupsS  s    rp   c                	       s:   e Zd Zdddddejdddf	 fdd	Zdd Z  ZS )	GroupNormAct    r   TNc                    sV   t t| jt||||||d |
d ur.|
 nt | _t|||	|d| _t	 | _
d S )Nr   r   r    )r!   rq   r"   rp   r   r   r$   r   r   r   
_fast_norm)r%   rm   rn   r   r   ro   r   r   r   r   r'   r(   r   r   r"   \  s    

zGroupNormAct.__init__c                 C   sR   | j r t|| j| j| j| j}nt|| j| j| j| j}| |}| 	|}|S N
rt   r   rn   r1   r2   r   r0   Z
group_normr$   r   r?   r   r   r   r4   t  s    

zGroupNormAct.forwardr5   r6   r7   r   r9   r"   r4   r:   r   r   r(   r   rq   Z  s   rq   c                       s6   e Zd Zdddejdddf fdd	Zdd Z  ZS )GroupNorm1Actr   TNc	           	         sN   t t| jd|||d |d ur&| nt | _t||||d| _t | _	d S )Nr	   rs   r    )
r!   rx   r"   r   r   r$   r   r   r   rt   	r%   rm   r   r   r   r   r   r   r'   r(   r   r   r"     s    zGroupNorm1Act.__init__c                 C   sR   | j r t|| j| j| j| j}nt|| j| j| j| j}| |}| 	|}|S ru   rv   r?   r   r   r   r4     s    

zGroupNorm1Act.forwardrw   r   r   r(   r   rx   ~  s   rx   c                       sL   e Zd Zdddejdddfeeee ej	f d fddZ
dd Z  ZS )	LayerNormActr   TN)normalization_shapec	           	         sT   t t| j|||d |d ur$| nt | _t|}t||||d| _t	 | _
d S N)r   Zelementwise_affiner    )r!   rz   r"   r   r   r$   r
   r   r   r   rt   )	r%   r{   r   r   r   r   r   r   r'   r(   r   r   r"     s
    zLayerNormAct.__init__c                 C   sR   | j r t|| j| j| j| j}nt|| j| j| j| j}| |}| 	|}|S ru   )
rt   r   normalized_shaper1   r2   r   r0   
layer_normr$   r   r?   r   r   r   r4     s    

zLayerNormAct.forward)r5   r6   r7   r   r9   r   r`   r   r@   Sizer"   r4   r:   r   r   r(   r   rz     s   rz   c                       s6   e Zd Zdddejdddf fdd	Zdd Z  ZS )LayerNormAct2dr   TNc	           	         sL   t t| j|||d |d ur$| nt | _t||||d| _t | _	d S r|   )
r!   r   r"   r   r   r$   r   r   r   rt   ry   r(   r   r   r"     s    zLayerNormAct2d.__init__c                 C   sr   | dddd}| jr0t|| j| j| j| j}nt|| j| j| j| j}| dddd}| 	|}| 
|}|S )Nr         r	   )Zpermutert   r   r}   r1   r2   r   r0   r~   r$   r   r?   r   r   r   r4     s    

zLayerNormAct2d.forwardrw   r   r   r(   r   r     s   r   )NFT)N)&r8   typingr   r   r   r   r@   r   Ztorch.nnr   r0   Ztorchvision.ops.miscr   Z
create_actr
   Z	fast_normr   r   r   Ztrace_utilsr   r   rg   r   rG   r;   rJ   ModulerN   rh   rl   rp   Z	GroupNormrq   rx   Z	LayerNormrz   r   r   r   r   r   <module>   s(   
_
*@,)$