a
    d&6                     @   s|  d Z ddlmZmZ ddlZddlmZ ddlm  mZ	 ddl
mZ ddlmZ d/edd	d
Zd0edddZd1edddZd2eeee f edddZd3eeedddZd4eeeedddZd5eedddZG dd dejZG dd  d ejZG d!d" d"ejZG d#d$ d$ejZG d%d& d&eZG d'd( d(ejZG d)d* d*eZG d+d, d,ejZ G d-d. d.e Z!dS )6a7   EvoNorm in PyTorch

Based on `Evolving Normalization-Activation Layers` - https://arxiv.org/abs/2004.02967
@inproceedings{NEURIPS2020,
 author = {Liu, Hanxiao and Brock, Andy and Simonyan, Karen and Le, Quoc},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {13539--13550},
 publisher = {Curran Associates, Inc.},
 title = {Evolving Normalization-Activation Layers},
 url = {https://proceedings.neurips.cc/paper/2020/file/9d4c03631b8b0c85ae08bf05eda37d0f-Paper.pdf},
 volume = {33},
 year = {2020}
}

An attempt at getting decent performing EvoNorms running in PyTorch.
While faster than other PyTorch impl, still quite a ways off the built-in BatchNorm
in terms of memory usage and throughput on GPUs.

I'm testing these modules on TPU w/ PyTorch XLA. Promising start but
currently working around some issues with builtin torch/tensor.var/std. Unlike
GPU, similar train speeds for EvoNormS variants and BatchNorm.

Hacked together by / Copyright 2020 Ross Wightman
    )SequenceUnionN   )create_act_layer)_asserth㈵>)epsc                 C   s2   |   jdddd| | j}|| jS )N      FTdimunbiasedkeepdim)floatvaraddsqrttodtypeexpandshapexr   std r   ]/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/evo_norm.pyinstance_std$   s    &r   c                 C   s"   t | dd| }|| jS )Nr	   )r   )
manual_varr   r   r   r   r   r   r   r   instance_std_tpu)   s    r   c                 C   s4   |    jddd| | j}|| jS )Nr	   Tr   r   )	r   squaremeanr   r   r   r   r   r   )r   r   rmsr   r   r   instance_rms/   s    (r$   Fr   diff_sqmc                 C   sR   | j |dd}|r4| |  j |dd||  d}n| | | |  j |dd}|S )NTr    r   )r"   clamp)r   r   r&   Zxmr   r   r   r   r   4   s
    "r       )groupsr   flattenc           
      C   s   | j \}}}}| j}t|| dkd |r^| ||d} |  jdddd| |}	n:| |||| ||} |  jdddd| |}	|		| j ||||S )	Nr    r
   FTr   r
   r      )
r   r   r   reshaper   r   r   r   r   r   )
r   r)   r   r*   BCHWx_dtyper   r   r   r   	group_std>   s    &$r5   )r)   r   r&   r*   c           
      C   s   | j \}}}}t|| dkd |rB| ||d} t| d|d}	n$| |||| ||} t| d|d}	|	| | j ||||S )Nr   r+   r,   r%   r-   )r   r   r/   r   r   r   r   )
r   r)   r   r&   r*   r0   r1   r2   r3   r   r   r   r   group_std_tpuK   s    r6   )r)   r   c           	      C   sz   | j \}}}}t|| dkd | j}| |||| ||} |   jddd| 	|}|
| j ||||S )Nr   r+   r-   Tr    )r   r   r   r/   r   r!   r"   r   sqrt_r   r   )	r   r)   r   r0   r1   r2   r3   r4   r#   r   r   r   	group_rmsZ   s    &r8   c                       s.   e Zd Zd
 fdd	Zdd Zdd	 Z  ZS )EvoNorm2dB0T皙?MbP?c                    sx   t    || _|| _|| _tt|| _	tt
|| _|rTtt|nd | _| dt| |   d S Nrunning_var)super__init__	apply_actmomentumr   nn	Parametertorchonesweightzerosbiasvregister_bufferreset_parametersselfnum_featuresr@   rA   r   _	__class__r   r   r?   d   s    
zEvoNorm2dB0.__init__c                 C   s8   t j| j t j| j | jd ur4t j| j d S NrB   initones_rF   zeros_rH   rI   rM   r   r   r   rK   o   s    
zEvoNorm2dB0.reset_parametersc           	      C   s  t | dkd |j}d}| jd ur| jr| jddd}| |jd  }| j	
| j	d| j  | | j ||d     n| j	}|| j |||}| j||}|| t|| j }||| }|| j|| | j|| S Nr.   expected 4D inputr   r,   r   r   )r   r
   r   F)r   r   r   )r   r   r   rI   trainingr   r   numelr   r=   copy_rA   detachr   r   r7   r   viewZ	expand_asr   maxrF   rH   )	rM   r   r4   v_shaper   nleftrI   rightr   r   r   forwardu   s$    
"zEvoNorm2dB0.forward)Tr:   r;   __name__
__module____qualname__r?   rK   re   __classcell__r   r   rP   r   r9   c   s   r9   c                       s.   e Zd Zd
 fdd	Zdd Zdd	 Z  ZS )EvoNorm2dB1Tr:   r   c                    s^   t    || _|| _|| _tt|| _	tt
|| _| dt| |   d S r<   r>   r?   r@   rA   r   rB   rC   rD   rE   rF   rG   rH   rJ   rK   rL   rP   r   r   r?      s    
zEvoNorm2dB1.__init__c                 C   s    t j| j t j| j d S rR   rB   rT   rU   rF   rV   rH   rW   r   r   r   rK      s    zEvoNorm2dB1.reset_parametersc                 C   s   t | dkd |j}d}| jr| jr| jddd}| |jd  }| j	
| j	d| j  | | j	j| j ||d     n| j	}|||}|| j }|d t|| j }||| }|| j|| | j|| S rX   r   r   r   r@   r[   r   r   r\   r   r=   r]   rA   r^   r   r_   r   r   r7   r$   r`   rF   rH   rM   r   r4   ra   r   rb   rc   rd   r   r   r   re      s$    "zEvoNorm2dB1.forward)Tr:   r   rf   r   r   rP   r   rk      s   
rk   c                       s.   e Zd Zd
 fdd	Zdd Zdd	 Z  ZS )EvoNorm2dB2Tr:   r   c                    s^   t    || _|| _|| _tt|| _	tt
|| _| dt| |   d S r<   rl   rL   rP   r   r   r?      s    
zEvoNorm2dB2.__init__c                 C   s    t j| j t j| j d S rR   rm   rW   r   r   r   rK      s    zEvoNorm2dB2.reset_parametersc                 C   s   t | dkd |j}d}| jr| jr| jddd}| |jd  }| j	
| j	d| j  | | j	j| j ||d     n| j	}|||}|| j }t|| j| }||| }|| j|| | j|| S rX   rn   ro   r   r   r   re      s$    "zEvoNorm2dB2.forward)Tr:   r   rf   r   r   rP   r   rp      s   
rp   c                       s.   e Zd Zd fdd	Zdd Zd	d
 Z  ZS )EvoNorm2dS0r(   NTr   c                    s   t    || _|r0|| dks$J || | _n|| _|| _tt|| _	tt
|| _|rttt|nd | _|   d S Nr   )r>   r?   r@   r)   r   rB   rC   rD   rE   rF   rG   rH   rI   rK   rM   rN   r)   
group_sizer@   r   rO   rP   r   r   r?      s    
zEvoNorm2dS0.__init__c                 C   s8   t j| j t j| j | jd ur4t j| j d S rR   rS   rW   r   r   r   rK      s    
zEvoNorm2dS0.reset_parametersc                 C   s   t | dkd |j}d}| jd urX| j||}|||   t|| j| j	 }|| j
|| | j|| S Nr.   rY   rZ   )r   r   r   rI   r_   r   sigmoidr5   r)   r   rF   rH   )rM   r   r4   ra   rI   r   r   r   re      s    
 zEvoNorm2dS0.forward)r(   NTr   rf   r   r   rP   r   rq      s   rq   c                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
EvoNorm2dS0ar(   NTr;   c                    s   t  j|||||d d S )N)r)   rt   r@   r   r>   r?   rs   rP   r   r   r?      s    
zEvoNorm2dS0a.__init__c                 C   s   t | dkd |j}d}t|| j| j}| jd urX| j||}||| 	  }|| }|| j
|| | j|| S ru   )r   r   r   r5   r)   r   rI   r_   r   rv   rF   rH   )rM   r   r4   ra   drI   r   r   r   re      s    
zEvoNorm2dS0a.forward)r(   NTr;   rg   rh   ri   r?   re   rj   r   r   rP   r   rw      s   rw   c                       s.   e Zd Zd fdd	Zdd Zd	d
 Z  ZS )EvoNorm2dS1r(   NTr   c                    s   t    |ptj}|| _|d ur2|r2t|| _n
t | _|r\|| dksPJ || | _n|| _|| _	d| _
tt|| _tt|| _|   d S )Nr   F)r>   r?   rB   SiLUr@   r   actIdentityr)   r   Zpre_act_normrC   rD   rE   rF   rG   rH   rK   rM   rN   r)   rt   r@   	act_layerr   rO   rP   r   r   r?     s    


zEvoNorm2dS1.__init__c                 C   s    t j| j t j| j d S rR   rm   rW   r   r   r   rK     s    zEvoNorm2dS1.reset_parametersc                 C   sd   t | dkd |j}d}| jr<| |t|| j| j }|| j	|
| | j	|
| S ru   )r   r   r   r@   r}   r5   r)   r   rF   r_   r   rH   rM   r   r4   ra   r   r   r   re     s    zEvoNorm2dS1.forward)r(   NTNr   rf   r   r   rP   r   r{      s
     r{   c                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
EvoNorm2dS1ar(   NTr;   c                    s   t  j||||||d d S N)r)   rt   r@   r   r   rx   r   rP   r   r   r?   $  s    zEvoNorm2dS1a.__init__c                 C   s^   t | dkd |j}d}| |t|| j| j }|| j|	| | j
|	| S ru   )r   r   r   r}   r5   r)   r   rF   r_   r   rH   r   r   r   r   re   *  s
    zEvoNorm2dS1a.forward)r(   NTNr;   rz   r   r   rP   r   r   #  s     r   c                       s.   e Zd Zd fdd	Zdd Zd	d
 Z  ZS )EvoNorm2dS2r(   NTr   c                    s   t    |ptj}|| _|d ur2|r2t|| _n
t | _|r\|| dksPJ || | _n|| _|| _	t
t|| _t
t|| _|   d S rr   )r>   r?   rB   r|   r@   r   r}   r~   r)   r   rC   rD   rE   rF   rG   rH   rK   r   rP   r   r   r?   3  s    


zEvoNorm2dS2.__init__c                 C   s    t j| j t j| j d S rR   rm   rW   r   r   r   rK   G  s    zEvoNorm2dS2.reset_parametersc                 C   sd   t | dkd |j}d}| jr<| |t|| j| j }|| j	|
| | j	|
| S ru   )r   r   r   r@   r}   r8   r)   r   rF   r_   r   rH   r   r   r   r   re   K  s    zEvoNorm2dS2.forward)r(   NTNr   rf   r   r   rP   r   r   2  s
     r   c                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
EvoNorm2dS2ar(   NTr;   c                    s   t  j||||||d d S r   rx   r   rP   r   r   r?   U  s    zEvoNorm2dS2a.__init__c                 C   s^   t | dkd |j}d}| |t|| j| j }|| j|	| | j
|	| S ru   )r   r   r   r}   r8   r)   r   rF   r_   r   rH   r   r   r   r   re   [  s
    zEvoNorm2dS2a.forward)r(   NTNr;   rz   r   r   rP   r   r   T  s     r   )r   )r   )r   )F)r(   r   F)r(   r   FF)r(   r   )"__doc__typingr   r   rD   Ztorch.nnrB   Ztorch.nn.functionalZ
functionalFZ
create_actr   Ztrace_utilsr   r   r   r   r$   intboolr   r5   r6   r8   Moduler9   rk   rp   rq   rw   r{   r   r   r   r   r   r   r   <module>   s,   
	'###"