a
    d                     @   s  d Z ddlZddlmZ ddlmZ ejjdd Zejjdd Z	G d	d
 d
ej
jZd9ddZG dd dejZejjdd Zejjdd ZG dd dej
jZd:ddZG dd dejZejjd;edddZejjdd ZG dd  d ej
jZd<edd!d"ZG d#d$ d$ejZejjd%d& Zejjd'd( ZG d)d* d*ej
jZd=d+d,ZG d-d. d.ejZejjd/d0 Z ejjd1d2 Z!G d3d4 d4ej
jZ"d>edd5d6Z#G d7d8 d8ejZ$dS )?a   Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use either
the JIT or basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalc                 C   s   |  t| S N)multorchsigmoidx r
   c/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/activations_me.pyswish_jit_fwd   s    r   c                 C   s"   t | }||d| d|     S N   )r   r   )r	   grad_output	x_sigmoidr
   r
   r   swish_jit_bwd   s    
r   c                   @   s4   e Zd ZdZedd Zedd Zedd ZdS )	SwishJitAutoFnz torch.jit.script optimised Swish w/ memory-efficient checkpoint
    Inspired by conversation btw Jeremy Howard & Adam Pazske
    https://twitter.com/jeremyphoward/status/1188251041835315200
    c                 C   s   |  d||  d|S )NMulZSigmoid)op)gr	   r
   r
   r   symbolic!   s    zSwishJitAutoFn.symbolicc                 C   s   |  | t|S r   )save_for_backwardr   ctxr	   r
   r
   r   forward%   s    
zSwishJitAutoFn.forwardc                 C   s   | j d }t||S Nr   )saved_tensorsr   r   r   r	   r
   r
   r   backward*   s    
zSwishJitAutoFn.backwardN)__name__
__module____qualname____doc__staticmethodr   r   r   r
   r
   r
   r   r      s   

r   Fc                 C   s
   t | S r   r   applyr	   inplacer
   r
   r   swish_me0   s    r(   c                       s,   e Zd Zded fddZdd Z  ZS )SwishMeFr'   c                    s   t t|   d S r   )superr)   __init__selfr'   	__class__r
   r   r,   5   s    zSwishMe.__init__c                 C   s
   t |S r   r$   r.   r	   r
   r
   r   r   8   s    zSwishMe.forward)Fr   r    r!   boolr,   r   __classcell__r
   r
   r/   r   r)   4   s   r)   c                 C   s   |  tt| S r   )r   r   tanhFsoftplusr   r
   r
   r   mish_jit_fwd<   s    r8   c                 C   s6   t | }t|  }||| | d||    S r   )r   r   r6   r7   r5   r   )r	   r   r   Z	x_tanh_spr
   r
   r   mish_jit_bwdA   s    
r9   c                   @   s(   e Zd ZdZedd Zedd ZdS )MishJitAutoFnz Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    A memory efficient, jit scripted variant of Mish
    c                 C   s   |  | t|S r   )r   r8   r   r
   r
   r   r   L   s    
zMishJitAutoFn.forwardc                 C   s   | j d }t||S r   )r   r9   r   r
   r
   r   r   Q   s    
zMishJitAutoFn.backwardNr   r    r!   r"   r#   r   r   r
   r
   r
   r   r:   H   s
   
r:   c                 C   s
   t | S r   r:   r%   r&   r
   r
   r   mish_meW   s    r=   c                       s,   e Zd Zded fddZdd Z  ZS )MishMeFr*   c                    s   t t|   d S r   )r+   r>   r,   r-   r/   r
   r   r,   \   s    zMishMe.__init__c                 C   s
   t |S r   r<   r1   r
   r
   r   r   _   s    zMishMe.forward)Fr2   r
   r
   r/   r   r>   [   s   r>   r*   c                 C   s   | d j ddddS N   r      minmax      @clampdivr&   r
   r
   r   hard_sigmoid_jit_fwdc   s    rI   c                 C   s&   t | | dk| dk@  d }|| S )N            @rE   )r   	ones_liker	   r   mr
   r
   r   hard_sigmoid_jit_bwdh   s    rO   c                   @   s$   e Zd Zedd Zedd ZdS )HardSigmoidJitAutoFnc                 C   s   |  | t|S r   )r   rI   r   r
   r
   r   r   o   s    
zHardSigmoidJitAutoFn.forwardc                 C   s   | j d }t||S r   )r   rO   r   r
   r
   r   r   t   s    
zHardSigmoidJitAutoFn.backwardN)r   r    r!   r#   r   r   r
   r
   r
   r   rP   n   s   
rP   c                 C   s
   t | S r   rP   r%   r&   r
   r
   r   hard_sigmoid_mez   s    rR   c                       s,   e Zd Zded fddZdd Z  ZS )HardSigmoidMeFr*   c                    s   t t|   d S r   )r+   rS   r,   r-   r/   r
   r   r,      s    zHardSigmoidMe.__init__c                 C   s
   t |S r   rQ   r1   r
   r
   r   r      s    zHardSigmoidMe.forward)Fr2   r
   r
   r/   r   rS   ~   s   rS   c                 C   s   | | d j dddd S r?   rF   r   r
   r
   r   hard_swish_jit_fwd   s    rT   c                 C   s<   t | | dk }t | dk| dk@ | d d |}|| S )NrK   rJ         ?r   rL   whererM   r
   r
   r   hard_swish_jit_bwd   s    "rX   c                   @   s4   e Zd ZdZedd Zedd Zedd ZdS )	HardSwishJitAutoFnz5A memory efficient, jit-scripted HardSwish activationc                 C   s   |  | t|S r   )r   rT   r   r
   r
   r   r      s    
zHardSwishJitAutoFn.forwardc                 C   s   | j d }t||S r   )r   rX   r   r
   r
   r   r      s    
zHardSwishJitAutoFn.backwardc                 C   s   |  d|| j dtjdtjdd}|  d|| j dtjdtjdd| j dtjdtjdd}|  d	|| j dtjdtjdd}|  d
||S )NAddConstantr@   )Zdtype)Zvalue_tZClipr   rA   Divr   )r   r   Ztensorfloat)r   r.   inputZ	hardtanh_r
   r
   r   r      s    $<$zHardSwishJitAutoFn.symbolicN)r   r    r!   r"   r#   r   r   r   r
   r
   r
   r   rY      s   

rY   c                 C   s
   t | S r   rY   r%   r&   r
   r
   r   hard_swish_me   s    r`   c                       s,   e Zd Zded fddZdd Z  ZS )HardSwishMeFr*   c                    s   t t|   d S r   )r+   ra   r,   r-   r/   r
   r   r,      s    zHardSwishMe.__init__c                 C   s
   t |S r   r_   r1   r
   r
   r   r      s    zHardSwishMe.forward)Fr2   r
   r
   r/   r   ra      s   ra   c                 C   s   d|  | d j ddd S )NrU      r   rB   )rG   r   r
   r
   r   hard_mish_jit_fwd   s    rc   c                 C   s8   t | | dk }t | dk| dk@ | d |}|| S )Ng       g        g      ?rV   rM   r
   r
   r   hard_mish_jit_bwd   s    rd   c                   @   s(   e Zd ZdZedd Zedd ZdS )HardMishJitAutoFnz A memory efficient, jit scripted variant of Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    c                 C   s   |  | t|S r   )r   rc   r   r
   r
   r   r      s    
zHardMishJitAutoFn.forwardc                 C   s   | j d }t||S r   )r   rd   r   r
   r
   r   r      s    
zHardMishJitAutoFn.backwardNr;   r
   r
   r
   r   re      s
   
re   c                 C   s
   t | S r   re   r%   r&   r
   r
   r   hard_mish_me   s    rg   c                       s,   e Zd Zded fddZdd Z  ZS )
HardMishMeFr*   c                    s   t t|   d S r   )r+   rh   r,   r-   r/   r
   r   r,      s    zHardMishMe.__init__c                 C   s
   t |S r   rf   r1   r
   r
   r   r      s    zHardMishMe.forward)Fr2   r
   r
   r/   r   rh      s   rh   )F)F)F)F)F)F)%r"   r   r   Ztorch.nnr   r6   Zjitscriptr   r   ZautogradZFunctionr   r(   Moduler)   r8   r9   r:   r=   r>   r3   rI   rO   rP   rR   rS   rT   rX   rY   r`   ra   rc   rd   re   rg   rh   r
   r
   r
   r   <module>   sL   











