a
    d	                     @   s   d Z ddlZddlmZ ddlmZ ejjdedddZ	ejjded	d
dZ
G dd dejZG dd dejZejjdedddZG dd dejZejjdedddZG dd dejZejjd edddZG dd dejZdS )!a   Activations

A collection of jit-scripted activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

All jit scripted activations are lacking in-place variations on purpose, scripted kernel fusion does not
currently work across in-place op boundaries, thus performance is equal to or less than the non-scripted
versions if they contain in-place ops.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalFinplacec                 C   s   |  |  S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )mulZsigmoidxr    r	   d/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/activations_jit.py	swish_jit   s    r   )_inplacec                 C   s   |  t|  S )ziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    )r   FZsoftplustanh)r   r   r	   r	   r
   mish_jit   s    r   c                       s,   e Zd Zded fddZdd Z  ZS )SwishJitFr   c                    s   t t|   d S N)superr   __init__selfr   	__class__r	   r
   r   !   s    zSwishJit.__init__c                 C   s   t |S r   )r   r   r   r	   r	   r
   forward$   s    zSwishJit.forward)F__name__
__module____qualname__boolr   r   __classcell__r	   r	   r   r
   r       s   r   c                       s,   e Zd Zded fddZdd Z  ZS )MishJitFr   c                    s   t t|   d S r   )r   r    r   r   r   r	   r
   r   )   s    zMishJit.__init__c                 C   s   t |S r   )r   r   r	   r	   r
   r   ,   s    zMishJit.forward)Fr   r	   r	   r   r
   r    (   s   r    c                 C   s   | d j ddddS N   r      minmaxg      @clampdivr   r	   r	   r
   hard_sigmoid_jit0   s    r*   c                       s,   e Zd Zded fddZdd Z  ZS )HardSigmoidJitFr   c                    s   t t|   d S r   )r   r+   r   r   r   r	   r
   r   7   s    zHardSigmoidJit.__init__c                 C   s   t |S r   )r*   r   r	   r	   r
   r   :   s    zHardSigmoidJit.forward)Fr   r	   r	   r   r
   r+   6   s   r+   c                 C   s   | | d j dddd S r!   r'   r   r	   r	   r
   hard_swish_jit>   s    r,   c                       s,   e Zd Zded fddZdd Z  ZS )HardSwishJitFr   c                    s   t t|   d S r   )r   r-   r   r   r   r	   r
   r   E   s    zHardSwishJit.__init__c                 C   s   t |S r   )r,   r   r	   r	   r
   r   H   s    zHardSwishJit.forward)Fr   r	   r	   r   r
   r-   D   s   r-   c                 C   s   d|  | d j ddd S )z Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      ?   r   r$   )r(   r   r	   r	   r
   hard_mish_jitL   s    r/   c                       s,   e Zd Zded fddZdd Z  ZS )HardMishJitFr   c                    s   t t|   d S r   )r   r0   r   r   r   r	   r
   r   V   s    zHardMishJit.__init__c                 C   s   t |S r   )r/   r   r	   r	   r
   r   Y   s    zHardMishJit.forward)Fr   r	   r	   r   r
   r0   U   s   r0   )F)F)F)F)F)__doc__Ztorchr   Ztorch.nnr   r   Zjitscriptr   r   r   Moduler   r    r*   r+   r,   r-   r/   r0   r	   r	   r	   r
   <module>   s$   