a
    dt                     @   s|  d Z ddlZddlmZ ddlmZ d.edddZG d	d
 d
ejZ	d/edddZ
G dd dejZd0edddZG dd dejZd1edddZG dd dejZd2edddZG dd dejZd3edddZG dd dejZd4eddd ZG d!d" d"ejZG d#d$ d$ejZd5ejeejd%d&d'ZG d(d) d)ejZd6ejeejd%d*d+ZG d,d- d-ejZdS )7z Activations

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalFinplacec                 C   s    |r|  |  S | |  S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )mul_sigmoidmulxr    r   `/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/activations.pyswish   s    r   c                       s,   e Zd Zded fddZdd Z  ZS )SwishFr   c                    s   t t|   || _d S N)superr   __init__r   selfr   	__class__r   r   r      s    zSwish.__init__c                 C   s   t || jS r   )r   r   r   r
   r   r   r   forward   s    zSwish.forward)F__name__
__module____qualname__boolr   r   __classcell__r   r   r   r   r      s   r   c                 C   s   |  t|  S )zMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    NOTE: I don't have a working inplace variant
    )r   FZsoftplustanhr	   r   r   r   mish   s    r    c                       s0   e Zd ZdZded fddZdd Z  ZS )	MishziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    Fr   c                    s   t t|   d S r   )r   r!   r   r   r   r   r   r   '   s    zMish.__init__c                 C   s   t |S r   )r    r   r   r   r   r   *   s    zMish.forward)F)r   r   r   __doc__r   r   r   r   r   r   r   r   r!   $   s   r!   c                 C   s   |r|   S |  S r   )sigmoid_r   r	   r   r   r   r   .   s    r   c                       s,   e Zd Zded fddZdd Z  ZS )SigmoidFr   c                    s   t t|   || _d S r   )r   r$   r   r   r   r   r   r   r   4   s    zSigmoid.__init__c                 C   s   | j r| S | S r   )r   r#   r   r   r   r   r   r   8   s    zSigmoid.forward)Fr   r   r   r   r   r$   3   s   r$   c                 C   s   |r|   S |  S r   )tanh_r   r	   r   r   r   r   <   s    r   c                       s,   e Zd Zded fddZdd Z  ZS )TanhFr   c                    s   t t|   || _d S r   )r   r&   r   r   r   r   r   r   r   B   s    zTanh.__init__c                 C   s   | j r| S | S r   )r   r%   r   r   r   r   r   r   F   s    zTanh.forward)Fr   r   r   r   r   r&   A   s   r&   c                 C   s,   t | d d}|r"| |S | |S )N      @      @)r   relu6div_r   r   )r
   r   innerr   r   r   
hard_swishJ   s    r,   c                       s,   e Zd Zded fddZdd Z  ZS )	HardSwishFr   c                    s   t t|   || _d S r   )r   r-   r   r   r   r   r   r   r   P   s    zHardSwish.__init__c                 C   s   t || jS r   )r,   r   r   r   r   r   r   T   s    zHardSwish.forward)Fr   r   r   r   r   r-   O   s   r-   c                 C   s2   |r|  ddddS t| d d S d S )Nr'   g        r(   )Zadd_Zclamp_r*   r   r)   r	   r   r   r   hard_sigmoidX   s    r.   c                       s,   e Zd Zded fddZdd Z  ZS )HardSigmoidFr   c                    s   t t|   || _d S r   )r   r/   r   r   r   r   r   r   r   `   s    zHardSigmoid.__init__c                 C   s   t || jS r   )r.   r   r   r   r   r   r   d   s    zHardSigmoid.forward)Fr   r   r   r   r   r/   _   s   r/   c                 C   s>   |r |  d| d jddd S d|  | d jddd S dS )z Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      ?   r   )minmaxN)r   clampr	   r   r   r   	hard_mishh   s    r4   c                       s,   e Zd Zded fddZdd Z  ZS )HardMishFr   c                    s   t t|   || _d S r   )r   r5   r   r   r   r   r   r   r   t   s    zHardMish.__init__c                 C   s   t || jS r   )r4   r   r   r   r   r   r   x   s    zHardMish.forward)Fr   r   r   r   r   r5   s   s   r5   c                       sB   e Zd ZdZdeeedd fddZej	ej	d	d
dZ
  ZS )PReLUz)Applies PReLU (w/ dummy inplace arg)
             ?FN)num_parametersinitr   returnc                    s   t t| j||d d S )N)r9   r:   )r   r6   r   )r   r9   r:   r   r   r   r   r      s    zPReLU.__init__inputr;   c                 C   s   t || jS r   )r   ZpreluZweightr   r=   r   r   r   r      s    zPReLU.forward)r7   r8   F)r   r   r   r"   intfloatr   r   torchTensorr   r   r   r   r   r   r6   |   s   r6   )r
   r   r;   c                 C   s
   t | S r   r   gelur	   r   r   r   rD      s    rD   c                       s<   e Zd ZdZd	ed fddZejejdddZ  Z	S )
GELULApplies the Gaussian Error Linear Units function (w/ dummy inplace arg)
    Fr   c                    s   t t|   d S r   )r   rE   r   r   r   r   r   r      s    zGELU.__init__r<   c                 C   s
   t |S r   rC   r>   r   r   r   r      s    zGELU.forward)F
r   r   r   r"   r   r   rA   rB   r   r   r   r   r   r   rE      s   rE   c                 C   s   t j| ddS Nr   )ZapproximaterC   r	   r   r   r   	gelu_tanh   s    rI   c                       s<   e Zd ZdZd	ed fddZejejdddZ  Z	S )
GELUTanhrF   Fr   c                    s   t t|   d S r   )r   rJ   r   r   r   r   r   r      s    zGELUTanh.__init__r<   c                 C   s   t j|ddS rH   rC   r>   r   r   r   r      s    zGELUTanh.forward)FrG   r   r   r   r   rJ      s   rJ   )F)F)F)F)F)F)F)F)F)r"   rA   r   Ztorch.nnr   r   r   r   Moduler   r    r!   r   r$   r   r&   r,   r-   r.   r/   r4   r5   r6   rB   rD   rE   rI   rJ   r   r   r   r   <module>   s,   	
					

