a
    d!                     @   s   d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd dej	Z
G d	d
 d
ej	ZeeejddZG dd dej	ZG dd dej	ZG dd dej	ZG dd dej	ZdS )zl MLP module w/ dropout and configurable activation layer

Hacked together by / Copyright 2020 Ross Wightman
    )partial)nn   )GlobalResponseNorm)	to_2tuplec                       s:   e Zd ZdZddejddddf fdd	Zdd	 Z  ZS )
MlpzG MLP as used in Vision Transformer, MLP-Mixer and related networks
    NT        Fc	                    s   t    |p|}|p|}t|}t|}	|r<ttjddntj}
|
|||d d| _| | _t	|	d | _
|d ur~||nt | _|
|||d d| _t	|	d | _d S )Nr   kernel_sizer   bias)super__init__r   r   r   Conv2dLinearfc1actDropoutdrop1Identitynormfc2drop2)selfin_featureshidden_featuresout_features	act_layer
norm_layerr   dropuse_conv
drop_probslinear_layer	__class__ X/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/mlp.pyr      s    
zMlp.__init__c                 C   s6   |  |}| |}| |}| |}| |}|S N)r   r   r   r   r   r   xr%   r%   r&   forward)   s    




zMlp.forward	__name__
__module____qualname____doc__r   ZGELUr   r*   __classcell__r%   r%   r#   r&   r      s   r   c                       sD   e Zd ZdZddejdddddf fdd	Zdd	 Zd
d Z  Z	S )GluMlpzi MLP w/ GLU style gating
    See: https://arxiv.org/abs/1612.08083, https://arxiv.org/abs/2002.05202
    NTr   Fc
                    s   t    |p|}|p|}|d dks*J t|}t|}
|rLttjddntj}|rZdnd| _|	| _||||d d| _	| | _
t|
d | _|d ur||d nt | _||d ||d d| _t|
d | _d S )N   r   r   r	   r   )r   r   r   r   r   r   r   	chunk_dim	gate_lastr   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r    r5   r!   r"   r#   r%   r&   r   6   s    
zGluMlp.__init__c                 C   sJ   | j jjd d }tj| j j|d   tjj| j j|d  dd d S )Nr   r2   ư>Zstd)r   r   shaper   initones_normal_weight)r   Zfc1_midr%   r%   r&   init_weightsS   s    zGluMlp.init_weightsc                 C   sl   |  |}|jd| jd\}}| jr2|| | n| || }| |}| |}| |}| |}|S )Nr2   )Zdim)	r   chunkr4   r5   r   r   r   r   r   )r   r)   x1Zx2r%   r%   r&   r*   Y   s    
"



zGluMlp.forward)
r,   r-   r.   r/   r   ZSigmoidr   r=   r*   r0   r%   r%   r#   r&   r1   2   s   r1   F)r   r5   c                       s@   e Zd ZdZddejdddf fdd	Zdd Zd	d
 Z  Z	S )SwiGLUz SwiGLU
    NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
    better matches some other common impl which makes mapping checkpoints simpler.
    NTr   c           	         s   t    |p|}|p|}t|}t|}tj|||d d| _tj|||d d| _| | _t|d | _	|d ur~||nt
 | _tj|||d d| _t|d | _t|| _d S )Nr   r   r   )r   r   r   r   r   fc1_gfc1_xr   r   r   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r!   r#   r%   r&   r   l   s    

zSwiGLU.__init__c                 C   s(   t j| jj t jj| jjdd d S )Nr6   r7   )r   r9   r:   rA   r   r;   r<   )r   r%   r%   r&   r=      s    zSwiGLU.init_weightsc                 C   sN   |  |}| |}| || }| |}| |}| |}| |}|S r'   )rA   rB   r   r   r   r   r   )r   r)   Zx_gater%   r%   r&   r*      s    





zSwiGLU.forward)
r,   r-   r.   r/   r   SiLUr   r=   r*   r0   r%   r%   r#   r&   r@   g   s   r@   c                       s:   e Zd ZdZddejddddf fdd	Zdd Z  ZS )	GatedMlpz MLP as used in gMLP
    NTr   c	           
         s   t    |p|}|p|}t|}t|}	tj|||d d| _| | _t|	d | _|d ur|d dkspJ ||| _	|d }n
t
 | _	|d ur||nt
 | _tj|||d d| _t|	d | _d S )Nr   r   r2   r   )r   r   r   r   r   r   r   r   r   gater   r   r   r   )
r   r   r   r   r   r   Z
gate_layerr   r   r!   r#   r%   r&   r      s     



zGatedMlp.__init__c                 C   sJ   |  |}| |}| |}| |}| |}| |}| |}|S r'   )r   r   r   rE   r   r   r   r(   r%   r%   r&   r*      s    






zGatedMlp.forwardr+   r%   r%   r#   r&   rD      s   rD   c                       s8   e Zd ZdZddejdddf fdd	Zdd Z  ZS )	ConvMlpz1 MLP using 1x1 convs that keeps spatial dims
    NTr   c                    s   t    |p|}|p|}t|}tj||d|d d| _|rF||nt | _| | _t	|| _
tj||d|d d| _d S )Nr   r   )r
   r   )r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r#   r%   r&   r      s    

zConvMlp.__init__c                 C   s6   |  |}| |}| |}| |}| |}|S r'   )r   r   r   r   r   r(   r%   r%   r&   r*      s    




zConvMlp.forward)	r,   r-   r.   r/   r   ZReLUr   r*   r0   r%   r%   r#   r&   rF      s   rF   c                       s8   e Zd ZdZddejdddf fdd	Zdd	 Z  ZS )
GlobalResponseNormMlpzG MLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d
    NTr   Fc           
         s   t    |p|}|p|}t|}t|}|r<ttjddntj}	|	|||d d| _| | _t	|d | _
t|| d| _|	|||d d| _t	|d | _d S )Nr   r	   r   r   )Zchannels_last)r   r   r   r   r   r   r   r   r   r   r   r   grnr   r   )
r   r   r   r   r   r   r   r    r!   r"   r#   r%   r&   r      s    

zGlobalResponseNormMlp.__init__c                 C   s@   |  |}| |}| |}| |}| |}| |}|S r'   )r   r   r   rH   r   r   r(   r%   r%   r&   r*      s    





zGlobalResponseNormMlp.forwardr+   r%   r%   r#   r&   rG      s   rG   N)r/   	functoolsr   Ztorchr   rH   r   Zhelpersr   Moduler   r1   rC   ZSwiGLUPackedr@   rD   rF   rG   r%   r%   r%   r&   <module>   s   %2/,!