a
    d                     @   s>  d Z ddlmZmZ ddlZddlmZ zddlm	Z	 dZ
W n eyR   dZ
Y n0 zddlmZmZ dZW n ey   dZY n0 dad	d
 ZdddZdejeeej eej eejdddZdejee eej eej eejdddZdejee eej edddZdejee eej eejdddZdS )a   'Fast' Normalization Functions

For GroupNorm and LayerNorm these functions bypass typical AMP upcast to float32.

Additionally, for LayerNorm, the APEX fused LN is used if available (which also does not upcast)

Hacked together by / Copyright 2022 Ross Wightman
    )ListOptionalN)
functional)fused_layer_norm_affineTF)fused_rms_norm_affinefused_rms_normc                   C   s   t S N_USE_FAST_NORM r   r   ^/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/fast_norm.pyis_fast_norm   s    r   c                 C   s   | a d S r   r	   )enabler   r   r   set_fast_norm#   s    r   h㈵>)x
num_groupsweightbiasepsreturnc                 C   s   t j rt| ||||S t  rZt  }| ||||d urN||nd   } }}t jj	j
dd" t| ||||W  d    S 1 s0    Y  d S NF)Zenabled)torchjitis_scriptingFZ
group_normis_autocast_enabledget_autocast_gpu_dtypetocudaampautocast)r   r   r   r   r   dtr   r   r   fast_group_norm(   s    
.r#   )r   normalized_shaper   r   r   r   c                 C   s   t j rt| ||||S tr0t| ||||S t  rnt  }| 	||	||d urb|	|nd   } }}t j
jjdd" t| ||||W  d    S 1 s0    Y  d S r   )r   r   r   r   Z
layer_normhas_apexr   r   r   r   r   r    r!   )r   r$   r   r   r   r"   r   r   r   fast_layer_norm>   s    
.r&   r   r$   r   r   c                 C   s   t |}tj r4|dksJ tj| ddd}n&ttd| d d}tj| |dd}| t||  } |d ur|| | } | S )N   )dimT)r*   Zkeepdim)	lenr   r   r   varZ	unsqueezetuplerangeZrsqrt)r   r$   r   r   Z	norm_ndimvdimsr   r   r   rms_normW   s    
r1   )r   r$   r   r   r   c                 C   sL   t j rt| |||S tr>|d u r0t| ||S t| |||S t| |||S r   )r   r   r   r1   has_apex_rmsnormr   r   r'   r   r   r   fast_rms_normm   s    
r3   )T)NNr   )NNr   )Nr   )Nr   )__doc__typingr   r   r   Ztorch.nnr   r   Z#apex.normalization.fused_layer_normr   r%   ImportErrorr   r   r2   r
   r   r   ZTensorintfloatr#   r&   r1   r3   r   r   r   r   <module>   sn   


          