a
    ždX  ć                   @   s$   d Z ddlZd
ddZddd	ZdS )aP   Adaptive Gradient Clipping

An impl of AGC, as per (https://arxiv.org/abs/2102.06171):

@article{brock2021high,
  author={Andrew Brock and Soham De and Samuel L. Smith and Karen Simonyan},
  title={High-Performance Large-Scale Image Recognition Without Normalization},
  journal={arXiv preprint arXiv:},
  year={2021}
}

Code references:
  * Official JAX impl (paper authors): https://github.com/deepmind/deepmind-research/tree/master/nfnets
  * Phil Wang's PyTorch gist: https://gist.github.com/lucidrains/0d6560077edac419ab5d3aa29e674d5c

Hacked together by / Copyright 2021 Ross Wightman
é    Nē       @c                 C   s4   | j dkr|  |”S | j|ttd| j ddS d S )Né   T)ZdimZkeepdim)ŚndimZnormŚtupleŚrange)ŚxŚ	norm_type© r	   śW/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/utils/agc.pyŚunitwise_norm   s    

r   ē{®Gįz?ēü©ńŅMbP?c                 C   s   t | tjr| g} | D ]~}|jd u r&q| ” }|j ” }t||dj|d |”}t||d}|||jdd  }	t 	||k ||	”}
|j ”  
|
” qd S )N)r   )Śmingķµ ÷Ę°>)Ś
isinstanceŚtorchZTensorZgradŚdetachr   Zclamp_Zmul_ŚclampŚwhereZcopy_)Ś
parametersZclip_factorZepsr   ŚpZp_dataZg_dataZmax_normZ	grad_normZclipped_gradZ	new_gradsr	   r	   r
   Śadaptive_clip_grad   s    

r   )r   )r   r   r   )Ś__doc__r   r   r   r	   r	   r	   r
   Ś<module>   s   
	