a
    d                     @   sV   d Z ddlZddlm  mZ ddlmZmZ ddl	Z	ddl
mZ G dd deZdS )a1  
SGDP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/sgdp.py

Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217
Code: https://github.com/clovaai/AdamP

Copyright (c) 2020-present NAVER Corp.
MIT license
    N)	Optimizerrequired   )
projectionc                       s@   e Zd Zedddddddf fdd	Ze d
dd	Z  ZS )SGDPr   Fg:0yE>g?c
              
      s.   t ||||||||	d}
tt| ||
 d S )N)lrmomentum	dampeningweight_decaynesterovepsdeltawd_ratio)dictsuperr   __init__)selfparamsr   r   r	   r
   r   r   r   r   defaults	__class__ X/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/optim/sgdp.pyr      s
    zSGDP.__init__Nc              
   C   s`  d }|d ur:t   | }W d    n1 s00    Y  | jD ]}|d }|d }|d }|d }|d D ]}|jd u r~qn|j}	| j| }
t|
dkrt ||
d< |
d }||j|	d| d |r|	||  }n|}d}t|j	d	krt
||	||d
 |d |d \}}|dkrD|d|d |d  | d	|    |j||d  d qnq@|S )Nr
   r   r	   r   r   r   g      ?)alphar   r   r   r   r   )torchZenable_gradZparam_groupsgradstatelenZ
zeros_likeZmul_Zadd_shaper   )r   closureZlossgroupr
   r   r	   r   pr   r   bufZd_pr   r   r   r   step   s8    
$

"
&z	SGDP.step)N)	__name__
__module____qualname__r   r   r   Zno_gradr#   __classcell__r   r   r   r   r      s
   
r   )__doc__r   Ztorch.nn.functionalnnZ
functionalFZtorch.optim.optimizerr   r   mathZadampr   r   r   r   r   r   <module>   s   
