a
    d                     @   sN   d Z ddlZddlZddlZddlZddlmZ ee	Z
G dd deZdS )z Cosine Scheduler

Cosine LR schedule with warmup, cycle/restarts, noise, k-decay.

Hacked together by / Copyright 2021 Ross Wightman
    N   )	Schedulerc                       sJ   e Zd ZdZdejjeeeeedd fddZ	dd Z
dddZ  ZS )CosineLRSchedulera\  
    Cosine decay with restarts.
    This is described in the paper https://arxiv.org/abs/1608.03983.

    Inspiration from
    https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py

    k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
                  ?r   r   FTNq=
ףp?*   )	optimizer	t_initiallr_min	cycle_mulcycle_decaycycle_limitreturnc              
      s   t  j|d|
|||||d |dks(J |dks4J |dkrV|dkrV|dkrVtd | _| _| _| _| _| _	 _
|	 _| _ j	r fdd jD  _t   j
 ndd  jD  _d S )	Nlr)Zparam_group_fieldt_in_epochsnoise_range_t	noise_pct	noise_std
noise_seed
initializer   r   zjCosine annealing scheduler will have no effect on the learning rate since t_initial = t_mul = eta_mul = 1.c                    s   g | ]}|  j  qS  )warmup_t.0vselfwarmup_lr_initr   a/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/scheduler/cosine_lr.py
<listcomp>K       z.CosineLRScheduler.__init__.<locals>.<listcomp>c                 S   s   g | ]}d qS )r   r   r   _r   r   r   r    N   r!   )super__init___loggerwarningr
   r   r   r   r   r   r   warmup_prefixk_decaybase_valueswarmup_stepsZupdate_groups)r   r	   r
   r   r   r   r   r   r   r(   r   r   r   r   r   r)   r   	__class__r   r   r%      s:    zCosineLRScheduler.__init__c                    s  j k r"fddjD }njr2j  jdkrttdj dj   j}j| j dj|  dj  j  nj }jj|  j|   fddj	D }j
|jk rfdd|D }nfddj	D }|S )Nc                    s   g | ]} j |  qS r   )r   )r   s)r   tr   r   r    R   r!   z-CosineLRScheduler._get_lr.<locals>.<listcomp>r   c                    s   g | ]}|  qS r   r   r   )gammar   r   r    a   r!   c              
      s@   g | ]8}j d |j   dttj         qS )g      ?r   )r   mathcospi)r   Zlr_max)kr   t_currt_ir   r   r    e   s   c                    s   g | ]
} j qS r   )r   r"   )r   r   r   r    j   r!   )r   r+   r(   r   r1   floorlogr
   r   r*   r)   r   )r   r/   ZlrsiZlr_max_valuesr   )r0   r4   r   r/   r5   r6   r   _get_lrP   s(    


($


zCosineLRScheduler._get_lrc                 C   sR   t d|p| j}| jdkr$| j| S tt| j | j| d  d| j  S d S )Nr   r   )maxr   r   r
   intr1   r7   )r   Zcyclesr   r   r   get_cycle_lengthn   s    

z"CosineLRScheduler.get_cycle_length)r   r   r   r   r   r   FTNr   r   r   r   T)r   )__name__
__module____qualname____doc__torchZoptimZ	Optimizerr<   floatr%   r:   r=   __classcell__r   r   r,   r   r      s4                 3r   )rA   loggingr1   numpynprB   Z	schedulerr   	getLoggerr>   r&   r   r   r   r   r   <module>   s   
