a
    d5/                     @   s  d dl Z d dl mZ ddlmZmZmZmZmZmZ d dl	m
Z
mZ d dlmZ ddgZG d	d deZd
djeeed e_de
e e
e e
e e
e ee eeeeeedddZe
e e
e e
e e
e eeeeeed
ddZe
e e
e e
e e
e eeeeeed
ddZdS )    N)Tensor   )	Optimizer_use_grad_for_differentiable_default_to_fused_or_foreach_differentiable_doc_foreach_doc_maximize_doc)ListOptional)"_group_tensors_by_device_and_dtypeRproprpropc                       sX   e Zd Zdddddee eed fddZ fd	d
Zdd ZedddZ	  Z
S )r   {Gz?g      ?g333333?gư>2   NFforeachmaximizedifferentiablec          	         s|   d|kst d|d|d   k r<d  k r<|d k sVn t d|d |d t||||||d}t || d S )Ng        zInvalid learning rate: {}r   g      ?r   zInvalid eta values: {}, {})lretas
step_sizesr   r   r   )
ValueErrorformatdictsuper__init__)	selfparamsr   r   r   r   r   r   defaults	__class__ Z/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/optim/rprop.pyr      s    (zRprop.__init__c                    s@   t  | | jD ](}|dd  |dd |dd qd S )Nr   r   Fr   )r   __setstate__param_groups
setdefault)r   stategroupr"   r$   r%   r&   &   s
    
zRprop.__setstate__c           	      C   s   |d D ]}|j d u rq|| |j }|jr6td|| | j| }t|dkrd|d< tj|tjd|d< |j	j
r| |t|d |d |d< n| ||d |d< ||d  ||d  |d  d	7  < qd S )
Nr    z'Rprop does not support sparse gradientsr   stepZmemory_formatprevr   	step_sizer   )gradappendZ	is_sparseRuntimeErrorr)   lentorchZ
zeros_likepreserve_formatZdtype
is_complexnewZ
resize_as_Zfill_complex)	r   r*   r    gradsprevsr   pr/   r)   r$   r$   r%   _init_group-   s4    




zRprop._init_groupc                 C   s   d}|dur:t   | }W d   n1 s00    Y  | jD ]p}g }g }g }g }|d \}}	|d \}
}|d }|d }| ||||| t|||||
|||	|||d d q@|S )zPerforms a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   )step_size_minstep_size_maxetaminusetaplusr   r   r   )r3   Zenable_gradr'   r;   r   )r   closureZlossr*   r    r8   r9   r   r>   r?   r<   r=   r   r   r$   r$   r%   r+   Q   s8    
$
z
Rprop.step)r   r   r   )N)__name__
__module____qualname__r   boolr   r&   r;   r   r+   __classcell__r$   r$   r"   r%   r      s       $a
  Implements the resilient backpropagation algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \theta_0 \in \mathbf{R}^d \text{ (params)},f(\theta)
                \text{ (objective)},                                                             \\
            &\hspace{13mm}      \eta_{+/-} \text{ (etaplus, etaminus)}, \Gamma_{max/min}
                \text{ (step sizes)}                                                             \\
            &\textbf{initialize} :   g^0_{prev} \leftarrow 0,
                \: \eta_0 \leftarrow \text{lr (learning rate)}                                   \\
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \textbf{for} \text{  } i = 0, 1, \ldots, d-1 \: \mathbf{do}            \\
            &\hspace{10mm}  \textbf{if} \:   g^i_{prev} g^i_t  > 0                               \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{min}(\eta^i_{t-1} \eta_{+},
                \Gamma_{max})                                                                    \\
            &\hspace{10mm}  \textbf{else if}  \:  g^i_{prev} g^i_t < 0                           \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{max}(\eta^i_{t-1} \eta_{-},
                \Gamma_{min})                                                                    \\
            &\hspace{15mm}  g^i_t \leftarrow 0                                                   \\
            &\hspace{10mm}  \textbf{else}  \:                                                    \\
            &\hspace{15mm}  \eta^i_t \leftarrow \eta^i_{t-1}                                     \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1}- \eta_t \mathrm{sign}(g_t)             \\
            &\hspace{5mm}g_{prev} \leftarrow  g_t                                                \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to the paper
    `A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm
    <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.1417>`_.
    a-  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        etas (Tuple[float, float], optional): pair of (etaminus, etaplus), that
            are multiplicative increase and decrease factors
            (default: (0.5, 1.2))
        step_sizes (Tuple[float, float], optional): a pair of minimal and
            maximal allowed step sizes (default: (1e-6, 50))
        {foreach}
        {maximize}
        {differentiable}

    r   F)r    r8   r9   r   r   r   r   r<   r=   r>   r?   c                C   sh   |du rt | |dd\}}|r0tj r0td|rDtj sDt}nt}|| ||||||	|
||d
 dS )zpFunctional API that performs rprop algorithm computation.

    See :class:`~torch.optim.Rprop` for details.
    NF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)r<   r=   r>   r?   r   r   )r   r3   ZjitZis_scriptingr1   _multi_tensor_rprop_single_tensor_rprop)r    r8   r9   r   r   r   r   r<   r=   r>   r?   _funcr$   r$   r%   r      s&    )
r    r8   r9   r   r<   r=   r>   r?   r   r   c                C   s
  t | D ]\}
}||
 }|s |n| }||
 }||
 }t|rht|}t|}t|}t|}|	r||  }n|| }|||d< |||d< d||	d< |
||| |jtjd}d||	|< |j| |dd || qd S )Nr   r   r,   value)	enumerater3   r5   view_as_realmulclonesigngtlteqZmul_clamp_r4   Zaddcmul_copy_)r    r8   r9   r   r<   r=   r>   r?   r   r   iparamr/   r-   r.   rQ   r$   r$   r%   rG      s*    




rG   c                C   sz  t | dkrd S |	rJ dt| |||g}
|
 D ]>\}}}}dd }||}||}||}||}|rxt|}t||}dd |D }|D ].}|||d< |||d< d||d< qt	|| |D ]}|
|| qt|}tt |D ]0}|| jtjd||< d|| || |< qd	d |D }tj|||d
d tt |D ]}|| ||  qZq4d S )Nr   z#_foreach ops don't support autogradc                 S   s   dd | D S )Nc                 S   s$   g | ]}t |rt |n|qS r$   )r3   r5   rN   ).0tr$   r$   r%   
<listcomp>   s   zF_multi_tensor_rprop.<locals>._view_complex_as_real.<locals>.<listcomp>r$   )Ztensor_listr$   r$   r%   _view_complex_as_real  s    z2_multi_tensor_rprop.<locals>._view_complex_as_realc                 S   s   g | ]}|  qS r$   rQ   )rY   sr$   r$   r%   r[   -      z'_multi_tensor_rprop.<locals>.<listcomp>r   r,   c                 S   s   g | ]}|  qS r$   r]   )rY   r/   r$   r$   r%   r[   @  r_   rJ   rK   )r2   r   valuesr3   Z_foreach_negZ_foreach_mulrR   rS   rT   Z_foreach_mul_rU   listrangerP   r4   Z_foreach_addcmul_rV   )r    r8   r9   r   r<   r=   r>   r?   r   r   Zgrouped_tensorsZgrouped_paramsZgrouped_gradsZgrouped_prevsZgrouped_step_sizesr\   ZsignsrQ   r.   rW   Z
grad_signsr$   r$   r%   rF   	  s:    
rF   )NFF)r3   r   Z	optimizerr   r   r   r   r   r	   typingr
   r   Ztorch.utils._foreach_utilsr   __all__r   r   __doc__rD   floatr   rG   rF   r$   r$   r$   r%   <module>   sb    p"9   //