a d}c@sddlZddlmZddlmZmZmZmZmZmZm Z m Z m Z m Z m Z ddlmZmZddlmZddgZGd ddeZd d je e e ee d e_deeeeeeeeeeeeeeeeeeeeeeeeeeeeedddZeeeeeeeeeeeeeeeeeeeeeeeeedddZeeeeeeeeeeeeeeeeeeeeeeeeedddZeeeeeeeeeeeeeeeeeeeeeeeeeddddZdS)N)Tensor) Optimizer_use_grad_for_differentiable _get_value_dispatch_sqrt_stack_if_compiling_capturable_doc_differentiable_doc _foreach_doc _fused_doc _maximize_doc_default_to_fused_or_foreach)ListOptional)"_group_tensors_by_device_and_dtypeAdamWadamwcsdeZdZdddddddeeeeeeedfdd Zfd d Zd d ZedddZ Z S)rMbP?g?g+?:0yE>{Gz?FN)maximizeforeach capturabledifferentiablefusedc s d|kstd|d|ks,td|d|dkrDdksXntd|dd|dkrpdksntd|dd|kstd |t|||||||| | | d } t|| | r| rtd d |_td d|jDstd|rtddS)NzInvalid learning rate: {}zInvalid epsilon value: {}rg?z%Invalid beta parameter at index 0: {}rz%Invalid beta parameter at index 1: {}zInvalid weight_decay value: {}) lrbetaseps weight_decayamsgradrrrrrz)`fused` does not support `differentiable`Tcss,|]$}|dD]}|jo t|VqqdS)paramsN)is_cudatorchZis_floating_point).0Zpgpr(Z/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/optim/adamw.py <sz!AdamW.__init__..zF`fused=True` requires all the params to be CUDA, floating point Tensorz0`fused` and `foreach` cannot be `True` together.) ValueErrorformatdictsuper__init__ RuntimeErrorZ_step_supports_amp_scalingall param_groups) selfr#rrr r!r"rrrrrdefaults __class__r(r)r/ sB zAdamW.__init__cst||jD]L}|dd|dd|dd|dd|dd|ddqt|j}t|dkot |dd }|s|D]}t t |d |d <qdS) Nr"Frrrrrrstep) r. __setstate__r2 setdefaultliststatevalueslenr%Z is_tensortensorfloat)r3r;groupZ state_valuesZstep_is_tensorsr5r(r)r8Ds        zAdamW.__setstate__c Cs|dD]} | jdurq|| | jjr2td|| j|j| } t| dkr|dsd|drxtjdtj| j dnt d| d <tj | tj d | d <tj | tj d | d <|rtj | tj d | d <|| d || d |r|| d || d qdS)Nr#z'AdamW does not support sparse gradientsrrrr)dtypedevicerr7)Z memory_formatexp_avg exp_avg_sqZmax_exp_avg_sq) gradappendZ is_sparser0r;r=r%zerosr?rDr>Z zeros_likeZpreserve_format) r3r@params_with_gradgradsr"exp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr'r;r(r(r) _init_groupUs<         zAdamW._init_groupc Cs|d}|durBt|}Wdn1s80Y|jD]}g}g}g}g}g}g} |d} |d\} } ||||| |||| t|||||| | | | |d|d|d|d|d|d |d |d t|d dt|d ddqH|S)zPerforms a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr"rrr!r rrrrr grad_scale found_inf) r"beta1beta2rr!r rrrrrrQrR)Z _cuda_graph_capture_health_checkr%Z enable_gradr2rPrgetattr) r3closureZlossr@rJrKrLrMrNrOr"rSrTr(r(r)r7s\ $     z AdamW.step)rrrrF)N) __name__ __module__ __qualname__boolrr/r8rPrr7 __classcell__r(r(r5r)r s,  7 2a Implements AdamW algorithm. .. math:: \begin{aligned} &\rule{110mm}{0.4pt} \\ &\textbf{input} : \gamma \text{(lr)}, \: \beta_1, \beta_2 \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)}, \: \epsilon \text{ (epsilon)} \\ &\hspace{13mm} \lambda \text{(weight decay)}, \: \textit{amsgrad}, \: \textit{maximize} \\ &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0 \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0 \\[-1.ex] &\rule{110mm}{0.4pt} \\ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm}\textbf{else} \\ &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ &\hspace{5mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ &\hspace{5mm}\textbf{if} \: amsgrad \\ &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_t}^{max}, \widehat{v_t}) \\ &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big) \\ &\hspace{5mm}\textbf{else} \\ &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ &\rule{110mm}{0.4pt} \\[-1.ex] &\bf{return} \: \theta_t \\[-1.ex] &\rule{110mm}{0.4pt} \\[-1.ex] \end{aligned} For further details regarding the algorithm we refer to `Decoupled Weight Decay Regularization`_. a Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-3) betas (Tuple[float, float], optional): coefficients used for computing running averages of gradient and its square (default: (0.9, 0.999)) eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) weight_decay (float, optional): weight decay coefficient (default: 1e-2) amsgrad (bool, optional): whether to use the AMSGrad variant of this algorithm from the paper `On the Convergence of Adam and Beyond`_ (default: False) {maximize} {foreach} {capturable} {differentiable} {fused} .. _Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101 .. _On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ )rrrrrF)r#rKrLrMrNrOrrrrrQrRr"rSrTrr!r rc Cstdd|Dstd| dur<|dur.zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers) r"rSrTrr!r rrrrQrR) r1r0rr%ZjitZ is_scripting _fused_adamw_multi_tensor_adamw_single_tensor_adamw)r#rKrLrMrNrOrrrrrQrRr"rSrTrr!r r_funcr(r(r)rsL)r#rKrLrMrNrOrQrRr"rSrTrr!r rrrc Cs@|dur|dusJt|D]\}}|s2||n|| }||}||}||}|rl|jrd|jslJdt|rt|}t|}t|}t|}|d7}|d| | || j|d| d|| j||d| d|s|r|}dt| |}dt| |}| |}| }| }|r|rJ|| }n||}|| t |||| ||| |}n| ||| |}|||qt|}d| |}d| |}| |}t|}|rtj |||||d|| || }n| || }|j||| dqdS)N@If capturable=True, params and state_steps must be CUDA tensors.ralpha)value)out) enumerater$r% is_complex view_as_realZmul_Zadd_Zaddcmul_pownegsqrtcloneZcopy_maximumZaddcdiv_rr)r#rKrLrMrNrOrQrRr"rSrTrr!r rrriparamrGrErFZstep_tr7bias_correction1bias_correction2 step_sizeZ step_size_negbias_correction2_sqrtZmax_exp_avg_sqs_idenomr(r(r)rbVsj        rbc st|dkrdS|r4tddt||Ds4Jd|r@Jd|durP|dusTJt||||||g}|D]\}}}}}}|rtt|}dd|D}dd|D}d d|D}d d|D}t|d t |d | t |tj||d d t |t |||d |rPfd d|D}fdd|D}t |d t |d t |t |t |}t|t |t|}|rt||t|}t|t||t || }t|t||}n@t|}t|t||t || }t|t||}t|||qpfdd|D}fdd|D}tfdd|D}dd|D}|rt||t|}t||t|| }n"t|}t||t|| }t||||qpdS)Nrcss|]\}}|jo|jVqdSr\)r$)r&r'r7r(r(r)r*sz&_multi_tensor_adamw..rez#_foreach ops don't support autogradcSs$g|]}t|rt|n|qSr(r%rkrlr&xr(r(r) r_z'_multi_tensor_adamw..cSs$g|]}t|rt|n|qSr(ryrzr(r(r)r|r_cSs$g|]}t|rt|n|qSr(ryrzr(r(r)r|scSs$g|]}t|rt|n|qSr(ryrzr(r(r)r|r_rrfcsg|]}t|qSr(r%rmr&r7rSr(r)r|r_csg|]}t|qSr(r}r~rTr(r)r|r_csg|]}dt|qSrBrr~rr(r)r|$r_csg|]}dt|qSrBrr~rr(r)r|%r_csg|]}|dqS)r(r&Zbc)rr(r)r|'r_cSsg|] }t|qSr()rrr(r(r)r|)r_)r=r1ziprr<r%Z _foreach_negtuple _foreach_add_Z _foreach_mul_Z_foreach_addcmul_ _foreach_sub_Z _foreach_neg_Z _foreach_divZ_foreach_reciprocal_Z _foreach_sqrtZ_foreach_maximum_Z _foreach_div_Z _foreach_mulZ _foreach_addZ_foreach_addcdiv_r) r#rKrLrMrNrOrQrRr"rSrTrr!r rrrgrouped_tensors device_params device_gradsdevice_exp_avgsdevice_exp_avg_sqsdevice_max_exp_avg_sqsdevice_state_stepsrtrurvrwZmax_exp_avg_sq_sqrtZeps_over_step_sizerxZexp_avg_sq_sqrtr()rSrTrr)ras                               ra)r#rKrLrMrNrOrQrRr"rSrTrr!r rrrreturnc Cs$|r td|dur|j|ind}|dur4|j|ind}t||||||g}|D]\}}|||f\}}}}}}d\}}|dur||vr|j|dd||<||}|dur||vr|j|dd||<||}t|dtj|||||||| | | | | |||d|durPt||gt|qPdS)Nz"_fused_adamw is not differentiable)NNT)Z non_blockingr) r"rrSrTr!r rrQrR) r0rDrtor%rZ _fused_adamw_rr=)r#rKrLrMrNrOrQrRr"rSrTrr!r rrrZgrad_scale_dictZfound_inf_dictrrDrCrrrrrrZdevice_grad_scaleZdevice_found_infr(r(r)r`;sV   r`)NFFNNN)r%rZ optimizerrrrrrr r r r r rtypingrrZtorch.utils._foreach_utilsr__all__rr,__doc__rZr?rrbrar`r(r(r(r)s 4 9&M O h