a d(@sXddlZddlmZddlmZmZmZmZmZmZm Z ddl m Z ddl m Z ddlmZmZdd gZd d ZGd ddeZd jee ede_deeeeeeeeeeeeeeeeeeeeeddd Zeeeeeeeeeeeeeeeeeeed ddZeeeeeeeeeeeeeeeeeeed ddZdS)N)Tensor) Optimizer_use_grad_for_differentiable _get_value_default_to_fused_or_foreach_differentiable_doc _foreach_doc _maximize_doc) is_compiling)"_group_tensors_by_device_and_dtype)ListOptionalASGDasgdcCst|tjst|S|SN isinstancetorchrtensorxrY/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/optim/asgd.py _to_tensor s  rcsNeZdZdeeeedfd d Zfd d Zd dZedddZ Z S)r{Gz?-C6??.ArNFforeachmaximizedifferentiablec sVd|kstd|d|ks,td|t|||||||| d} t|| dS)NzInvalid learning rate: {}zInvalid weight_decay value: {})lrlambdalphat0 weight_decayr r!r") ValueErrorformatdictsuper__init__) selfparamsr$r%r&r'r(r r!r"defaults __class__rrr-s  z ASGD.__init__cst||jD](}|dd|dd|ddqt|j}t|dkoft |dd}|s|D]}t t |d|d<qpt|dkot |dd}|s|D]}t |d|d<qt|dkot |dd}|s|D]}t t |d|d<qdS) Nr r!Fr"rstepetamu) r, __setstate__ param_groups setdefaultliststatevalueslenrZ is_tensorrfloat)r.r:groupZ state_valuesZstep_is_tensorsZ eta_is_tensorZ mu_is_tensorr1rrr60s0       zASGD.__setstate__c Cs|dD]}|jdur|||jjr0td||j|j|} t| dkrtd| d<t|d| d<td| d <tj|tj d | d <|| d || d || d|| dqdS) Nr/z&ASGD does not support sparse gradientsrr#r3r$r4g?r5)Z memory_formatax) gradappendZ is_sparse RuntimeErrorr:r<rrZ zeros_likeZpreserve_format) r.r>params_with_gradgradsmusaxsetas state_stepspr:rrr _init_groupJs$       zASGD._init_groupc Csd}|dur:t|}Wdn1s00Y|jD]v}g}g}g}g}g}g} |||||||| t|||||| |d|d|d|d|d|d|d|d d q@|S) zPerforms a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr%r$r'r&r(r r!r")r%r$r'r&r(r r!r")rZ enable_gradr7rKr) r.closureZlossr>rDrErFrGrHrIrrrr3as: $ z ASGD.step)rrrrrNFF)N) __name__ __module__ __qualname__rboolr-r6rKrr3 __classcell__rrr1rrs" aImplements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) {foreach} {maximize} {differentiable} .. _Acceleration of stochastic approximation by averaging: https://dl.acm.org/citation.cfm?id=131098 rF)r/rErGrFrHrIr r!r"r%r$r'r&r(c Csn|durt||dd\}}|r0tjr0td|rDtjsDt}nt}|||||||| | | | | ||d dS)znFunctional API that performs asgd algorithm computation. See :class:`~torch.optim.ASGD` for details. NF)Z use_fusedz6torch.jit.script not supported with foreach optimizers)r%r$r'r&r(r!r")rrZjitZ is_scriptingrC_multi_tensor_asgd_single_tensor_asgd)r/rErGrFrHrIr r!r"r%r$r'r&r(_funcrrrrs,) r/rErGrFrHrIr%r$r'r&r(r!r"cCsHdd} t|D]0\}}||}| s*|n| }||}||}||}||}t|rxt|}t|}t|}|d7}t|}| dkr|j|| d}t|}|d|||j|| dts| dkr|| | |n | || |d|||| }| || dt d||}| |qdS)NcSst|tjst|S|Srrrrrrrs  z'_single_tensor_asgd.._to_tensorrrr&) enumerater is_complex view_as_realraddZmul_add_r itemsubmulcopy_max)r/rErGrFrHrIr%r$r'r&r(r!r"riparamrAr5r@r4Zstep_tr3Z eta_valuenew_etanew_murrrrSs4      rSc Cst|dkrdS| rJdt||||||g} | D]j\}}}}}}| rXt|}dd}||}||}||}t|d| dkrtj||| d}t|d}t|d||tj||| dt t|D]V}t s|| dkr|| || ||||q||||qt t|D]d}t|d||t||| }|||tdtdt|||}|||q>q8dS)Nrz#_foreach ops don't support autogradcSsdd|DS)NcSs$g|]}t|rt|n|qSr)rrXrY).0trrr 0szE_multi_tensor_asgd.._view_complex_as_real..r)Z tensor_listrrr_view_complex_as_real/sz1_multi_tensor_asgd.._view_complex_as_realrrV)r<r r;rZ _foreach_negZ _foreach_add_Z _foreach_addrZ _foreach_mul_ranger r\r[r]r^r_rr`)r/rErGrFrHrIr%r$r'r&r(r!r"Zgrouped_tensorsZgrouped_paramsZ grouped_gradsZ grouped_axsZ grouped_musZ grouped_etasZgrouped_state_stepsrhr4rarcrdrrrrRs<     (rR)NFF)rrZ optimizerrrrrrr r Z torch._utilsr Ztorch.utils._foreach_utilsr typingr r__all__rrr*__doc__rPr=rrSrRrrrrst $  z  5 =