a d@sjUddlmZmZmZmZddlZddlmmZ ddlm Z gZ ee e d<ejjGdddZdS))DictListOptionalTupleN)Tensor__all__c @sdeZdZdeeeeeefeeeeeeed ddZee ed d d Z ee ed d dZ dS)_FunctionalAdamWMbP?g?g+?:0yE>{Gz?F) paramslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc Csd|kstd|d|ks,td|d|dkrDdksXntd|dd|dkrpdksntd|dd|kstd ||||d|d|d |_||_||_||_| |_tj t tj t t tj ffi|_ t|dkr | s td d |i|_dS) NzInvalid learning rate: {}zInvalid epsilon value: {}rg?z%Invalid beta parameter at index 0: {}z%Invalid beta parameter at index 1: {}zInvalid weight_decay value: {})rrbeta1beta2rz%optimizer got an empty parameter listr ) ValueErrorformatdefaultsrrrrtorchjitZannotaterrstrstatelen param_group) selfr rrrrrrrrrr%q/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/distributed/optim/functional_adamw.py__init__s0 $z_FunctionalAdamW.__init__)paramgradc Csjg}g}g}g}g}g}|dur4||||||jvri|j|<|j|} td| d<tj|tjd| d<tj|tjd| d<|jrtj|tjd| d<|j|} || d|| d|jr|| d|| dtbtj |||||||j|j |j d|j d|j d |j d |j d |j |j ddd Wdn1s\0YdS) NrstepZ memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrrrr rrrrrrrrrZ grad_scaleZ found_inf)appendr!rtensor zeros_likepreserve_formatrno_gradFadamwrrrr) r$r(r)params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr!r%r%r& step_paramAsb          z_FunctionalAdamW.step_param) gradientsc Cs|jd}g}g}g}g}g}g}t|t|krXtddt|ddt|t|jd|D]\} } | durh|| || | |jvri|j| <|j| } td| d<tj| tj d| d <tj| tj d| d <|j rtj| tj d| d <|j| } || d || d |j r6|| d || dqht bt j |||||||j |j|jd |jd |jd|jd|jd|j|jdddWdn1s0YdS)Nr zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: rr*r+r,r-r.rrrrrr/)r#r"rzipr0r!rr1r2r3rr4r5r6rrrr) r$r>r r7r8r9r:r;r<r(Zgradientr!r%r%r&r*|sv            z_FunctionalAdamW.stepN) r r r r FFFFF) __name__ __module__ __qualname__rrfloatrboolr'rr=r*r%r%r%r&rs.  ,;r)typingrrrrrZtorch.optim._functionalZoptimZ _functionalr5rrr __annotations__rscriptrr%r%r%r&s