a d'@sdZddlZddlZddlmZddlmZd)ddZdd Ze d d d Z Gd ddej j Z ddZGdddej j Zd*ddZddZddZddZddZGdd d ejZGd!d"d"ejZd#d$Zd%d&Zd'd(ZdS)+a adopted from https://github.com/openai/improved-diffusion/blob/main/improved_diffusion/gaussian_diffusion.py and https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py and https://github.com/openai/guided-diffusion/blob/0ba878e517b276c45d1195eb29f6f5f72659a05b/guided_diffusion/nn.py thanks! N)repeat-C6?{Gz?cCs0|dkr(tj|d|d|tjdd}|S)Nlinearg?)dtype)torchlinspacefloat64numpy)scheduleZ n_timestep linear_start linear_endbetasrh/var/www/html/stable-diffusion-webui/repositories/generative-models/sgm/modules/diffusionmodules/util.pymake_beta_schedulesrcCs4|j^}}|d|}|j|gdt|dRS)N)r)shapegatherreshapelen)atZx_shapeb_outrrrextract_into_tensor#s  rinputsc s|rfddD}fddD}fddD}fddD}t|t|t|}tj|t|t|||g|RS|fiSdS)a Evaluate a function without caching intermediate activations, allowing for reduced memory at the expense of extra compute in the backward pass. This differs from the original checkpoint function borrowed from https://github.com/openai/guided-diffusion/blob/0ba878e517b276c45d1195eb29f6f5f72659a05b/guided_diffusion/nn.py in that it also works with non-tensor inputs :param func: the function to evaluate. :param inputs: the argument dictionary to pass to `func`. :param params: a sequence of parameters `func` depends on but does not explicitly take as arguments. :param flag: if False, disable gradient checkpointing. cs g|]}t|tjr|qSr isinstancerTensor.0keyrrr 6z$mixed_checkpoint..cs$g|]}t|tjr|qSrr!r$rrrr'7scs g|]}t|tjs|qSrr!r$rrrr':scs$g|]}t|tjs|qSrr!r$rrrr'=sN)tupleMixedCheckpointFunctionapplyr) funcr paramsflag tensor_keys tensor_inputsnon_tensor_keysZnon_tensor_inputsargsrrrmixed_checkpoint)s,     r3c@s$eZdZeddZeddZdS)r*cGs||_|||_tttd|_t||krBt||ksFJddt|t |d|jD|_ ddt|t ||j|jD|_ ||_ t ||jd|_ t(|j fi|j |j }Wdn1s0Y|S)Nenabledr cache_enabledcSsi|]\}}||qSrrr%r&valrrr dsz3MixedCheckpointFunction.forward..cSsi|]\}}||qSrrr7rrrr9gs) end_tensorsend_non_tensorsris_autocast_enabledget_autocast_gpu_dtypeis_autocast_cache_enabledgpu_autocast_kwargsrziplist input_tensorsinput_non_tensors run_function input_paramsno_grad)ctxrDZlength_tensorsZlength_non_tensorsr/r1r2output_tensorsrrrforwardNs6     $zMixedCheckpointFunction.forwardc sfddjD_tptjjjfij:fddjD}jfi|j}Wdn1sv0YWdn1s0Ytj j |t j j |dd}`` ~d|djdjj|jdS)Ncs"i|]}|j|dqST)rBdetachrequires_grad_r$rGrrr9ysz4MixedCheckpointFunction.backward..cs$i|]}|j|j|qSr)rBview_asr$rMrrr9sT allow_unused)NNNNNN)rBr enable_gradcudaampautocastr?rDrCautogradgradrAvaluesrEr:r;rG output_gradsZshallow_copiesrHZ input_gradsrrMrbackwardvs2 " R  z MixedCheckpointFunction.backwardN__name__ __module__ __qualname__ staticmethodrIr[rrrrr*Ms 'r*cCs8|r,t|t|}tj|t|g|RS||SdS)a Evaluate a function without caching intermediate activations, allowing for reduced memory at the expense of extra compute in the backward pass. :param func: the function to evaluate. :param inputs: the argument sequence to pass to `func`. :param params: a sequence of parameters `func` depends on but does not explicitly take as arguments. :param flag: if False, disable gradient checkpointing. N)r)CheckpointFunctionr+r)r,r r-r.r2rrr checkpoints rbc@s$eZdZeddZeddZdS)racGs|||_t|d||_t||d|_tttd|_t |j|j}Wdn1sn0Y|S)Nr4) rDrArBrErr<r=r>r?rF)rGrDlengthr2rHrrrrIs *zCheckpointFunction.forwardc Gsdd|jD|_t`tjjjfi|j*dd|jD}|j|}Wdn1sb0YWdn1s0Ytjj ||j|j |dd}|`|` ~d|S)NcSsg|]}|dqSrJ)rKrLr%xrrrr'r(z/CheckpointFunction.backward..cSsg|]}||qSr)rNrdrrrr'r(TrO)NN) rBrrRrSrTrUr?rDrVrWrErYrrrr[s"F zCheckpointFunction.backwardNr\rrrrras ra'FcCs|s|d}tt| tjd|tjd|j|jd}|dddf|d}tj t |t |gdd}|drtj |t |ddddfgdd}nt |d |d }|S) aX Create sinusoidal timestep embeddings. :param timesteps: a 1-D Tensor of N indices, one per batch element. These may be fractional. :param dim: the dimension of the output. :param max_period: controls the minimum frequency of the embeddings. :return: an [N x dim] Tensor of positional embeddings. rr)startendr)deviceNrdimrzb -> b d)d)rexpmathlogarangefloat32torifloatcatcossin zeros_liker) timestepsrkZ max_periodZ repeat_onlyhalffreqsr2 embeddingrrrtimestep_embeddings&  r|cCs|D]}|q|S)z< Zero out the parameters of a module and return it. ) parametersrKzero_)moduleprrr zero_modules rcCs |D]}||q|S)z9 Scale the parameters of a module and return it. )r}rKmul_)rscalerrrr scale_modules rcCs|jttdt|jdS)z6 Take the mean over all non-batch dimensions. rrj)meanrArangerr)tensorrrr mean_flatsrcCs td|S)z Make a standard normalization layer. :param channels: number of input channels. :return: an nn.Module for normalization. ) GroupNorm32)channelsrrr normalizationsrc@seZdZddZdS)SiLUcCs|t|SrQ)rsigmoidselfrerrrrI sz SiLU.forwardN)r]r^r_rIrrrrr srcseZdZfddZZS)rcst||jSrQ)superrIrstyperr __class__rrrIszGroupNorm32.forward)r]r^r_rI __classcell__rrrrrsrcOsZ|dkrtj|i|S|dkr0tj|i|S|dkrHtj|i|Std|dS)z4 Create a 1D, 2D, or 3D convolution module. rrunsupported dimensions: N)nnConv1dConv2dConv3d ValueErrordimsr2kwargsrrrconv_ndsrcOstj|i|S)z! Create a linear module. )rLinear)r2rrrrr#srcOsZ|dkrtj|i|S|dkr0tj|i|S|dkrHtj|i|Std|dS)z8 Create a 1D, 2D, or 3D average pooling module. rrrrN)r AvgPool1d AvgPool2d AvgPool3drrrrr avg_pool_nd*sr)rr)rfF)__doc__rnrtorch.nnreinopsrrrdictr3rVFunctionr*rbrar|rrrrModuler GroupNormrrrrrrrrs,    $L$