a dJy@srddlmZddlZddlZddlZddlmZddl mm Z ddl m Z mZmZmZmZmZmZddlmZddlmZddZdd ZGd d d ejZGd d d ejZGdddejeZGdddejZGdddejZ GdddejZ!GdddeZ"GdddejZ#ddZ$GdddejZ%GdddejZ&Gd d!d!ejZ'Gd"d#d#ejZ(dS)$)abstractmethodN) checkpointconv_ndlinear avg_pool_nd zero_module normalizationtimestep_embedding)SpatialTransformer)existscCsdSNxr r {/var/www/html/stable-diffusion-webui/repositories/stable-diffusion-stability-ai/ldm/modules/diffusionmodules/openaimodel.pyconvert_module_to_f16srcCsdSr r rr r rconvert_module_to_f32srcs6eZdZdZdeeeedfdd ZddZZS) AttentionPool2dzS Adapted from CLIP: https://github.com/openai/CLIP/blob/main/clip/model.py N) spacial_dim embed_dimnum_heads_channels output_dimcspttt||dd|d|_td|d|d|_td||pN|d|_ |||_ t |j |_ dS)Ng?) super__init__nn Parameterthrandnpositional_embeddingrqkv_projc_proj num_heads QKVAttention attention)selfrrrr __class__r rr$s  $ zAttentionPool2d.__init__cCs|j^}}}|||d}tj|jddd|gdd}||jdddddf|j}||}| |}| |}|dddddfS)NT)dimkeepdimr+r) shapereshapercatmeanr!todtyper"r&r#)r'rbc_spatialr r rforward2s $   zAttentionPool2d.forward)N)__name__ __module__ __qualname____doc__intrr7 __classcell__r r r(rrs rc@seZdZdZeddZdS) TimestepBlockzT Any module where forward() takes timestep embeddings as a second argument. cCsdS)zJ Apply the module to `x` given `emb` timestep embeddings. Nr r'rembr r rr7BszTimestepBlock.forwardN)r8r9r:r;rr7r r r rr>=sr>c@seZdZdZdddZdS)TimestepEmbedSequentialzt A sequential module that passes timestep embeddings to the children that support it as an extra input. NcCsB|D]8}t|tr|||}qt|tr4|||}q||}q|Sr ) isinstancer>r )r'rr@contextlayerr r rr7Os     zTimestepEmbedSequential.forward)N)r8r9r:r;r7r r r rrAIsrAcs*eZdZdZd fdd ZddZZS) UpsampleaA An upsampling layer with an optional convolution. :param channels: channels in the inputs and outputs. :param use_conv: a bool determining if a convolution is applied. :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then upsampling occurs in the inner-two dimensions. rNrcsFt||_|p||_||_||_|rBt||j|jd|d|_dS)Nrpadding)rrchannels out_channelsuse_convdimsrconv)r'rHrJrKrIrGr(r rrcs  zUpsample.__init__cCst|jd|jksJ|jdkrPtj||jd|jdd|jddfdd}ntj|ddd}|jrp||}|S)Nrrrnearest)mode) scale_factorrO)r.rHrKF interpolaterJrLr'rr r rr7ls & zUpsample.forward)rNrr8r9r:r;rr7r=r r r(rrEZs rEcs*eZdZdZdfdd ZddZZS) TransposedUpsamplez%Learned 2x upsampling without paddingNcs6t||_|p||_tj|j|j|dd|_dS)Nr kernel_sizestride)rrrHrIrConvTranspose2dup)r'rHrIksr(r rrzs  zTransposedUpsample.__init__cCs ||Sr )r[rSr r rr7szTransposedUpsample.forward)NrVrTr r r(rrUxsrUcs*eZdZdZd fdd ZddZZS) DownsampleaD A downsampling layer with an optional convolution. :param channels: channels in the inputs and outputs. :param use_conv: a bool determining if a convolution is applied. :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then downsampling occurs in the inner-two dimensions. rNrcszt||_|p||_||_||_|dkr2dnd}|rVt||j|jd||d|_n |j|jksfJt|||d|_dS)Nrr)rrr)rYrGrW) rrrHrIrJrKropr)r'rHrJrKrIrGrYr(r rrs   zDownsample.__init__cCs|jd|jksJ||S)Nr)r.rHr^rSr r rr7szDownsample.forward)rNrrTr r r(rr]sr]cs2eZdZdZd fdd ZddZd d ZZS) ResBlocka A residual block that can optionally change the number of channels. :param channels: the number of input channels. :param emb_channels: the number of timestep embedding channels. :param dropout: the rate of dropout. :param out_channels: if specified, the number of out channels. :param use_conv: if True and out_channels is specified, use a spatial convolution instead of a smaller 1x1 convolution to change the channels in the skip connection. :param dims: determines if the signal is 1D, 2D, or 3D. :param use_checkpoint: if True, use gradient checkpointing on this module. :param up: if True, use this block for upsampling. :param down: if True, use this block for downsampling. NFrc slt||_||_||_|p"||_||_||_||_t t |t t |||jddd|_| pf| |_| rt|d||_t|d||_n2| rt|d||_t|d||_nt |_|_t t t||rd|jn|j|_t t |jt t j|dtt ||j|jddd|_|j|kr8t |_n0|rVt |||jddd|_nt |||jd|_dS)NrrrFFr)p)rrrH emb_channelsdropoutrIrJuse_checkpointuse_scale_shift_normr SequentialrSiLUr in_layersupdownrEh_updx_updr]Identityr emb_layersDropoutr out_layersskip_connection) r'rHrarbrIrJrdrKrcr[downr(r rrsV       zResBlock.__init__cCst|j||f||jS)a Apply the block to a Tensor, conditioned on a timestep embedding. :param x: an [N x C x ...] Tensor of features. :param emb: an [N x emb_channels] Tensor of timestep embeddings. :return: an [N x C x ...] Tensor of outputs. )r_forward parametersrcr?r r rr7szResBlock.forwardc Cs|jrF|jdd|jd}}||}||}||}||}n ||}|||j}t|jt|jkr|d}qb|j r|j d|j dd}}t j |ddd\} } ||d| | }||}n||}| |}| ||S)Nr*).Nrrrr-)rhrgrirjrltyper3lenr.rdrnrchunkro) r'rr@in_restin_convhemb_outout_normout_restscaleshiftr r rrqs$       zResBlock._forward)NFFrFFFr8r9r:r;rr7rqr=r r r(rr_s@ r_cs2eZdZdZd fdd ZddZd d ZZS) AttentionBlocka An attention block that allows spatial positions to attend to each other. Originally ported from here, but adapted to the N-d case. https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/models/unet.py#L66. rr*Fcst||_|dkr ||_n*||dks@Jd|d||||_||_t||_td||dd|_|rt |j|_ n t |j|_ t td||d|_ dS)Nr*rzq,k,v channels z' is not divisible by num_head_channels rr)rrrHr$rcrnormrqkvr%r&QKVAttentionLegacyrproj_out)r'rHr$num_head_channelsrcuse_new_attention_orderr(r rrs      zAttentionBlock.__init__cCst|j|f|dS)NT)rrqrrrSr r rr79szAttentionBlock.forwardcCsV|j^}}}|||d}|||}||}||}||j||g|RS)Nr*)r.r/rrr&r)r'rr4r5spatialrrxr r rrq=s    zAttentionBlock._forward)rr*FFr~r r r(rrs rcCsL|dj^}}}tt|}d||d|}|jt|g7_dS)a( A counter for the `thop` package to count the operations in an attention operation. Meant to be used like: macs, params = thop.profile( model, inputs=(inputs, timestamps), custom_ops={QKVAttention: QKVAttention.count_flops}, ) rrN)r.r<npprod total_opsr DoubleTensor)model_xyr4r5r num_spatial matmul_opsr r rcount_flops_attnFs rcs4eZdZdZfddZddZeddZZS)rzh A module which performs QKV attention. Matches legacy QKVAttention + input/ouput heads shaping cst||_dSr rrn_headsr'rr(r rr_s zQKVAttentionLegacy.__init__c Cs|j\}}}|d|jdks"J|d|j}|||j|d|j|dd\}}}dtt|} td|| || } tj| dd | j } td| |} | |d|S)z Apply QKV attention. :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs. :return: an [N x (H * C) x T] tensor after attention. rrrr- bct,bcs->btsr* bts,bcs->bct) r.rr/splitmathsqrtreinsumsoftmaxfloatrsr3 r'rbswidthlengthchqkvr|weightar r rr7cs (zQKVAttentionLegacy.forwardcCs t|||Sr rrrrr r r count_flopsuszQKVAttentionLegacy.count_flops r8r9r:r;rr7 staticmethodrr=r r r(rrZs  rcs4eZdZdZfddZddZeddZZS)r%zP A module which performs QKV attention and splits in a different order. cst||_dSr rrr(r rrs zQKVAttention.__init__c Cs|j\}}}|d|jdks"J|d|j}|jddd\}}}dtt|} td|| ||j|||| ||j||} tj| dd | j } td| | ||j||} | |d|S)z Apply QKV attention. :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs. :return: an [N x (H * C) x T] tensor after attention. rrrr-rr*r) r.rrurrrrviewrrrsr3r/rr r rr7s zQKVAttention.forwardcCs t|||Sr rrr r rrszQKVAttention.count_flopsrr r r(rr%zs  r%cs$eZdZfddZddZZS)Timestepcst||_dSr )rrr+)r'r+r(r rrs zTimestep.__init__cCs t||jSr )r r+)r'tr r rr7szTimestep.forward)r8r9r:rr7r=r r r(rrs rcs<eZdZdZdfd d Zd d ZddZdddZZS) UNetModela The full UNet model with attention and timestep embedding. :param in_channels: channels in the input Tensor. :param model_channels: base channel count for the model. :param out_channels: channels in the output Tensor. :param num_res_blocks: number of residual blocks per downsample. :param attention_resolutions: a collection of downsample rates at which attention will take place. May be a set, list, or tuple. For example, if this contains 4, then at 4x downsampling, attention will be used. :param dropout: the dropout probability. :param channel_mult: channel multiplier for each level of the UNet. :param conv_resample: if True, use learned convolutions for upsampling and downsampling. :param dims: determines if the signal is 1D, 2D, or 3D. :param num_classes: if specified (as an int), then this model will be class-conditional with `num_classes` classes. :param use_checkpoint: use gradient checkpointing to reduce memory usage. :param num_heads: the number of attention heads in each attention layer. :param num_heads_channels: if specified, ignore num_heads and instead use a fixed channel width per attention head. :param num_heads_upsample: works with num_heads to set a different number of heads for upsampling. Deprecated. :param use_scale_shift_norm: use a FiLM-like conditioning mechanism. :param resblock_updown: use residual blocks for up/downsampling. :param use_new_attention_order: use a different attention pattern for potentially increased efficiency. rrrrMTrNFr*rc-st|r|dusJd|durR|s2Jdddlm}t||krRt|}|dkr^|}|dkrv|dksvJd|dkr|dksJd|_|_|_|_ t |t rt ||g_ nt |t |krtd|_ |durt |t |ksJdurZt t j ksJttfdd tt sDJtd d |d |_|_|_| _| _| _| rtjntj_|rtjnj_|_|_|_ |du_!|d } t"#t$|| t"%t$| | _&jdurt jt rt"'| | _(nljdkr8tdt")d| _(nHjdkrz|dusRJt"#t"#t$|| t"%t$| | _(ntt"*t+t,| ||dddg_-|_.|g}!|}"d}#t/|D]\}$}%tj |$D]}&t0|"| ||%|| | |dg}'|%|}"|#|vr|dkr|"|}(n |"|}|}(|r>|r:|"|n|}(t1|rR||$})nd})t1rn|&|$kr|'2|st3|"| ||(|dnt4|"||(|||)|| dj-2t+|'j.|"7_.|!2|"q|$t |dkr|"}*j-2t+|r t0|"| ||*| | |ddnt5|"| | |*d|*}"|!2|"|#d9}#j.|"7_.q|dkr\|"|}(n |"|}|}(|r|r||"|n|}(t+t0|"| || | |d|st3|"| ||(|dnt4|"||(||||| dt0|"| || | |d_6j.|"7_.t"*g_7tt/|dddD]r\}$}%tj |$dD]R}+|!8},t0|"|,| |||%| | |dg}'||%}"|#|vr|dkrz|"|}(n |"|}|}(|r|r|"|n|}(t1|r||$})nd})t1r|+|$kr|'2|st3|"| ||(|dnt4|"||(|||)|| d|$rZ|+j |$krZ|"}*|'2|r@t0|"| ||*| | |ddnt9|"| | |*d|#d}#j72t+|'j.|"7_.q&q t"#t:|"t"%t;t,| ||ddd_<j!rt"#t:|"t,| ||d_=dS)NzRFool!! You forgot to include the dimension of your cross-attention conditioning...zYFool!! You forgot to use the spatial transformer for your cross-attention conditioning...r) ListConfigr*z3Either num_heads or num_head_channels has to be setzprovide num_res_blocks either as an int (globally constant) or as a list/tuple (per-level) with the same length as channel_multcsj||kSr )num_res_blocks)inum_attention_blocksr'r rz$UNetModel.__init__..z7Constructor of UNetModel received num_attention_blocks=z;. This option has LESS priority than attention_resolutions zz, i.e., in cases where num_attention_blocks[i] > 0 but 2**i not in attention_resolutions, attention will still not be set.rM continuousz'setting up linear c_adm embedding layerr sequentialrrF)rIrKrcrdF)rcr$rr)depth context_dimdisable_self_attn use_linearrcT)rIrKrcrdrp)rKrIr)rKrcrd)rIrKrcrdr[)>rromegaconf.listconfigrrslist image_size in_channelsmodel_channelsrIrBr<rtr ValueErrorallmaprangeprintattention_resolutionsrb channel_mult conv_resample num_classesrcrfloat16float32r3bfloat16r$rnum_heads_upsamplepredict_codebook_idsrrerrf time_embed Embedding label_embLinear ModuleListrAr input_blocks _feature_size enumerater_r appendrr r] middle_block output_blockspoprErrout id_predictor)-r'rrrrIrrrbrrrKrrcuse_fp16Zuse_bf16r$rrrdresblock_updownruse_spatial_transformertransformer_depthrn_embedlegacydisable_self_attentionsrdisable_middle_self_attnuse_linear_in_transformeradm_in_channelsrtime_embed_diminput_block_chansrdslevelmultnrlayersdim_head disabled_saout_chrichr(rrrs!      &                                 zUNetModel.__init__cCs(|jt|jt|jtdS)z< Convert the torso of the model to float16. N)rapplyrrrr'r r rconvert_to_fp16s  zUNetModel.convert_to_fp16cCs(|jt|jt|jtdS)z< Convert the torso of the model to float32. N)rrrrrrr r rconvert_to_fp32s  zUNetModel.convert_to_fp32c Ks|du|jduksJdg}t||jdd}||}|jdurh|jd|jdksZJ|||}||j} |jD]} | | ||} | | qz| | ||} |j D]&} t j | |gdd} | | ||} q| |j} |jr|| S|| SdS)aQ Apply the model to an input batch. :param x: an [N x C x ...] Tensor of inputs. :param timesteps: a 1-D batch of timesteps. :param context: conditioning plugged in via crossattn :param y: an [N] Tensor of labels, if class-conditional. :return: an [N x C x ...] Tensor of outputs. Nz%6r))abcrrnumpyrtorchrtorch.nnrZtorch.nn.functional functionalrQ!ldm.modules.diffusionmodules.utilrrrrrrr Zldm.modules.attentionr ldm.utilr rrModulerr>rerArErUr]r_rrrr%rrr r r rs.  $    s1 "