a  d-@sdZddlZddlmZmZddlZddlmZddlmm Z ddl m Z ddl mZdeeefeeeefeejdd d ZGd d d ejZdeeefeeefdddZGdddejZdeeedddZeeejejejdddZGdddejZdS)zf Relative position embedding modules and functions Hacked together by / Copyright 2022 Ross Wightman N)OptionalTuple)Mlp) trunc_normal_F)q_sizek_size class_tokenreturnc Cs|durttt|dt|dgd}|dddddf|dddddf}|ddd}d|ddd|ddd}nttt|dt|dgd}ttt|dt|dgd}|dddddf|dddddf}|ddd}|d|dd|d|ddd}tj|ddddd\}} |rt | gd} |d| dddf<|d| dddf<|d| d <| S) NrrT)Zreturn_inverseZdim)rrrrrr) torchstackmeshgridarangeflattenpermuteuniqueviewFpad contiguous) rrr ZcoordsZrelative_coordsnum_relative_distanceZq_coordsZk_coords_relative_position_indexrb/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/pos_embed_rel.pygen_relative_position_indexsL  ,&    ,, rcsPeZdZdZdfdd ZddZejddd Zde ejd d d Z Z S) RelPosBiasz_ Relative Position Bias Adapted from Swin-V1 relative position bias impl, modularized. rcst|dksJ||_|d|d|_|j|fd|f|_d|ddd|ddd|}tt|||_ |j dt |j|dkd ddd | dS) Nrrr r r)r r F persistent)super__init__ window_size window_area bias_shapenn Parameterrzerosrelative_position_bias_tableregister_bufferrr init_weights)selfr% num_heads prefix_tokensr __class__rrr$Is  (zRelPosBias.__init__cCst|jdddSNg{Gz?)Zstd)rr+r.rrrr-ZszRelPosBias.init_weightsr cCs0|j|j}||jddd}|dS)Nr rr)r+rrr'r unsqueezerr.Zrelative_position_biasrrrget_bias]s zRelPosBias.get_biasNshared_rel_poscCs ||SNr8r.Zattnr:rrrforwardcszRelPosBias.forward)r)N __name__ __module__ __qualname____doc__r$r-rTensorr8rr> __classcell__rrr1rr Ds r rswin)win_sizepretrained_win_sizecCsr|dvs Jtj|dd |dtjd}tj|dd |dtjd}tt||g}|ddd}|dkrR|ddkr|dddddf|dd<|dddddf|dd<nL|dddddf|dd<|dddddf|dd<|d9}t|td| t d}nt|t d| }|S) N)rFcrrr)Zdtyper rFg?) rrfloat32rrrrsignlog2absmathlog)rGrHmodeZrelative_coords_hZrelative_coords_wZrelative_coords_tablerrrgen_relative_log_coordsgs*     &(&&    rRcsHeZdZdZdfdd Zejd d d Zdeejd ddZ Z S) RelPosMlpz Log-Coordinate Relative Position MLP Based on ideas presented in Swin-V2 paper (https://arxiv.org/abs/2111.09883) This impl covers the 'swin' implementation as well as two timm specific modes ('cr', and 'rw') rJrrIrcst||_|jd|jd|_||_||_|jfd|f|_|dkrdt|_ d|_ d}nt |_ d|_ d}t d||tj |dd |_|jd t|d d d |jdt|||dd d dS)Nrrr rF)TFT)g?g)Zhidden_featuresZ out_featuresZ act_layerZbiasZdroprr Fr!rel_coords_log)rQ)r#r$r%r&r0r/r'r(ZSigmoidbias_act bias_gainZIdentityrZReLUmlpr,rrrR)r.r%r/Z hidden_dimr0rQZpretrained_window_sizeZmlp_biasr1rrr$s>      zRelPosMlp.__init__r5cCs||j}|jdur6|d|j|j}||j}|ddd}||}|jdurb|j|}|j rt ||j d|j dg}| d S)Nr r rr)rYrVrrr/r'rrWrXr0rrr6rr7rrrr8s      zRelPosMlp.get_biasNr9cCs ||Sr;r<r=rrrr>szRelPosMlp.forward)rJrTrrIr)N) r@rArBrCr$rrDr8rr>rErrr1rrSs, rS)lengthmax_relative_positioncCsv|dur|d}d|d}t|||}t|D]>}t|D]0}|||}t|||kr`q>d||||f<q>q2|S)aGenerate a one_hot lookup tensor to reindex embeddings along one dimension. Args: length: the length to reindex to. max_relative_position: the maximum relative position to consider. Relative position embeddings for distances above this threshold are zeroed out. Returns: a lookup Tensor of size [length, length, vocab_size] that satisfies ret[n,m,v] = 1{m - n + max_relative_position = v}. Nrr )rr*rangerN)rZr[Z vocab_sizeretixvrrrgenerate_lookup_tensors    ra)heightwidth height_lookup width_lookupr cCs8td||}td||}||}||jd||S)a\Reindex 2d relative position bias with 2 independent einsum lookups. Adapted from: https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py Args: relative_position_tensor: tensor of shape [..., vocab_height, vocab_width, ...]. height: height to reindex to. width: width to reindex to. height_lookup: one-hot height lookup width_lookup: one-hot width lookup Returns: reindexed_tensor: a Tensor of shape [..., height * width, height * width, ...] z nhw,ixh->nixwznixw,jyw->nijxyr)rZeinsumZreshapeshape)Zrelative_position_tensorrbrcrdreZreindexed_tensorarearrrreindex_2d_einsum_lookupsrhcsPeZdZdZdfdd ZddZejddd Zde ejd d d Z Z S) RelPosBiasTfz Relative Position Bias Impl (Compatible with Tensorflow MaxViT models) Adapted from: https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py rcst|dksJ||_|d|d|_||_d|dd}d|dd}|j||f|_tt |j|_ |j dt |ddd|j dt |ddd| dS)Nrrr rdFr!re)r#r$r%r&r/r'r(r)rr*r+r,rar-)r.r%r/r0Z vocab_heightZ vocab_widthr1rrr$s  zRelPosBiasTf.__init__cCstjj|jdddSr3)r(initZnormal_r+r4rrrr-szRelPosBiasTf.init_weightsr5cCs"t|j|jd|jd|j|jS)Nrr)rhr+r%rdrer4rrrr8szRelPosBiasTf.get_biasNr9cCs ||Sr;r<r=rrrr>"szRelPosBiasTf.forward)r)Nr?rrr1rris  ri)NF)rrF)N)rCrOtypingrrrZtorch.nnr(Ztorch.nn.functionalZ functionalrrYrZ weight_initrintboolrDrModuler rRrSrarhrirrrrsF     4%  E