a  d7@sdZddlZddlmZmZmZmZddlZddlmZddl m Z ddej dfe e eejeejd d d Zd d ej dfe e e ejeejejdddZdd ddej dfee e e eeejeejejdddZdddd ddddej df ee eeje e e eeeeee ejeejeejd ddZGdddejZddZejddd Zeejdd!d"Zejdd#d$Zd%d&Zdddd dddej df ee eeje e e eeeee ejeejd' d(d)ZGd*d+d+ejZGd,d-d-ejZdS).zv Sin-cos, fourier, rotary position embedding modules and functions Hacked together by / Copyright 2022 Ross Wightman N)ListTupleOptionalUnion)nn)_assertgl@T) num_bandsmax_freq linear_bandsdtypedevicecCsL|rtjd|d|||d}n$dtjdt|dd|||d}|tjS)N?r r rr)torchlinspacemathlogpi)r r r r r bandsre/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/pos_embed_sincos.pypixel_freq_bandss$rg@r)r temperaturestepr r returnc Cs$d|tjd||||d|}|S)NrrrrZarange)r rrr r rrrr freq_bandss r@F) feat_shapedimr reverse_coordinterleave_sin_cosr r rc s|ddksJd|d}t||dd}|r@|ddd}ttfdd |Dddd} | d|d} |rd nd} tjt| t| g| d d} | S) a Args: feat_shape: dim: temperature: reverse_coord: stack grid order W, H instead of H, W interleave_sin_cos: sin, cos, sin, cos stack instead of sin, sin, cos, cos dtype: device: Returns: rzHEmbed dimension must be divisible by 4 for sin-cos 2D position embeddingrrrr r Ncsg|]}tj|dqSr r r.0sr(rr Ez,build_sincos2d_pos_embed..rr!) rrstackmeshgridflatten transpose unsqueezesincos) r r!rr"r#r r Zpos_dimrgridpos2Z stack_dimZpos_embrr(rbuild_sincos2d_pos_embed's  $r8) r rr max_resrr include_grid in_pixelsref_feat_shaper r rc s|dur8|r$t|t||d}qTt||dd}ndurF|jdurT|j|rnfdd|D} nfdd|D} |durddt| ||D} tjt| d d } | d } | |} | | }}|r| ||gn||g}|S) a[ Args: feat_shape: Feature shape for embedding. bands: Pre-calculated frequency bands. num_bands: Number of frequency bands (determines output dim). max_res: Maximum resolution for pixel based freq. temperature: Temperature for non-pixel freq. linear_bands: Linear band spacing for pixel based freq. include_grid: Include the spatial grid in output. in_pixels: Output in pixel freq. ref_feat_shape: Reference feature shape for resize / fine-tune. dtype: Output dtype. device: Output device. Returns: N)r r r rr%c s g|]}tjdd|dqS)gr)Zstepsr r )rrr)r(rrr,r-z+build_fourier_pos_embed..csg|]}tj|dqSr'rr)r(rrr,r-cSsg|]\}}}|||qSrr)r*xfrrrrr,r-r&r.) rfloatrr r ziprr/r0r3r4r5)r rr r:rr r;r<r=r r tr6posZpos_sinZpos_cosoutrr(rbuild_fourier_pos_embedNs> rFcs.eZdZd eedfdd Zdd ZZS) FourierEmbedr9rTF)r:r cs<t||_||_||_||_|jdt||dddS)NrF persistent)super__init__r:r concat_grid keep_spatialregister_bufferr)selfr:r rLrM __class__rrrKs zFourierEmbed.__init__cCs|jdd\}}|jdd}t||j|j|j|jd}tj|dd}|dd t |}|fd|j d}|j rtj|| d|dd ddgdd}nBCr embZ batch_expandrrrforwards",*zFourierEmbed.forward)r9rTF)__name__ __module__ __qualname__intrKr] __classcell__rrrPrrGsrGcCs6t|ddddf |ddddfgd|jS)N.rrr&)rr/rYrTr>rrrrotsrdrccCsJ|jdkr6||d|t||d|S||t||S)NrSr)rWr3 expand_asrdr>sin_embcos_embrrrapply_rot_embeds ,rics&t|tjr|g}fdd|DS)Ncs g|]}|t|qSr)rd)r*rCrhrgrrr,r-z(apply_rot_embed_list..) isinstancerTensorrfrrjrapply_rot_embed_lists rmcCsZ|dd\}}|jdkrF||d|t||d|S||t||S)Nrr&rSr)Z tensor_splitrWr3rerd)r>r\rgrhrrrapply_rot_embed_cats ,rnc Cs@|d|jddd}|d|ddd|jd}|S)Nrr&r)r3rXrTZgather)r> pos_embedZ keep_indicesrrrapply_keep_indices_nlcs"rp) r rr!r:rr r<r=r r c  Csjt|||d|||||| |d \} } d} |D] } | | 9} q,| | ddd} | | ddd} | | fS)a Args: feat_shape: Spatial shape of the target tensor for embedding. bands: Optional pre-generated frequency bands dim: Output dimension of embedding tensor. max_res: Maximum resolution for pixel mode. temperature: Temperature (inv freq) for non-pixel mode linear_bands: Linearly (instead of log) spaced bands for pixel mode in_pixels: Pixel vs language (inv freq) mode. dtype: Output dtype. device: Output device. Returns: r$) rr r:rr r<r=r r rr&r)rFrYZrepeat_interleave)r rr!r:rr r<r=r r rgrhZnum_spatial_dimr>rrrbuild_rotary_pos_embeds$  rqcs\eZdZdZdeeeeeeedfdd Zdeeed d d Z d dZ Z S)RotaryEmbeddinga Rotary position embedding NOTE: This is my initial attempt at impl rotary embedding for spatial use, it has not been well tested, and will likely change. It will be moved to its own file. The following impl/resources were referenced for this impl: * https://github.com/lucidrains/vit-pytorch/blob/6f3a5fcf0bca1c5ec33a35ef48d97213709df4ba/vit_pytorch/rvt.py * https://blog.eleuther.ai/rotary-embeddings/ r9'TFNr r r=c st||_||_||_||_||_||_|dur|rRt|dt ||d}nt |d|dd}t ||j d|ddd|_ d|_n@t||||||jd\} } d|_|j d | dd|j d | dddS) Nr$r rrrrFrHr r!r:r r<r= pos_embed_sin pos_embed_cos)rJrKr!r:rr<r r=rrArprintrNrxryrqr) rOr!r:rr<r r r=rZemb_sinZemb_cosrPrrrKs\  zRotaryEmbedding.__init__rTcCs8|jdur(|dusJt||j|jdS|j|jfSdS)Nr<)rrqr<rxry)rOrTrrr get_embedYs  zRotaryEmbedding.get_embedcCs$||jdd\}}t|||SNr)r}rTri)rOr>rgrhrrrr]eszRotaryEmbedding.forward)r9rsTFNN)N r^r_r`__doc__boolrrrarKr}r]rbrrrPrrrs   > rrcs\eZdZdZdeeeeeeedfdd Zdeeed d d Z d dZ Z S)RotaryEmbeddingCata Rotary position embedding w/ concatenatd sin & cos The following impl/resources were referenced for this impl: * https://github.com/lucidrains/vit-pytorch/blob/6f3a5fcf0bca1c5ec33a35ef48d97213709df4ba/vit_pytorch/rvt.py * https://blog.eleuther.ai/rotary-embeddings/ r9rsTFNrtc st||_||_||_||_||_||_|dur|rRt|dt ||d}nt |d|dd}t ||j d|ddd|_ n4t||||||jd} d|_|j d t| d dddS) Nr$rurrvrFrHrwror&)rJrKr!r:rr<r r=rrArrzrNembedrqrrrU) rOr!r:rr<r r r=rembedsrPrrrKssP  zRotaryEmbeddingCat.__init__r{cCs@|jdur6t|dudt||j|jd}t|dS|jSdS)Nzvalid shape neededr|r&)rrrqr<rrUro)rOrTrrrrr}s  zRotaryEmbeddingCat.get_embedcCs||jdd}t||Sr~)r}rTrn)rOr>rorrrr]szRotaryEmbeddingCat.forward)r9rsTFNN)NrrrrPrrks   8 r) rrtypingrrrrrrZ trace_utilsrfloat32rarArr r rrlrr8rFModulerGrdrirmrnrprqrrrrrrrs     )  H,  1[