a
    
d
Q                     @   s  d dl Z d dlmZ d dlmZmZ d dlZd dlm  m	Z
 d dlmZmZ d dlmZ d dlmZ eejedkrdZd d	lmZmZ ejdd
d
dejd
dd
dejd
d
dddddddiZn*d dlmZ d
ZeZi Zedej d zd dlZd dlZdZW n   d
Zed Y n0 ddl m!Z! dd Z"dd Z#dd Z$dd Z%dd Z&G dd dej'Z(G dd dej'Z)d d! Z*d"d# Z+G d$d% d%ej'Z,G d&d' d'ej'Z-G d(d) d)ej'Z.G d*d+ d+ej'Z/G d,d- d-ej'Z0G d.d/ d/ej'Z1G d0d1 d1ej'Z2dS )2    N)
isfunction)AnyOptional)	rearrangerepeat)version)nn2.0.0T)
SDPBackend
sdp_kernelF)enable_mathenable_flashenable_mem_efficient)nullcontextzsNo SDP backend available, likely because you are running in pytorch versions < 2.0. In fact, you are using PyTorch z'. You might want to consider upgrading.z+no module 'xformers'. Processing without...   )
checkpointc                 C   s   | d uS N )valr   r   \/var/www/html/stable-diffusion-webui/repositories/generative-models/sgm/modules/attention.pyexists8   s    r   c                 C   s   dd | D   S )Nc                 S   s   i | ]
}|d qS )Tr   ).0elr   r   r   
<dictcomp>=       zuniq.<locals>.<dictcomp>)keys)arrr   r   r   uniq<   s    r   c                 C   s   t | r| S t|r| S |S r   )r   r   )r   dr   r   r   default@   s    r   c                 C   s   t | jj S r   )torchfinfodtypemaxtr   r   r   max_neg_valueF   s    r&   c                 C   s*   | j d }dt| }| | | | S )Nr   )shapemathsqrtuniform_)tensordimstdr   r   r   init_J   s    
r/   c                       s$   e Zd Z fddZdd Z  ZS )GEGLUc                    s    t    t||d | _d S )N   )super__init__r   Linearproj)selfdim_indim_out	__class__r   r   r3   S   s    
zGEGLU.__init__c                 C   s&   |  |jddd\}}|t| S )Nr1   r'   r-   )r5   chunkFgelu)r6   xgater   r   r   forwardW   s    zGEGLU.forward__name__
__module____qualname__r3   rA   __classcell__r   r   r9   r   r0   R   s   r0   c                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
FeedForwardN   F        c                    sh   t    t|| }t||}|s<tt||t nt||}t|t	|t||| _
d S r   )r2   r3   intr   r   
Sequentialr4   GELUr0   Dropoutnet)r6   r-   r8   multgludropout	inner_dimZ
project_inr9   r   r   r3   ]   s    

zFeedForward.__init__c                 C   s
   |  |S r   )rN   )r6   r?   r   r   r   rA   k   s    zFeedForward.forward)NrH   FrI   rB   r   r   r9   r   rG   \   s   rG   c                 C   s   |   D ]}|   q| S )z<
    Zero out the parameters of a module and return it.
    )
parametersdetachzero_)modulepr   r   r   zero_moduleo   s    rX   c                 C   s   t jjd| dddS )N    gư>T)
num_groupsnum_channelsepsaffine)r    r   	GroupNorm)in_channelsr   r   r   	Normalizex   s    r`   c                       s&   e Zd Zd fdd	Zdd Z  ZS )LinearAttentionrH   rY   c                    sD   t    || _|| }tj||d ddd| _t||d| _d S )N   r   Fbias)r2   r3   headsr   Conv2dto_qkvto_out)r6   r-   re   dim_head
hidden_dimr9   r   r   r3      s
    
zLinearAttention.__init__c                 C   sv   |j \}}}}| |}t|d| jdd\}}}	|jdd}td||	}
td|
|}t|d| j||d	}| |S )
Nz*b (qkv heads c) h w -> qkv b heads c (h w)rb   )re   qkvr'   r;   zbhdn,bhen->bhdezbhde,bhdn->bhenz"b heads c (h w) -> b (heads c) h w)re   hw)r(   rg   r   re   softmaxr    einsumrh   )r6   r?   bcrl   rm   rk   qkvcontextoutr   r   r   rA      s    

zLinearAttention.forward)rH   rY   rB   r   r   r9   r   ra   ~   s   ra   c                       s$   e Zd Z fddZdd Z  ZS )SpatialSelfAttentionc                    s~   t    || _t|| _tjj||dddd| _tjj||dddd| _	tjj||dddd| _
tjj||dddd| _d S )Nr   r   kernel_sizestridepadding)r2   r3   r_   r`   normr    r   rf   rr   rs   rt   proj_out)r6   r_   r9   r   r   r3      s    





zSpatialSelfAttention.__init__c                 C   s   |}|  |}| |}| |}| |}|j\}}}}	t|d}t|d}td||}
|
t|d  }
tj	j
j|
dd}
t|d}t|
d}
td||
}t|d|d	}| |}|| S )
Nb c h w -> b (h w) czb c h w -> b c (h w)zbij,bjk->bik      r1   r;   zb i j -> b j izb c (h w) -> b c h wrl   )r|   rr   rs   rt   r(   r   r    ro   rJ   r   
functionalrn   r}   )r6   r?   h_rr   rs   rt   rp   rq   rl   rm   w_r   r   r   rA      s"    








zSpatialSelfAttention.forwardrB   r   r   r9   r   rw      s   rw   c                       s(   e Zd Zd
 fdd	Zddd	Z  ZS )CrossAttentionN   @   rI   c                    s   t    || }t||}|d | _|| _tj||dd| _tj||dd| _tj||dd| _	t
t||t|| _|| _d S )Nr   Frc   )r2   r3   r   scalere   r   r4   to_qto_kto_vrK   rM   rh   backend)r6   	query_dimcontext_dimre   ri   rQ   r   rR   r9   r   r   r3      s    	


zCrossAttention.__init__r   c                    sL  | j  |d ur*|jd }tj||gdd}| |}t||}| |}| |}	|r|jd | dkslJ |jd | }
t|d d | d|
d}t|	d d | d|
d}	t	 fdd|||	f\}}}	t
f i t| j " tj|||	|d}W d    n1 s0    Y  ~~~	t|d	 d
}|d urB|d d |d f }| |S )Nr   r;   r   b ... -> (b n) ...nc                    s   t | d dS )Nzb n (h d) -> b h n dr   )r   r$   r   r   r   <lambda>   r   z(CrossAttention.forward.<locals>.<lambda>)	attn_maskzb h n d -> b n (h d)r   )re   r(   r    catr   r   r   r   r   mapr   BACKEND_MAPr   r=   scaled_dot_product_attentionr   rh   )r6   r?   ru   maskadditional_tokensn_times_crossframe_attn_in_selfn_tokens_to_maskrr   rs   rt   Zn_cprv   r   r   r   rA      s6    




&
zCrossAttention.forward)Nr   r   rI   N)NNNr   rB   r   r   r9   r   r      s            r   c                       s(   e Zd Zd
 fdd	Zddd	Z  ZS )MemoryEfficientCrossAttentionNr   r   rI   c                    s   t    td| jj d| d| d| d| d || }t||}|| _|| _tj	||dd| _
tj	||dd| _tj	||dd| _tt	||t|| _d | _d S )	NzSetting up z. Query dim is z, context_dim is z and using z heads with a dimension of .Frc   )r2   r3   printr:   rC   r   re   ri   r   r4   r   r   r   rK   rM   rh   attention_op)r6   r   r   re   ri   rQ   kwargsrR   r9   r   r   r3      s&    

z&MemoryEfficientCrossAttention.__init__r   c                    sR  |d ur$|j d }tj||gdd}|}t||}|}|}	|r|j d | dksfJ t|d d | d|d}t|	d d | d|d}	|j \ }
}
t fdd|||	f\}}}	t	j
j|||	d jd}t|rt|d j|j d jdd	dd
 |j d jj }|d urH|d d |d f }|S )Nr   r;   r   r   r   c                    sH   |  d | jd jjdddd j | jd j S )Nrb   r   r   r1   )	unsqueezereshaper(   re   ri   permute
contiguousr$   rp   r6   r   r   r   Z  s   
z7MemoryEfficientCrossAttention.forward.<locals>.<lambda>)	attn_biasopr1   rb   )r(   r    r   r   r   r   r   r   r   xformersopsmemory_efficient_attentionr   r   NotImplementedErrorr   r   re   ri   r   rh   )r6   r?   ru   r   r   r   r   rr   rs   rt   _rv   r   r   r   rA   7  sP    








z%MemoryEfficientCrossAttention.forward)Nr   r   rI   )NNNr   rB   r   r   r9   r   r     s        r   c                       s<   e Zd ZeedZd fdd	Zdd
dZdddZ  Z	S )BasicTransformerBlockrn   softmax-xformersrI   NTFrn   c                    sJ  t    |	| jv sJ |	dkr@ts@td|	 dtj  d}	n2|	dkrrtsrtd tsfdsrJ dntd d}	| j|	 }t	tjt	d	kr|
d u st
|
tsJ n|
d u sJ || _|||||| jr|nd |
d
| _t|||d| _|||||||
d| _t|| _t|| _t|| _|| _| jrFt| jj d d S )Nrn   zAttention mode 'z' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version zKWe do not support vanilla attention anymore, as it is too expensive. Sorry.Fz?Please install xformers via e.g. 'pip install xformers==0.0.16'z-Falling back to xformers efficient attention.r   r	   )r   re   ri   rQ   r   r   rQ   rP   )r   r   re   ri   rQ   r   z is using checkpointing)r2   r3   ATTENTION_MODESXFORMERS_IS_AVAILABLEr   r    __version__SDP_IS_AVAILABLEr   parse
isinstancer
   disable_self_attnattn1rG   ffattn2r   	LayerNormnorm1norm2norm3r   r:   rC   )r6   r-   n_headsd_headrQ   r   gated_ffr   r   	attn_modesdp_backendattn_clsr9   r   r   r3   |  s`    

zBasicTransformerBlock.__init__r   c                 C   s`   d|i}|d ur| d|i |d ur4| d|i |rF| d|i t| j||f|  | jS )Nr?   ru   r   r   )updater   _forwardrS   )r6   r?   ru   r   r   r   r   r   r   rA     s    zBasicTransformerBlock.forwardc                 C   sb   | j | || jr|nd || js$|ndd| }| j| |||d| }| | || }|S )Nr   )ru   r   r   )ru   r   )r   r   r   r   r   r   r   )r6   r?   ru   r   r   r   r   r   r     s&    zBasicTransformerBlock._forward)rI   NTTFrn   N)NNr   )NNr   
rC   rD   rE   r   r   r   r3   rA   r   rF   r   r   r9   r   r   v  s   
       ? 
 r   c                       s<   e Zd ZeedZd fdd	Zddd	Zdd
dZ  Z	S ) BasicTransformerSingleLayerBlockr   rI   NTrn   c	           
         sh   t    || jv sJ | j| }	|	|||||d| _t|||d| _t|| _t|| _	|| _
d S )N)r   re   ri   rQ   r   r   )r2   r3   r   r   rG   r   r   r   r   r   r   )
r6   r-   r   r   rQ   r   r   r   r   r   r9   r   r   r3     s    

z)BasicTransformerSingleLayerBlock.__init__c                 C   s   t | j||f|  | j S r   )r   r   rS   r6   r?   ru   r   r   r   rA     s    z(BasicTransformerSingleLayerBlock.forwardc                 C   s0   | j | ||d| }| | || }|S )Nru   )r   r   r   r   r   r   r   r   r     s    z)BasicTransformerSingleLayerBlock._forward)rI   NTTrn   )N)Nr   r   r   r9   r   r     s        
r   c                       s,   e Zd ZdZd fdd		Zdd
dZ  ZS )SpatialTransformera  
    Transformer block for image-like data.
    First, project the input (aka embedding)
    and reshape to b, t, d.
    Then apply standard transformer action.
    Finally, reshape to image
    NEW: use_linear for more efficiency instead of the 1x1 convs
    r   rI   NFrn   Tc                    s  t    td| jj d| d| d d	 ddlm} trXtt	|fsXgtrtt	r|t
krtd| jj d	 dt
 d
| d|d g  d ttfddsJ d|d g nd u rd g| || _ t|| _|s&tj|dddd| _nt|| _t f	ddt|D | _|sttj|dddd| _ntt|| _|| _d S )Nzconstructing z
 of depth z w/ z channels and z headsr   )
ListConfigz	WARNING: z: Found context dims z0, which does not match the specified 'depth' of z. Setting context_dim to z now.c                    s   |  d kS )Nr   r   )r?   )r   r   r   r   <  r   z-SpatialTransformer.__init__.<locals>.<lambda>z8need homogenous context_dim to match depth automaticallyr   rx   c                    s*   g | ]"}t |  d 	qS ))rQ   r   r   r   r   r   )r   )r   r   		attn_typer   r   r   rQ   rR   r   r   use_checkpointr   r   
<listcomp>L  s   z/SpatialTransformer.__init__.<locals>.<listcomp>)r2   r3   r   r:   rC   	omegaconfr   r   r   listlenallr   r_   r`   r|   r   rf   proj_inr4   
ModuleListrangetransformer_blocksrX   r}   
use_linear)r6   r_   r   r   depthrQ   r   r   r   r   r   r   r   r9   r   r   r3     sV    
 



zSpatialTransformer.__init__c           
      C   s   t |ts|g}|j\}}}}|}| |}| js<| |}t|d }| jrZ| |}t| j	D ]0\}}	|dkrt
|dkrd}|	||| d}qd| jr| |}t|d||d }| js| |}|| S )Nr~   r   r   r   zb (h w) c -> b c h w)rl   rm   )r   r   r(   r|   r   r   r   r   	enumerater   r   r}   )
r6   r?   ru   rp   rq   rl   rm   x_iniblockr   r   r   rA   d  s(    





zSpatialTransformer.forward)r   rI   NFFrn   TN)N)rC   rD   rE   __doc__r3   rA   rF   r   r   r9   r   r     s           Gr   )3r)   inspectr   typingr   r   r    Ztorch.nn.functionalr   r   r=   einopsr   r   	packagingr   r   r   r   torch.backends.cudar
   r   MATHFLASH_ATTENTIONEFFICIENT_ATTENTIONr   
contextlibr   r   r   xformers.opsr   Zdiffusionmodules.utilr   r   r   r   r&   r/   Moduler0   rG   rX   r`   ra   rw   r   r   r   r   r   r   r   r   r   <module>   sx   
	-\Xq,