a
    
d.                     @   sd  d dl mZ d dlZd dlZd dlm  mZ d dlmZmZ d dl	m
Z
mZ d dlmZmZ d dlmZ zd dlZd dlZdZW n   dZY n0 d dlZejd	d
Zdd Zdd Zdd Zdd Zdd ZG dd dejZG dd dejZdd Z dd Z!G dd dejZ"G dd  d ejZ#G d!d" d"ejZ$G d#d$ d$ejZ%G d%d& d&ejZ&dS )'    )
isfunctionN)nneinsum)	rearrangerepeat)OptionalAny)
checkpointTFZATTN_PRECISIONfp32c                 C   s   | d uS N )valr   r   h/var/www/html/stable-diffusion-webui/repositories/stable-diffusion-stability-ai/ldm/modules/attention.pyexists   s    r   c                 C   s   dd | D   S )Nc                 S   s   i | ]
}|d qS )Tr   ).0elr   r   r   
<dictcomp>       zuniq.<locals>.<dictcomp>)keys)arrr   r   r   uniq   s    r   c                 C   s   t | r| S t|r| S |S r   )r   r   )r   dr   r   r   default   s    r   c                 C   s   t | jj S r   )torchfinfodtypemaxtr   r   r   max_neg_value%   s    r   c                 C   s*   | j d }dt| }| | | | S )N   )shapemathsqrtuniform_)tensordimstdr   r   r   init_)   s    
r)   c                       s$   e Zd Z fddZdd Z  ZS )GEGLUc                    s    t    t||d | _d S )N   )super__init__r   Linearproj)selfdim_indim_out	__class__r   r   r-   2   s    
zGEGLU.__init__c                 C   s&   |  |jddd\}}|t| S )Nr+   r    r'   )r/   chunkFgelu)r0   xgater   r   r   forward6   s    zGEGLU.forward__name__
__module____qualname__r-   r;   __classcell__r   r   r3   r   r*   1   s   r*   c                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
FeedForwardN   F        c                    sh   t    t|| }t||}|s<tt||t nt||}t|t	|t||| _
d S r   )r,   r-   intr   r   
Sequentialr.   GELUr*   Dropoutnet)r0   r'   r2   multgludropout	inner_dim
project_inr3   r   r   r-   <   s    



zFeedForward.__init__c                 C   s
   |  |S r   )rH   )r0   r9   r   r   r   r;   K   s    zFeedForward.forward)NrB   FrC   r<   r   r   r3   r   rA   ;   s   rA   c                 C   s   |   D ]}|   q| S )z<
    Zero out the parameters of a module and return it.
    )
parametersdetachzero_)modulepr   r   r   zero_moduleO   s    rS   c                 C   s   t jjd| dddS )N    gư>T)
num_groupsnum_channelsepsaffine)r   r   	GroupNorm)in_channelsr   r   r   	NormalizeX   s    r[   c                       s$   e Zd Z fddZdd Z  ZS )SpatialSelfAttentionc                    s~   t    || _t|| _tjj||dddd| _tjj||dddd| _	tjj||dddd| _
tjj||dddd| _d S )Nr!   r   kernel_sizestridepadding)r,   r-   rZ   r[   normr   r   Conv2dqkvproj_out)r0   rZ   r3   r   r   r-   ]   s6    

zSpatialSelfAttention.__init__c                 C   s   |}|  |}| |}| |}| |}|j\}}}}	t|d}t|d}td||}
|
t|d  }
tj	j
j|
dd}
t|d}t|
d}
td||
}t|d|d	}| |}|| S )
Nb c h w -> b (h w) czb c h w -> b c (h w)zbij,bjk->bik      r+   r5   zb i j -> b j izb c (h w) -> b c h wh)ra   rc   rd   re   r"   r   r   r   rD   r   
functionalsoftmaxrf   )r0   r9   h_rc   rd   re   bcrj   ww_r   r   r   r;   w   s"    








zSpatialSelfAttention.forwardr<   r   r   r3   r   r\   \   s   r\   c                       s(   e Zd Zd	 fdd	Zd
ddZ  ZS )CrossAttentionN   @   rC   c                    s   t    || }t||}|d | _|| _tj||dd| _tj||dd| _tj||dd| _	t
t||t|| _d S )Nrh   Fbias)r,   r-   r   scaleheadsr   r.   to_qto_kto_vrE   rG   to_outr0   	query_dimcontext_dimrx   dim_headrK   rL   r3   r   r   r-      s    



zCrossAttention.__init__c           
         s,  | j  | |}t||}| |}| |}t fdd|||f\}}}tdkrtjddd4 |	 |	  }}t
d||| j }W d    q1 s0    Y  nt
d||| j }~~t|rt|d}t|jj }t|d	 d
}|| | |jdd}t
d||}	t|	d d
}	| |	S )Nc                    s   t | d dS )Nzb n (h d) -> (b h) n dri   )r   r   ri   r   r   <lambda>   r   z(CrossAttention.forward.<locals>.<lambda>r
   Fcuda)enableddevice_typezb i d, b j d -> b i jzb ... -> b (...)zb j -> (b h) () jri   r    r5   zb i j, b j d -> b i dz(b h) n d -> b n (h d))rx   ry   r   rz   r{   map_ATTN_PRECISIONr   autocastfloatr   rw   r   r   r   r   r   r   masked_fill_rl   r|   )
r0   r9   contextmaskrc   rd   re   simr   outr   ri   r   r;      s*    



2
zCrossAttention.forward)Nrs   rt   rC   )NNr<   r   r   r3   r   rr      s   rr   c                       s(   e Zd Zd	 fdd	Zd
ddZ  ZS )MemoryEfficientCrossAttentionNrs   rt   rC   c              
      s   t    td| jj d| d| d| d	 || }t||}|| _|| _tj	||dd| _
tj	||dd| _tj	||dd| _tt	||t|| _d | _d S )NzSetting up z. Query dim is z, context_dim is z and using z heads.Fru   )r,   r-   printr4   r=   r   rx   r   r   r.   ry   rz   r{   rE   rG   r|   attention_opr}   r3   r   r   r-      s    


z&MemoryEfficientCrossAttention.__init__c           	         s    |}t||}|}|}|j\ }}t fdd|||f\}}}tjj|||d j	d}t
|rxt|d j|jd jdddd |jd jj }|S )Nc                    sH   |  d | jd jjdddd j | jd j S )N   r!   r   r+   )	unsqueezereshaper"   rx   r   permute
contiguousr   rn   r0   r   r   r      s   
z7MemoryEfficientCrossAttention.forward.<locals>.<lambda>)	attn_biasopr   r!   r+   r   )ry   r   rz   r{   r"   r   xformersopsmemory_efficient_attentionr   r   NotImplementedErrorr   r   rx   r   r   r|   )	r0   r9   r   r   rc   rd   re   _r   r   r   r   r;      s*    






z%MemoryEfficientCrossAttention.forward)Nrs   rt   rC   )NNr<   r   r   r3   r   r      s   r   c                       s<   e Zd ZeedZd fdd	Zddd	Zdd
dZ  Z	S )BasicTransformerBlock)rl   softmax-xformersrC   NTFc	                    s   t    trdnd}	|	| jv s$J | j|	 }
|| _|
||||| jrH|nd d| _t|||d| _|
|||||d| _t	
|| _t	
|| _t	
|| _|| _d S )Nr   rl   )r~   rx   r   rK   r   )rK   rJ   )r~   r   rx   r   rK   )r,   r-   XFORMERS_IS_AVAILBLEATTENTION_MODESdisable_self_attnattn1rA   ffattn2r   	LayerNormnorm1norm2norm3r	   )r0   r'   n_headsd_headrK   r   gated_ffr	   r   	attn_modeattn_clsr3   r   r   r-      s     


zBasicTransformerBlock.__init__c                 C   s   t | j||f|  | j S r   )r	   _forwardrN   r0   r9   r   r   r   r   r;     s    zBasicTransformerBlock.forwardc                 C   sR   | j | || jr|nd d| }| j| ||d| }| | || }|S )Nr   )r   r   r   r   r   r   r   r   r   r   r   r     s    "zBasicTransformerBlock._forward)rC   NTTF)N)N)
r=   r>   r?   rr   r   r   r-   r;   r   r@   r   r   r3   r   r      s     
r   c                       s,   e Zd ZdZd fdd	Zdd	d
Z  ZS )SpatialTransformera  
    Transformer block for image-like data.
    First, project the input (aka embedding)
    and reshape to b, t, d.
    Then apply standard transformer action.
    Finally, reshape to image
    NEW: use_linear for more efficiency instead of the 1x1 convs
    r!   rC   NFTc
           
   	      s   t    t r"t ts" g || _ t|| _|sVtj	|dddd| _
nt|| _
t fddt|D | _|sttj	|dddd| _ntt|| _|| _d S )Nr!   r   r]   c                    s&   g | ]}t  | d qS ))rK   r   r   r	   )r   )r   r   r   r   r   rK   rL   r   use_checkpointr   r   
<listcomp>3  s   z/SpatialTransformer.__init__.<locals>.<listcomp>)r,   r-   r   
isinstancelistrZ   r[   ra   r   rb   proj_inr.   
ModuleListrangetransformer_blocksrS   rf   
use_linear)
r0   rZ   r   r   depthrK   r   r   r   r   r3   r   r   r-     s8    


zSpatialTransformer.__init__c           
      C   s   t |ts|g}|j\}}}}|}| |}| js<| |}t|d }| jrZ| |}t| j	D ]\}}	|	||| d}qd| jr| 
|}t|d||d }| js| 
|}|| S )Nrg   r   zb (h w) c -> b c h w)rj   rp   )r   r   r"   ra   r   r   r   r   	enumerater   rf   )
r0   r9   r   rn   ro   rj   rp   x_iniblockr   r   r   r;   A  s$    





zSpatialTransformer.forward)r!   rC   NFFT)N)r=   r>   r?   __doc__r-   r;   r@   r   r   r3   r   r     s   	   "r   )'inspectr   r#   r   torch.nn.functionalr   rk   r7   r   einopsr   r   typingr   r   !ldm.modules.diffusionmodules.utilr	   r   Zxformers.opsr   osenvirongetr   r   r   r   r   r)   Moduler*   rA   rS   r[   r\   rr   r   r   r   r   r   r   r   <module>   s:   

	541 