a
    
dc]                     @   sj  d dl Z d dlmZmZmZ d dlZd dlZd dlm	Z	 d dl
mZ d dlmZ zd dlZd dlZdZW n   dZed Y n0 dd	lmZmZ d
d Zdd Zd(ddZG dd de	jZG dd de	jZG dd de	jZG dd deZG dd de	jZG dd de	jZG dd deZd)d d!Z G d"d# d#e	jZ!G d$d% d%e	jZ"G d&d' d'e	jZ#dS )*    N)AnyCallableOptional	rearrange)versionTFz+no module 'xformers'. Processing without...   )LinearAttentionMemoryEfficientCrossAttentionc                 C   s   t | jdksJ |d }td|d  }ttj|tjd|  }|j| j	d}| 
 dddf |dddf  }tjt|t|gdd}|d dkrtjj|d}|S )	a  
    This matches the implementation in Denoising Diffusion Probabilistic Models:
    From Fairseq.
    Build sinusoidal embeddings.
    This matches the implementation in tensor2tensor, but differs slightly
    from the description in Section 3.5 of "Attention Is All You Need".
          i'  )dtype)deviceNdim)r   r   r   r   )lenshapemathlogtorchexparangefloat32tor   floatcatsincosnn
functionalpad)	timestepsembedding_dimZhalf_dimemb r$   i/var/www/html/stable-diffusion-webui/repositories/generative-models/sgm/modules/diffusionmodules/model.pyget_timestep_embedding   s    $r&   c                 C   s   | t |  S N)r   sigmoidxr$   r$   r%   nonlinearity,   s    r+       c                 C   s   t jj|| dddS )Ngư>T)
num_groupsnum_channelsepsaffine)r   r   	GroupNorm)in_channelsr-   r$   r$   r%   	Normalize1   s    r3   c                       s$   e Zd Z fddZdd Z  ZS )Upsamplec                    s2   t    || _| jr.tjj||dddd| _d S )Nr   r   kernel_sizestridepaddingsuper__init__	with_convr   r   Conv2dconvselfr2   r<   	__class__r$   r%   r;   8   s    

zUpsample.__init__c                 C   s(   t jjj|ddd}| jr$| |}|S )Ng       @nearest)scale_factormode)r   r   r   interpolater<   r>   )r@   r*   r$   r$   r%   forward@   s    
zUpsample.forward__name__
__module____qualname__r;   rG   __classcell__r$   r$   rA   r%   r4   7   s   r4   c                       s$   e Zd Z fddZdd Z  ZS )
Downsamplec                    s2   t    || _| jr.tjj||dddd| _d S )Nr   r   r   r5   r9   r?   rA   r$   r%   r;   H   s    

zDownsample.__init__c                 C   sD   | j r,d}tjjj||ddd}| |}ntjjj|ddd}|S )N)r   r   r   r   constantr   )rE   valuer   )r6   r7   )r<   r   r   r   r    r>   
avg_pool2d)r@   r*   r    r$   r$   r%   rG   Q   s    zDownsample.forwardrH   r$   r$   rA   r%   rM   G   s   	rM   c                       s.   e Zd Zdddd fdd
Zdd Z  ZS )	ResnetBlockNFi   )out_channelsconv_shortcuttemb_channelsc                   s   t    || _|d u r|n|}|| _|| _t|| _tjj	||dddd| _
|dkrftj||| _t|| _tj|| _tjj	||dddd| _| j| jkr| jrtjj	||dddd| _ntjj	||dddd| _d S )Nr   r   r5   r   )r:   r;   r2   rR   use_conv_shortcutr3   norm1r   r   r=   conv1Linear	temb_projnorm2Dropoutdropoutconv2rS   nin_shortcut)r@   r2   rR   rS   r\   rT   rA   r$   r%   r;   \   s0    	







zResnetBlock.__init__c                 C   s   |}|  |}t|}| |}|d urN|| t|d d d d d d f  }| |}t|}| |}| |}| j| jkr| j	r| 
|}n
| |}|| S r'   )rV   r+   rW   rY   rZ   r\   r]   r2   rR   rU   rS   r^   )r@   r*   tembhr$   r$   r%   rG      s    

&



zResnetBlock.forwardrH   r$   r$   rA   r%   rQ   [   s
   $rQ   c                       s    e Zd ZdZ fddZ  ZS )LinAttnBlockzto match AttnBlock usagec                    s   t  j|d|d d S )Nr   )r   headsdim_head)r:   r;   r@   r2   rA   r$   r%   r;      s    zLinAttnBlock.__init__)rI   rJ   rK   __doc__r;   rL   r$   r$   rA   r%   ra      s   ra   c                       s8   e Zd Z fddZejejdddZdd Z  ZS )	AttnBlockc                    s~   t    || _t|| _tjj||dddd| _tjj||dddd| _	tjj||dddd| _
tjj||dddd| _d S Nr   r   r5   )r:   r;   r2   r3   normr   r   r=   qkvproj_outrd   rA   r$   r%   r;      s    





zAttnBlock.__init__h_returnc           	      C   sv   |  |}| |}| |}| |}|j\}}}}tdd |||f\}}}tjj	|||}t
|d||||dS )Nc                 S   s   t | d S )Nzb c h w -> b 1 (h w) c)r   
contiguousr)   r$   r$   r%   <lambda>       z%AttnBlock.attention.<locals>.<lambda>zb 1 (h w) c -> b c h w)r`   wcb)rh   ri   rj   rk   r   mapr   r   r   scaled_dot_product_attentionr   )	r@   rn   ri   rj   rk   ru   rt   r`   rs   r$   r$   r%   	attention   s    




zAttnBlock.attentionc                 K   s    |}|  |}| |}|| S r'   rx   rl   r@   r*   kwargsrn   r$   r$   r%   rG      s    

zAttnBlock.forward)	rI   rJ   rK   r;   r   Tensorrx   rG   rL   r$   r$   rA   r%   rf      s   rf   c                       s<   e Zd ZdZ fddZejejdddZdd Z  Z	S )	MemoryEfficientAttnBlockz
    Uses xformers efficient implementation,
    see https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223
    Note: this is a single-head self-attention operation
    c                    s   t    || _t|| _tjj||dddd| _tjj||dddd| _	tjj||dddd| _
tjj||dddd| _d | _d S rg   )r:   r;   r2   r3   rh   r   r   r=   ri   rj   rk   rl   attention_oprd   rA   r$   r%   r;      s     





z!MemoryEfficientAttnBlock.__init__rm   c                    s   |  |}| |}| |}| |}|j\ }}tdd |||f\}}}t fdd|||f\}}}tjj|||d | j	d}|
d d|jd dddd |jd }t|d	 ||d
S )Nc                 S   s
   t | dS )Nb c h w -> b (h w) cr   r)   r$   r$   r%   rq      rr   z4MemoryEfficientAttnBlock.attention.<locals>.<lambda>c                    s@   |  d | jd ddddd d | jd  S )Nr   r   r   r   )	unsqueezereshaper   permuterp   )tBCr$   r%   rq      s   
)	attn_biasopr   r   r   r   b (h w) c -> b c h w)ru   r`   rs   rt   )rh   ri   rj   rk   r   rv   xformersopsZmemory_efficient_attentionr~   r   r   r   r   )r@   rn   ri   rj   rk   HWoutr$   r   r%   rx      s,    





z"MemoryEfficientAttnBlock.attentionc                 K   s    |}|  |}| |}|| S r'   ry   rz   r$   r$   r%   rG     s    

z MemoryEfficientAttnBlock.forward)
rI   rJ   rK   re   r;   r   r|   rx   rG   rL   r$   r$   rA   r%   r}      s   r}   c                       s   e Zd Zd fdd	Z  ZS )$MemoryEfficientCrossAttentionWrapperNc           
         sD   |j \}}}}t|d}t j|||d}	t|	d|||d}	||	 S )Nr   )contextmaskr   )r`   rs   rt   )r   r   r:   rG   )
r@   r*   r   r   unused_kwargsru   rt   r`   rs   r   rA   r$   r%   rG   
  s
    
z,MemoryEfficientCrossAttentionWrapper.forward)NN)rI   rJ   rK   rG   rL   r$   r$   rA   r%   r   	  s   r   vanillac                 C   s   |dv sJ d| dt tjt dk rP|dkrPtsLJ dtj dd}td	| d
|  d |dkr|d u szJ t| S |dkrtd|  d t| S tdkr| |d< t	f i |S |dkrt
| S t| S d S )N)r   vanilla-xformersmemory-efficient-cross-attnlinearnonez
attn_type z unknownz2.0.0r   z'We do not support vanilla attention in za anymore, as it is too expensive. Please install xformers via e.g. 'pip install xformers==0.0.16'r   zmaking attention of type 'z' with z in_channelsr   z'building MemoryEfficientAttnBlock with z in_channels...r   Z	query_dim)r   parser   __version__XFORMERS_IS_AVAILABLEprintrf   r}   typer   r   Identityra   )r2   	attn_typeattn_kwargsr$   r$   r%   	make_attn  s0    


r   c                       s>   e Zd Zddddddd fdd
Zdd
dZdd Z  ZS )Modelr   r                 TFr   )ch_multr\   resamp_with_convuse_timestepuse_linear_attnr   c             
      s  t    |rd}|| _| jd | _t|| _|| _|	| _|| _|
| _	| j	rt
 | _t
tj
| j| jtj
| j| jg| j_tj
j|| jdddd| _|	}dt| }t
 | _t| jD ]}t
 }t
 }|||  }|||  }t| jD ]:}|t||| j|d |}||v r|t||d qt
 }||_||_|| jd krlt|||_|d	 }| j| qt
 | _t||| j|d| j_t||d| j_t||| j|d| j_ t
 | _!t"t| jD ]}t
 }t
 }|||  }|||  }t| jd D ]Z}|| jkr.|||  }|t|| || j|d |}||v r|t||d qt
 }||_||_|d
krt#|||_$|d	 }| j!%d
| qt&|| _'tj
j||dddd| _(d S )Nr   r   r   r   r5   r   r2   rR   rT   r\   r   r   r   ))r:   r;   chtemb_chr   num_resolutionsnum_res_blocks
resolutionr2   r   r   Moduler_   
ModuleListr   rX   denser=   conv_intupledownrangeappendrQ   r   blockattnrM   
downsamplemidblock_1attn_1block_2upreversedr4   upsampleinsertr3   norm_outconv_out)r@   r   out_chr   r   attn_resolutionsr\   r   r2   r   r   r   r   curr_res
in_ch_multi_levelr   r   block_in	block_outi_blockr   Zskip_inr   rA   r$   r%   r;   4  s    











zModel.__init__Nc           	      C   s  |d urt j||fdd}| jrb|d us,J t|| j}| jjd |}t|}| jjd |}nd }| |g}t	| j
D ]}t	| jD ]P}| j| j| |d |}t| j| jdkr| j| j| |}|| q|| j
d kr||| j| |d  q||d }| j||}| j|}| j||}tt	| j
D ]}t	| jd D ]X}| j| j| t j|| gdd|}t| j| jdkrV| j| j| |}qV|dkrD| j| |}qD| |}t|}| |}|S )Nr   r   r   )r   r   r   r&   r   r_   r   r+   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   popr   r   r   )	r@   r*   r   r   r_   hsr   r   r`   r$   r$   r%   rG     sF    


zModel.forwardc                 C   s   | j jS r'   r   weightr@   r$   r$   r%   get_last_layer  s    zModel.get_last_layer)NN)rI   rJ   rK   r;   rG   r   rL   r$   r$   rA   r%   r   3  s   y
1r   c                       s4   e Zd Zddddddd fdd
Zd	d
 Z  ZS )Encoderr   r   TFr   )r   r\   r   double_zr   r   c             
      s  t    |rd}|| _d| _t|| _|| _|	| _|| _t	j
j|| jdddd| _|	}dt| }|| _t
 | _t| jD ]}t
 }t
 }|||  }|||  }t| jD ]:}|t||| j|d |}||v r|t||d qt
 }||_||_|| jd kr(t|||_|d	 }| j| q~t
 | _t||| j|d| j_t||d| j_t||| j|d| j_t|| _t	j
j||rd	|
 n|
dddd| _ d S )
Nr   r   r   r   r5   r   r   r   r   )!r:   r;   r   r   r   r   r   r   r2   r   r   r=   r   r   r   r   r   r   r   rQ   r   r   r   r   rM   r   r   r   r   r   r3   r   r   )r@   r   r   r   r   r   r\   r   r2   r   
z_channelsr   r   r   ignore_kwargsr   r   r   r   r   r   r   r   r   rA   r$   r%   r;     s|    






zEncoder.__init__c                 C   s   d }|  |g}t| jD ]}t| jD ]P}| j| j| |d |}t| j| jdkrn| j| j| |}|| q(|| jd kr|| j| 	|d  q|d }| j
||}| j
|}| j
||}| |}t|}| |}|S )Nr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r+   r   )r@   r*   r_   r   r   r   r`   r$   r$   r%   rG   ;  s$    

zEncoder.forwardrH   r$   r$   rA   r%   r     s   Xr   c                       sh   e Zd Zdddddddd fdd
Zed	d
dZed	ddZed	ddZdd Zdd Z	  Z
S )Decoderr   r   TFr   )r   r\   r   give_pre_endtanh_outr   r   c             
      s   t    |rd}|| _d| _t|| _|| _|	| _|| _|| _	|| _
dt| }||| jd   }|	d| jd   }d|
||f| _td| jt| j |  }|  }|  }tjj|
|dddd| _t | _|||| j|d	| j_|||d
| j_|||| j|d	| j_t | _tt | jD ]}t }t }|||  }t | jd D ]>}|!|||| j|d	 |}||v rZ|!|||d
 qZt }||_"||_#|dkrt$|||_%|d }| j&d| q,t'|| _(|||dddd| _)d S )Nr   r   r   r   r   z+Working with z of shape {} = {} dimensions.r   r5   r   r   )*r:   r;   r   r   r   r   r   r   r2   r   r   r   Zz_shaper   formatnpprod
_make_attn_make_resblock
_make_convr   r   r=   r   r   r   r   r   r   r   r   r   r   r   r   r   r4   r   r   r3   r   r   )r@   r   r   r   r   r   r\   r   r2   r   r   r   r   r   r   Zignorekwargsr   r   r   Zmake_attn_clsZmake_resblock_clsZmake_conv_clsr   r   r   r   r   r   rA   r$   r%   r;   X  s    










zDecoder.__init__)ro   c                 C   s   t S r'   )r   r   r$   r$   r%   r     s    zDecoder._make_attnc                 C   s   t S r'   )rQ   r   r$   r$   r%   r     s    zDecoder._make_resblockc                 C   s   t jjS r'   )r   r   r=   r   r$   r$   r%   r     s    zDecoder._make_convc                 K   s   | j jS r'   r   )r@   r{   r$   r$   r%   r     s    zDecoder.get_last_layerc                 K   s(  |j | _d }| |}| jj||fi |}| jj|fi |}| jj||fi |}tt| j	D ]~}t| j
d D ]R}| j| j| ||fi |}t| j| jdkrv| j| j| |fi |}qv|dkrd| j| |}qd| jr|S | |}t|}| j|fi |}| jr$t|}|S )Nr   r   )r   Zlast_z_shaper   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r+   r   r   r   tanh)r@   zr{   r_   r`   r   r   r$   r$   r%   rG     s*    


zDecoder.forward)rI   rJ   rK   r;   r   r   r   r   r   rG   rL   r$   r$   rA   r%   r   W  s   br   )r,   )r   N)$r   typingr   r   r   numpyr   r   torch.nnr   einopsr   	packagingr   r   Zxformers.opsr   r   Zmodules.attentionr	   r
   r&   r+   r3   r   r4   rM   rQ   ra   rf   r}   r   r   r   r   r   r$   r$   r$   r%   <module>   s:   
<+@	
! 0u