a
    
dP                     @   s  d dl Z d dlZd dlmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlmZ zd dlZd dlZdZW n   dZed Y n0 dd	 Zd
d Zd2ddZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd deZd3ddZG dd dejZG d d! d!ejZG d"d# d#ejZG d$d% d%ejZG d&d' d'ejZ G d(d) d)ejZ!G d*d+ d+ejZ"G d,d- d-ejZ#G d.d/ d/ejZ$G d0d1 d1ejZ%dS )4    N	rearrange)OptionalAny)MemoryEfficientCrossAttentionTFz,No module 'xformers'. Proceeding without it.c                 C   s   t | jdksJ |d }td|d  }ttj|tjd|  }|j| j	d}| 
 dddf |dddf  }tjt|t|gdd}|d dkrtjj|d}|S )	a  
    This matches the implementation in Denoising Diffusion Probabilistic Models:
    From Fairseq.
    Build sinusoidal embeddings.
    This matches the implementation in tensor2tensor, but differs slightly
    from the description in Section 3.5 of "Attention Is All You Need".
          i'  )dtype)deviceNdim)r   r   r   r   )lenshapemathlogtorchexparangefloat32tor
   floatcatsincosnn
functionalpad)	timestepsembedding_dimhalf_dimemb r!   u/var/www/html/stable-diffusion-webui/repositories/stable-diffusion-stability-ai/ldm/modules/diffusionmodules/model.pyget_timestep_embedding   s    $r#   c                 C   s   | t |  S N)r   sigmoidxr!   r!   r"   nonlinearity)   s    r(       c                 C   s   t jj|| dddS )Ngư>T)
num_groupsnum_channelsepsaffine)r   r   	GroupNorm)in_channelsr*   r!   r!   r"   	Normalize.   s    r0   c                       s$   e Zd Z fddZdd Z  ZS )Upsamplec                    s2   t    || _| jr.tjj||dddd| _d S )N   r   kernel_sizestridepaddingsuper__init__	with_convr   r   Conv2dconvselfr/   r:   	__class__r!   r"   r9   3   s    
zUpsample.__init__c                 C   s(   t jjj|ddd}| jr$| |}|S )Ng       @nearest)scale_factormode)r   r   r   interpolater:   r<   r>   r'   r!   r!   r"   forward=   s    
zUpsample.forward__name__
__module____qualname__r9   rF   __classcell__r!   r!   r?   r"   r1   2   s   
r1   c                       s$   e Zd Z fddZdd Z  ZS )
Downsamplec                    s2   t    || _| jr.tjj||dddd| _d S )Nr2   r   r   r3   r7   r=   r?   r!   r"   r9   E   s    
zDownsample.__init__c                 C   sD   | j r,d}tjjj||ddd}| |}ntjjj|ddd}|S )N)r   r   r   r   constantr   )rC   valuer   )r4   r5   )r:   r   r   r   r   r<   
avg_pool2d)r>   r'   r   r!   r!   r"   rF   P   s    zDownsample.forwardrG   r!   r!   r?   r"   rL   D   s   rL   c                       s.   e Zd Zdddd fdd
Zdd Z  ZS )	ResnetBlockNFi   )out_channelsconv_shortcuttemb_channelsc                   s   t    || _|d u r|n|}|| _|| _t|| _tjj	||dddd| _
|dkrftj||| _t|| _tj|| _tjj	||dddd| _| j| jkr| jrtjj	||dddd| _ntjj	||dddd| _d S )Nr2   r   r3   r   )r8   r9   r/   rQ   use_conv_shortcutr0   norm1r   r   r;   conv1Linear	temb_projnorm2Dropoutdropoutconv2rR   nin_shortcut)r>   r/   rQ   rR   r[   rS   r?   r!   r"   r9   [   sL    



zResnetBlock.__init__c                 C   s   |}|  |}t|}| |}|d urN|| t|d d d d d d f  }| |}t|}| |}| |}| j| jkr| j	r| 
|}n
| |}|| S r$   )rU   r(   rV   rX   rY   r[   r\   r/   rQ   rT   rR   r]   )r>   r'   tembhr!   r!   r"   rF      s    

&



zResnetBlock.forwardrG   r!   r!   r?   r"   rP   Z   s   &rP   c                       s$   e Zd Z fddZdd Z  ZS )	AttnBlockc                    s~   t    || _t|| _tjj||dddd| _tjj||dddd| _	tjj||dddd| _
tjj||dddd| _d S Nr   r   r3   )r8   r9   r/   r0   normr   r   r;   qkvproj_outr>   r/   r?   r!   r"   r9      s6    

zAttnBlock.__init__c                 C   s   |}|  |}| |}| |}| |}|j\}}}}	|||||	 }|ddd}|||||	 }t||}
|
t	|d  }
tj
jj|
dd}
|||||	 }|
ddd}
t||
}|||||	}| |}|| S )Nr   r   r   g      r   )rb   rc   rd   re   r   reshapepermuter   bmmintr   r   softmaxrf   )r>   r'   h_rc   rd   re   bcr_   ww_r!   r!   r"   rF      s$    




zAttnBlock.forwardrG   r!   r!   r?   r"   r`      s   r`   c                       s(   e Zd ZdZ fddZdd Z  ZS )MemoryEfficientAttnBlockz
        Uses xformers efficient implementation,
        see https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223
        Note: this is a single-head self-attention operation
    c                    s   t    || _t|| _tjj||dddd| _tjj||dddd| _	tjj||dddd| _
tjj||dddd| _d | _d S ra   )r8   r9   r/   r0   rb   r   r   r;   rc   rd   re   rf   attention_oprg   r?   r!   r"   r9      s8    

z!MemoryEfficientAttnBlock.__init__c           	         s   |}|  |}| |}| |}| |}|j\ }}tdd |||f\}}}t fdd|||f\}}}tjj|||d | j	d}|
d d|jd dddd |jd }t|d	 ||d
}| |}|| S )Nc                 S   s
   t | dS )Nb c h w -> b (h w) cr   r&   r!   r!   r"   <lambda>       z2MemoryEfficientAttnBlock.forward.<locals>.<lambda>c                    s@   |  d | jd ddddd d | jd  S )Nr2   r   r   r   )	unsqueezerh   r   ri   
contiguous)tBCr!   r"   ru      s   
)	attn_biasopr   r   r   r2   b (h w) c -> b c h w)rn   r_   rp   ro   )rb   rc   rd   re   r   mapxformersopsmemory_efficient_attentionrs   rw   rh   ri   r   rf   )	r>   r'   rm   rc   rd   re   HWoutr!   rz   r"   rF      s.    






z MemoryEfficientAttnBlock.forward)rH   rI   rJ   __doc__r9   rF   rK   r!   r!   r?   r"   rr      s   rr   c                       s   e Zd Zd fdd	Z  ZS )$MemoryEfficientCrossAttentionWrapperNc           	         sD   |j \}}}}t|d}t j|||d}t|d|||d}|| S )Nrt   )contextmaskr   )r_   rp   ro   )r   r   r8   rF   )	r>   r'   r   r   rn   ro   r_   rp   r   r?   r!   r"   rF     s
    
z,MemoryEfficientCrossAttentionWrapper.forward)NN)rH   rI   rJ   rF   rK   r!   r!   r?   r"   r     s   r   vanillac                 C   s   |dv sJ d| dt r(|dkr(d}td| d|  d |dkrZ|d u sRJ t| S |dkrztd	|  d
 t| S tdkr| |d< tf i |S |dkrt| S t d S )N)r   vanilla-xformersmemory-efficient-cross-attnlinearnonez
attn_type z unknownr   r   zmaking attention of type 'z' with z in_channelsz'building MemoryEfficientAttnBlock with z in_channels...r   	query_dimr   )	XFORMERS_IS_AVAILBLEprintr`   rr   typer   r   IdentityNotImplementedError)r/   	attn_typeattn_kwargsr!   r!   r"   	make_attn  s     
r   c                       s>   e Zd Zddddddd fdd
Zdd
dZdd Z  ZS )Modelr   r                 TFr   )ch_multr[   resamp_with_convuse_timestepuse_linear_attnr   c             
      s  t    |rd}|| _| jd | _t|| _|| _|	| _|| _|
| _	| j	rt
 | _t
tj
| j| jtj
| j| jg| j_tj
j|| jdddd| _|	}dt| }t
 | _t| jD ]}t
 }t
 }|||  }|||  }t| jD ]:}|t||| j|d |}||v r|t||d qt
 }||_||_|| jd krlt|||_|d	 }| j| qt
 | _t||| j|d| j_t||d| j_t||| j|d| j_ t
 | _!t"t| jD ]}t
 }t
 }|||  }|||  }t| jd D ]Z}|| jkr.|||  }|t|| || j|d |}||v r|t||d qt
 }||_||_|d
krt#|||_$|d	 }| j!%d
| qt&|| _'tj
j||dddd| _(d S )Nr   r   r2   r   r3   r   r/   rQ   rS   r[   r   r   r   ))r8   r9   chtemb_chr   num_resolutionsnum_res_blocks
resolutionr/   r   r   Moduler^   
ModuleListr   rW   denser;   conv_intupledownrangeappendrP   r   blockattnrL   
downsamplemidblock_1attn_1block_2upreversedr1   upsampleinsertr0   norm_outconv_out)r>   r   out_chr   r   attn_resolutionsr[   r   r/   r   r   r   r   curr_res
in_ch_multi_levelr   r   block_in	block_outi_blockr   skip_inr   r?   r!   r"   r9   -  s    













zModel.__init__Nc           	      C   s  |d urt j||fdd}| jrb|d us,J t|| j}| jjd |}t|}| jjd |}nd }| |g}t	| j
D ]}t	| jD ]P}| j| j| |d |}t| j| jdkr| j| j| |}|| q|| j
d kr||| j| |d  q||d }| j||}| j|}| j||}tt	| j
D ]}t	| jd D ]X}| j| j| t j|| gdd|}t| j| jdkrV| j| j| |}qV|dkrD| j| |}qD| |}t|}| |}|S )Nr   r   r   )r   r   r   r#   r   r^   r   r(   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   popr   r   r   )	r>   r'   ry   r   r^   hsr   r   r_   r!   r!   r"   rF     sF    


zModel.forwardc                 C   s   | j jS r$   )r   weight)r>   r!   r!   r"   get_last_layer  s    zModel.get_last_layer)NN)rH   rI   rJ   r9   rF   r   rK   r!   r!   r?   r"   r   ,  s   c
0r   c                       s4   e Zd Zddddddd fdd
Zd	d
 Z  ZS )Encoderr   r   TFr   )r   r[   r   double_zr   r   c             
      s  t    |rd}|| _d| _t|| _|| _|	| _|| _t	j
j|| jdddd| _|	}dt| }|| _t
 | _t| jD ]}t
 }t
 }|||  }|||  }t| jD ]:}|t||| j|d |}||v r|t||d qt
 }||_||_|| jd kr(t|||_|d	 }| j| q~t
 | _t||| j|d| j_t||d| j_t||| j|d| j_t|| _t	j
j||rd	|
 n|
dddd| _ d S )
Nr   r   r2   r   r3   r   r   r   r   )!r8   r9   r   r   r   r   r   r   r/   r   r   r;   r   r   r   r   r   r   r   rP   r   r   r   r   rL   r   r   r   r   r   r0   r   r   )r>   r   r   r   r   r   r[   r   r/   r   
z_channelsr   r   r   ignore_kwargsr   r   r   r   r   r   r   r   r   r?   r!   r"   r9     st    






zEncoder.__init__c                 C   s   d }|  |g}t| jD ]}t| jD ]P}| j| j| |d |}t| j| jdkrn| j| j| |}|| q(|| jd kr|| j| 	|d  q|d }| j
||}| j
|}| j
||}| |}t|}| |}|S )Nr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r(   r   )r>   r'   r^   r   r   r   r_   r!   r!   r"   rF     s$    

zEncoder.forwardrG   r!   r!   r?   r"   r     s
   Ar   c                       s6   e Zd Zdddddddd fdd
Zd	d
 Z  ZS )Decoderr   r   TFr   )r   r[   r   give_pre_endtanh_outr   r   c             
      s  t    |rd}|| _d| _t|| _|| _|	| _|| _|| _	|| _
dt| }||| jd   }|	d| jd   }d|
||f| _td| jt| j tjj|
|dddd| _t | _t||| j|d	| j_t||d
| j_t||| j|d	| j_t | _tt| jD ]}t }t }|||  }t| jd D ]>}| t||| j|d	 |}||v rB| t||d
 qBt }||_!||_"|dkrt#|||_$|d }| j%d| qt&|| _'tjj||dddd| _(d S )Nr   r   r   r   r   z+Working with z of shape {} = {} dimensions.r2   r3   r   r   ))r8   r9   r   r   r   r   r   r   r/   r   r   r   z_shaper   formatnpprodr   r   r;   r   r   r   rP   r   r   r   r   r   r   r   r   r   r   r   r1   r   r   r0   r   r   )r>   r   r   r   r   r   r[   r   r/   r   r   r   r   r   r   ignorekwargsr   r   r   r   r   r   r   r   r   r?   r!   r"   r9   #  s~    








zDecoder.__init__c                 C   s   |j | _d }| |}| j||}| j|}| j||}tt| j	D ]n}t| j
d D ]B}| j| j| ||}t| j| jdkr^| j| j| |}q^|dkrL| j| |}qL| jr|S | |}t|}| |}| jrt|}|S )Nr   r   )r   last_z_shaper   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r(   r   r   r   tanh)r>   zr^   r_   r   r   r!   r!   r"   rF   k  s*    



zDecoder.forwardrG   r!   r!   r?   r"   r   "  s   Hr   c                       s$   e Zd Z fddZdd Z  ZS )SimpleDecoderc                    s   t    tt||dt|d| dddtd| d| dddtd| d| dddtd| |dt|ddg| _t|| _	t
jj||d	ddd
| _d S )Nr   r   r   r   r   r   T)r:   r2   r3   )r8   r9   r   r   r;   rP   r1   modelr0   r   r   r   )r>   r/   rQ   argskwargsr?   r!   r"   r9     s0    


zSimpleDecoder.__init__c                 C   sP   t | jD ]$\}}|dv r&||d }q
||}q
| |}t|}| |}|S )N)r   r   r2   )	enumerater   r   r(   r   )r>   r'   ilayerr_   r!   r!   r"   rF     s    


zSimpleDecoder.forwardrG   r!   r!   r?   r"   r     s   r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )UpsampleDecoderr   r   r   c              
      s   t    d| _t|| _|| _|}|d| jd   }	t | _t | _	t
| jD ]~}
g }|||
  }t
| jd D ] }|t||| j|d |}qv| jt| |
| jd krT| j	t|d |	d }	qTt|| _tjj||dddd| _d S )Nr   r   r   r   Tr2   r3   )r8   r9   r   r   r   r   r   r   
res_blocksupsample_blocksr   r   rP   r1   r0   r   r   r;   r   )r>   r/   rQ   r   r   r   r   r[   r   r   r   Z	res_blockr   r   r?   r!   r"   r9     s:    





zUpsampleDecoder.__init__c                 C   s   |}t t| jD ]L\}}t| jd D ]}| j| | |d }q(|| jd kr| j| |}q| |}t|}| |}|S )Nr   )	r   r   r   r   r   r   r   r(   r   )r>   r'   r_   rd   r   r   r!   r!   r"   rF     s    

zUpsampleDecoder.forward)r   r   rG   r!   r!   r?   r"   r     s    !r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )LatentRescalerr   c                    s   t    || _tj| dddd| _t fddt|D | _t	 | _
t fddt|D | _tj |dd| _d S )Nr2   r   r3   c                    s   g | ]}t   d ddqS r   r   r   rP   .0_mid_channelsr!   r"   
<listcomp>  s   z+LatentRescaler.__init__.<locals>.<listcomp>c                    s   g | ]}t   d ddqS r   r   r   r   r!   r"   r     s   )r4   )r8   r9   factorr   r;   r   r   r   
res_block1r`   r   
res_block2r   )r>   r   r/   r   rQ   depthr?   r   r"   r9     s&    



zLatentRescaler.__init__c                 C   s   |  |}| jD ]}||d }qtjjj|tt|jd | j	 tt|jd | j	 fd}| 
|}| jD ]}||d }qn| |}|S )Nr   r2   )size)r   r   r   r   r   rD   rk   roundr   r   r   r   r   )r>   r'   r   r!   r!   r"   rF     s    

>


zLatentRescaler.forward)r   rG   r!   r!   r?   r"   r     s   r   c                       s&   e Zd Zd
 fdd	Zdd	 Z  ZS )MergedRescaleEncoderr   Tr         ?r   c                    sN   t    ||	d  }t||||	|d||||d d| _t|
||||d| _d S )Nr   F)r/   r   r   r   r   r   r   r   r[   r   r   r   r/   r   rQ   r   )r8   r9   r   encoderr   rescaler)r>   r/   r   r   r   r   r   r[   r   r   rescale_factorrescale_module_depthZintermediate_chnr?   r!   r"   r9   	  s    

zMergedRescaleEncoder.__init__c                 C   s   |  |}| |}|S r$   )r   r   rE   r!   r!   r"   rF     s    

zMergedRescaleEncoder.forward)r   Tr   r   r   rG   r!   r!   r?   r"   r     s     r   c                       s&   e Zd Zd
 fdd	Zdd	 Z  ZS )MergedRescaleDecoderr   r   Tr   r   c                    sL   t    ||d  }t|||||	d ||||d
| _t|
||||d| _d S )Nr   )
r   r   r   r[   r   r/   r   r   r   r   r   )r8   r9   r   decoderr   r   )r>   r   r   r   r   r   r   r   r[   r   r   r   Ztmp_chnr?   r!   r"   r9     s    

zMergedRescaleDecoder.__init__c                 C   s   |  |}| |}|S r$   r   r   rE   r!   r!   r"   rF   &  s    

zMergedRescaleDecoder.forward)r   r   Tr   r   rG   r!   r!   r?   r"   r     s     
r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )	Upsamplerr   c                    s   t    ||ksJ tt|| d }d||  }td| jj d| d| d|  t||d| |d| _	t
|||dg d | fd	d
t|D d| _d S )Nr   r   z	Building z with in_size: z --> out_size z and factor r   )r   r/   r   rQ   c                    s   g | ]} qS r!   r!   r   r   r!   r"   r   7  rv   z&Upsampler.__init__.<locals>.<listcomp>)r   r   r   r   r   r/   r   r   )r8   r9   rk   r   log2r   r@   rH   r   r   r   r   r   )r>   in_sizeout_sizer/   rQ   r   
num_blocksZ	factor_upr?   r  r"   r9   -  s    
$
zUpsampler.__init__c                 C   s   |  |}| |}|S r$   r   rE   r!   r!   r"   rF   9  s    

zUpsampler.forward)r   rG   r!   r!   r?   r"   r   ,  s   r   c                       s(   e Zd Zd	 fdd	Zd
ddZ  ZS )ResizeNFbilinearc                    sd   t    || _|| _| jr`td| jj d| d t |d usHJ tj	j
||dddd| _d S )NzNote: z5 uses learned downsampling and will ignore the fixed z moder   r   r   r3   )r8   r9   r:   rC   r   r@   Z_Resize__namer   r   r   r;   r<   )r>   r/   ZlearnedrC   r?   r!   r"   r9   @  s    
zResize.__init__r   c                 C   s(   |dkr|S t jjj|| jd|d}|S )Nr   F)rC   align_cornersrB   )r   r   r   rD   rC   )r>   r'   rB   r!   r!   r"   rF   O  s    zResize.forward)NFr  )r   rG   r!   r!   r?   r"   r  ?  s   r  )r)   )r   N)&r   r   torch.nnr   numpyr   einopsr   typingr   r   ldm.modules.attentionr   r   Zxformers.opsr   r   r#   r(   r0   r   r1   rL   rP   r`   rr   r   r   r   r   r   r   r   r   r   r   r   r  r!   r!   r!   r"   <module>   sD   
>5B	
 ^m$0%