a
    Aþd—<  ã                   @   s"  d Z ddlZddlmZ ddlm  mZ ddlmZ ddl	m
Z
 dd„ Zejjdd„ ƒZG d	d
„ d
ejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZe
 ¡ G dd„ dejƒƒZe
 ¡ G dd„ dejƒƒZdS )z¦
VQGAN code, adapted from the original created by the Unleashing Transformers authors:
https://github.com/samb-t/unleashing-transformers/blob/master/models/vqgan.py

é    N)Úget_root_logger)ÚARCH_REGISTRYc                 C   s   t jjd| dddS )Né    gíµ ÷Æ°>T)Ú
num_groupsÚnum_channelsÚepsÚaffine)ÚtorchÚnnÚ	GroupNorm)Úin_channels© r   úE/var/www/html/stable-diffusion-webui/modules/codeformer/vqgan_arch.pyÚ	normalize   s    r   c                 C   s   | t  | ¡ S ©N)r	   Úsigmoid)Úxr   r   r   Úswish   s    r   c                       s,   e Zd Z‡ fdd„Zdd„ Zdd„ Z‡  ZS )ÚVectorQuantizerc                    sT   t t| ƒ ¡  || _|| _|| _t | j| j¡| _| jj	j
 d| j d| j ¡ d S )Ng      ð¿ç      ð?)Úsuperr   Ú__init__Úcodebook_sizeÚemb_dimÚbetar
   Ú	EmbeddingÚ	embeddingÚweightÚdataÚuniform_)Úselfr   r   r   ©Ú	__class__r   r   r      s    zVectorQuantizer.__init__c              	   C   sh  |  dddd¡ ¡ }| d| j¡}|d jddd| jjd  d¡ dt || jj 	¡ ¡  }t 
|¡}tj|dddd	\}}t | d
 ¡}t |jd | j¡ |¡}| d|d¡ t || jj¡ |j¡}t 
| ¡ | d ¡| jt 
|| ¡  d ¡  }	|||  ¡  }tj
|dd}
t t |
t |
d ¡ ¡ ¡}|  dddd¡ ¡ }||	|||||dœfS )Nr   é   é   é   éÿÿÿÿT)ÚdimÚkeepdimF)r'   Úlargesté
   ©r'   ç»½×Ùß|Û=)Ú
perplexityÚmin_encodingsÚmin_encoding_indicesÚmin_encoding_scoresÚmean_distance)ÚpermuteÚ
contiguousÚviewr   Úsumr   r   r	   ÚmatmulÚtÚmeanÚtopkÚexpÚzerosÚshaper   ÚtoÚscatter_Údetachr   Úlog)r    ÚzZz_flattenedÚdr1   r0   r/   r.   Úz_qÚlossZe_meanr-   r   r   r   Úforward!   s.    "ÿ
2 ûzVectorQuantizer.forwardc                 C   sn   |  dd¡}t |jd | j¡ |¡}| d|d¡ t | ¡ | j	j
¡}|d urj|  |¡ dddd¡ ¡ }|S )Nr&   r%   r   r$   r#   )r4   r	   r;   r<   r   r=   r>   r6   Úfloatr   r   r2   r3   )r    Úindicesr<   r.   rC   r   r   r   Úget_codebook_featI   s    z!VectorQuantizer.get_codebook_feat)Ú__name__Ú
__module__Ú__qualname__r   rE   rH   Ú__classcell__r   r   r!   r   r      s   (r   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )	ÚGumbelQuantizerFçü©ñÒMb@?r   c                    sJ   t ƒ  ¡  || _|| _|| _|| _|| _t ||d¡| _	t 
||¡| _d S )Nr%   )r   r   r   r   Ústraight_throughÚtemperatureÚ	kl_weightr
   ÚConv2dÚprojr   Úembed)r    r   r   Znum_hiddensrO   rQ   Z	temp_initr!   r   r   r   Y   s    
zGumbelQuantizer.__init__c           	      C   s”   | j r| jnd}|  |¡}tj|| jd|d}t d|| jj	¡}tj
|dd}| jtj|t || j d ¡ dd ¡  }|jdd}||d|ifS )NTr%   )Útaur'   Úhardzb n h w, n d -> b d h wr+   r,   r/   )ÚtrainingrO   rS   ÚFÚgumbel_softmaxrP   r	   ÚeinsumrT   r   ÚsoftmaxrQ   r5   r@   r   r8   Úargmax)	r    rA   rV   ÚlogitsÚsoft_one_hotrC   ÚqyÚdiffr/   r   r   r   rE   c   s    
,ÿzGumbelQuantizer.forward)FrN   r   ©rI   rJ   rK   r   rE   rL   r   r   r!   r   rM   X   s   
rM   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú
Downsamplec                    s&   t ƒ  ¡  tjj||dddd| _d S )Nr$   r#   r   ©Úkernel_sizeÚstrideÚpadding)r   r   r	   r
   rR   Úconv©r    r   r!   r   r   r   w   s    
zDownsample.__init__c                 C   s(   d}t jjj||ddd}|  |¡}|S )N)r   r%   r   r%   Úconstantr   )ÚmodeÚvalue)r	   r
   Ú
functionalÚpadrg   )r    r   rm   r   r   r   rE   {   s    
zDownsample.forwardra   r   r   r!   r   rb   v   s   rb   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚUpsamplec                    s$   t ƒ  ¡  tj||dddd| _d S )Nr$   r%   rc   )r   r   r
   rR   rg   rh   r!   r   r   r   ƒ   s    
zUpsample.__init__c                 C   s   t j|ddd}|  |¡}|S )Ng       @Únearest)Úscale_factorrj   )rX   Úinterpolaterg   ©r    r   r   r   r   rE   ‡   s    
zUpsample.forwardra   r   r   r!   r   rn   ‚   s   rn   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚResBlockNc                    sŒ   t t| ƒ ¡  || _|d u r |n|| _t|ƒ| _tj||dddd| _	t|ƒ| _
tj||dddd| _| j| jkrˆtj||dddd| _d S )Nr$   r%   rc   r   )r   rs   r   r   Úout_channelsr   Únorm1r
   rR   Úconv1Únorm2Úconv2Úconv_out)r    r   rt   r!   r   r   r      s    

zResBlock.__init__c                 C   sZ   |}|   |¡}t|ƒ}|  |¡}|  |¡}t|ƒ}|  |¡}| j| jkrR|  |¡}|| S r   )ru   r   rv   rw   rx   r   rt   ry   )r    Úx_inr   r   r   r   rE   š   s    




zResBlock.forward)Nra   r   r   r!   r   rs   Ž   s   rs   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú	AttnBlockc                    s~   t ƒ  ¡  || _t|ƒ| _tjj||dddd| _tjj||dddd| _	tjj||dddd| _
tjj||dddd| _d S )Nr%   r   rc   )r   r   r   r   Únormr	   r
   rR   ÚqÚkÚvÚproj_outrh   r!   r   r   r   ©   s>    

ûûûûzAttnBlock.__init__c                 C   sä   |}|   |¡}|  |¡}|  |¡}|  |¡}|j\}}}}	| ||||	 ¡}| ddd¡}| ||||	 ¡}t ||¡}
|
t	|ƒd  }
t
j|
dd}
| ||||	 ¡}|
 ddd¡}
t ||
¡}| ||||	¡}|  |¡}|| S )Nr   r#   r%   g      à¿r+   )r|   r}   r~   r   r<   Úreshaper2   r	   ÚbmmÚintrX   r[   r€   )r    r   Úh_r}   r~   r   ÚbÚcÚhÚwÚw_r   r   r   rE   Ë   s$    




zAttnBlock.forwardra   r   r   r!   r   r{   ¨   s   "r{   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚEncoderc              	      sH  t ƒ  ¡  || _t|ƒ| _|| _|| _|| _| j}dt|ƒ }	g }
|
 	t
j||dddd¡ t| jƒD ]z}||	|  }|||  }t| jƒD ].}|
 	t||ƒ¡ |}||v rŒ|
 	t|ƒ¡ qŒ|| jd krf|
 	t|ƒ¡ |d }qf|
 	t||ƒ¡ |
 	t|ƒ¡ |
 	t||ƒ¡ |
 	t|ƒ¡ |
 	t
j||dddd¡ t
 |
¡| _d S )N)r%   r$   r%   rc   r#   )r   r   ÚnfÚlenÚnum_resolutionsÚnum_res_blocksÚ
resolutionÚattn_resolutionsÚtupleÚappendr
   rR   Úrangers   r{   rb   r   Ú
ModuleListÚblocks)r    r   r‹   r   Úch_multrŽ   r   r   Úcurr_resÚ
in_ch_multr•   ÚiÚblock_in_chÚblock_out_chÚ_r!   r   r   r   ç   s6    


zEncoder.__init__c                 C   s   | j D ]}||ƒ}q|S r   ©r•   ©r    r   Úblockr   r   r   rE     s    

zEncoder.forwardra   r   r   r!   r   rŠ   æ   s   'rŠ   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú	Generatorc              	      sj  t ƒ  ¡  || _|| _t| jƒ| _|| _|| _|| _|| _	d| _
| j| jd  }| jd| jd   }g }	|	 tj| j	|dddd¡ |	 t||ƒ¡ |	 t|ƒ¡ |	 t||ƒ¡ tt| jƒƒD ]n}
| j| j|
  }t| jƒD ]0}|	 t||ƒ¡ |}|| jv rÞ|	 t|ƒ¡ qÞ|
dkrÀ|	 t|ƒ¡ |d }qÀ|	 t|ƒ¡ |	 tj|| j
dddd¡ t |	¡| _d S )Nr$   r&   r#   r%   rc   r   )r   r   r‹   r–   rŒ   r   rŽ   r   r   r   rt   r’   r
   rR   rs   r{   Úreversedr“   rn   r   r”   r•   )r    r‹   r   r–   Ú
res_blocksÚimg_sizer   rš   r—   r•   r™   r›   rœ   r!   r   r   r     s:    


zGenerator.__init__c                 C   s   | j D ]}||ƒ}q|S r   r   rž   r   r   r   rE   @  s    

zGenerator.forwardra   r   r   r!   r   r      s   *r    c                	       s&   e Zd Zd‡ fd	d
„	Zdd„ Z‡  ZS )ÚVQAutoEncoderro   r#   Né   é   ç      Ð?Fç:Œ0âŽyE>c                    s~  t ƒ  ¡  tƒ }d| _|| _|| _|| _|| _|| _|| _	|pBdg| _
|| _t| j| j| j| j| j| j	| j
ƒ| _| jdkr–|	| _t| j| j| jƒ| _n8| jdkrÎ|| _|
| _|| _t| j| j| j| j| jƒ| _t| j| j| j| j| j	| j
ƒ| _|d urztj|dd}d|v r<|  tj|ddd ¡ | d|› d	¡ n>d
|v rr|  tj|ddd
 ¡ | d|› d¡ ntdƒ‚d S )Nr$   é   ro   ÚgumbelÚcpu©Úmap_locationÚ
params_emazvqgan is loaded from: z [params_ema]Úparamsz	 [params]úWrong params!)r   r   r   r   r‹   Ún_blocksr   Ú	embed_dimr–   r   r   Zquantizer_typerŠ   Úencoderr   r   ÚquantizeZgumbel_num_hiddensrO   rQ   rM   r    Ú	generatorr	   ÚloadÚload_state_dictÚinfoÚ
ValueError)r    r£   r‹   r–   Z	quantizerr¢   r   r   r   r   Zgumbel_straight_throughZgumbel_kl_weightÚ
model_pathÚloggerÚchkptr!   r   r   r   I  sf    
ù	

ûú	


zVQAutoEncoder.__init__c                 C   s.   |   |¡}|  |¡\}}}|  |¡}|||fS r   )r³   r´   rµ   )r    r   ÚquantZcodebook_lossZquant_statsr   r   r   rE   ‚  s    

zVQAutoEncoder.forward)	ro   r#   Nr¥   r¦   r§   Fr¨   Nra   r   r   r!   r   r¤   G  s     ÿ9r¤   c                       s&   e Zd Zd	‡ fdd„	Zdd„ Z‡  ZS )
ÚVQGANDiscriminatorr$   é@   é   Nc           
   
      sp  t ƒ  ¡  tj||ddddt dd¡g}d}d}td|ƒD ]P}|}td| dƒ}|tj|| || ddddd	t || ¡t dd¡g7 }q<|}td| dƒ}|tj|| || ddddd	t || ¡t dd¡g7 }|tj|| dddddg7 }tj|Ž | _	|d urlt
j|d
d}	d|	v r@|  t
j|d
dd ¡ n,d|	v rd|  t
j|d
dd ¡ ntdƒ‚d S )NrÀ   r#   r%   rc   gš™™™™™É?Té   F)rd   re   rf   Úbiasr«   r¬   Zparams_dr¯   r°   )r   r   r
   rR   Ú	LeakyReLUr“   ÚminÚBatchNorm2dÚ
SequentialÚmainr	   r¶   r·   r¹   )
r    ÚncZndfÚn_layersrº   ÚlayersZndf_multZndf_mult_prevÚnr¼   r!   r   r   r     s<    
 
ý
ýÿ


zVQGANDiscriminator.__init__c                 C   s
   |   |¡S r   )rÇ   rr   r   r   r   rE   ²  s    zVQGANDiscriminator.forward)r$   r¿   rÀ   Nra   r   r   r!   r   r¾   ‹  s   %r¾   )Ú__doc__r	   Útorch.nnr
   Útorch.nn.functionalrl   rX   Úbasicsr.utilsr   Úbasicsr.utils.registryr   r   ÚjitÚscriptr   ÚModuler   rM   rb   rn   rs   r{   rŠ   r    Úregisterr¤   r¾   r   r   r   r   Ú<module>   s(   
@>/2C