a
    Aþd<.  ã                   @   sÄ   d dl Zd dlZd dlZd dlm  mZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlZd dlmZ G d	d
„ d
ejƒZG dd„ deƒZeejj_eejj_dS )é    N)Úcontextmanager)ÚLambdaLR)ÚLitEma)ÚVectorQuantizer2)ÚEncoderÚDecoder)Úinstantiate_from_config)Úversionc                       s¬   e Zd Zd(‡ fdd„	Zed)dd„ƒZd*d	d
„Zdd„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zd+dd„Zdd„ Zdd„ Zdd„ Zd,dd„Zd d!„ Zd"d#„ Zd-d$d%„Zd&d'„ Z‡  ZS ).ÚVQModelNÚimageç      ð?Fc              	      sP  t ƒ  ¡  || _|| _|| _tf i |¤Ž| _tf i |¤Ž| _t	|ƒ| _
t||d||d| _tj |d |d¡| _tj ||d d¡| _|d ur¶t|ƒtksžJ ‚|  dt d|dd¡¡ |	d urÄ|	| _|
| _| jd urìt| jj› d|
› dƒ || _| jr"t| ƒ| _td	tt| j  ¡ ƒƒ› dƒ |d ur@| j!||p:g d
 || _"|| _#d S )Ng      Ð?)ÚbetaÚremapÚsane_index_shapeÚ
z_channelsé   Úcolorizeé   z$: Using per-batch resizing in range Ú.zKeeping EMAs of )Úignore_keys)$ÚsuperÚ__init__Ú	embed_dimÚn_embedÚ	image_keyr   Úencoderr   Údecoderr   ÚlossÚVectorQuantizerÚquantizeÚtorchÚnnÚConv2dÚ
quant_convÚpost_quant_convÚtypeÚintÚregister_bufferÚrandnÚmonitorÚbatch_resize_rangeÚprintÚ	__class__Ú__name__Úuse_emar   Ú	model_emaÚlenÚlistÚbuffersÚinit_from_ckptÚscheduler_configÚlr_g_factor)ÚselfÚddconfigÚ
lossconfigr   r   Ú	ckpt_pathr   r   Úcolorize_nlabelsr)   r*   r4   r5   r   r   r.   ©r,   © úU/var/www/html/stable-diffusion-webui/extensions-builtin/LDSR/sd_hijack_autoencoder.pyr      s:    

þ


zVQModel.__init__c              
   c   s¢   | j r8| j |  ¡ ¡ | j | ¡ |d ur8t|› dƒ z6d V  W | j rž| j |  ¡ ¡ |d uržt|› dƒ n.| j rœ| j |  ¡ ¡ |d urœt|› dƒ 0 d S )Nz: Switched to EMA weightsz: Restored training weights)r.   r/   ÚstoreÚ
parametersÚcopy_tor+   Úrestore)r6   Úcontextr<   r<   r=   Ú	ema_scopeE   s    ýzVQModel.ema_scopec           	      C   s²   t j|ddd }t| ¡ ƒ}|D ]0}|p,g D ]"}| |¡r.td |¡ƒ ||= q.q"| j|dd\}}td|› dt|ƒ› d	t|ƒ› d
ƒ |rœtd|› ƒ |r®td|› ƒ d S )NÚcpu)Úmap_locationÚ
state_dictz Deleting key {} from state_dict.F)ÚstrictzRestored from z with z missing and z unexpected keyszMissing Keys: zUnexpected Keys: )	r    Úloadr1   ÚkeysÚ
startswithr+   ÚformatÚload_state_dictr0   )	r6   Úpathr   ÚsdrI   ÚkÚikÚmissingÚ
unexpectedr<   r<   r=   r3   T   s    

$zVQModel.init_from_ckptc                 O   s   | j r|  | ¡ d S ©N)r.   r/   )r6   ÚargsÚkwargsr<   r<   r=   Úon_train_batch_endc   s    zVQModel.on_train_batch_endc                 C   s.   |   |¡}|  |¡}|  |¡\}}}|||fS rS   )r   r#   r   )r6   ÚxÚhÚquantÚemb_lossÚinfor<   r<   r=   Úencodeg   s    

zVQModel.encodec                 C   s   |   |¡}|  |¡}|S rS   ©r   r#   ©r6   rW   rX   r<   r<   r=   Úencode_to_prequantm   s    

zVQModel.encode_to_prequantc                 C   s   |   |¡}|  |¡}|S rS   )r$   r   )r6   rY   Údecr<   r<   r=   Údecoder   s    

zVQModel.decodec                 C   s   | j  |¡}|  |¡}|S rS   )r   Z
embed_codera   )r6   Zcode_bZquant_br`   r<   r<   r=   Údecode_codew   s    
zVQModel.decode_codec                 C   s6   |   |¡\}}\}}}|  |¡}|r.|||fS ||fS rS   )r\   ra   )r6   ÚinputÚreturn_pred_indicesrY   ÚdiffÚ_Úindr`   r<   r<   r=   Úforward|   s
    

zVQModel.forwardc                 C   s®   || }t |jƒdkr|d }| dddd¡jtjd ¡ }| jd urª| jd }| jd }| jdkrj|}nt	j
 t	 ||d d¡¡}||jd kr¢tj||d	d
}| ¡ }|S )Nr   ).Nr   r   é   )Úmemory_formaté   é   Úbicubic)ÚsizeÚmode)r0   ÚshapeÚpermuteÚtor    Úcontiguous_formatÚfloatr*   Úglobal_stepÚnpÚrandomÚchoiceÚarangeÚFÚinterpolateÚdetach)r6   ÚbatchrO   rW   Z
lower_sizeZ
upper_sizeZ
new_resizer<   r<   r=   Ú	get_inputƒ   s    



zVQModel.get_inputc              
   C   sª   |   || j¡}| |dd\}}}|dkrd| j||||| j|  ¡ d|d\}}	| j|	ddddd |S |dkr¦| j||||| j|  ¡ dd	\}
}| j|ddddd |
S d S )
NT©rd   r   Útrain©Ú
last_layerÚsplitÚpredicted_indicesF)Úprog_barÚloggerÚon_stepÚon_epochr   )r‚   rƒ   )r~   r   r   ru   Úget_last_layerÚlog_dict)r6   r}   Ú	batch_idxÚoptimizer_idxrW   ÚxrecÚqlossrg   ÚaelossÚlog_dict_aeÚdisclossÚlog_dict_discr<   r<   r=   Útraining_step•   s    þ
ÿ
zVQModel.training_stepc                 C   sH   |   ||¡}|  ¡   | j ||dd W d   ƒ n1 s:0    Y  |S )NÚ_ema)Úsuffix)Ú_validation_steprC   )r6   r}   r‹   rŠ   r<   r<   r=   Úvalidation_step«   s    
.zVQModel.validation_stepÚ c              
   C   sþ   |   || j¡}| |dd\}}}| j|||d| j|  ¡ d| |d\}}	| j|||d| j|  ¡ d| |d\}
}|	d|› d }| jd|› d|dddddd	 | jd|› d
|dddddd	 t tj	¡t d¡krä|	d|› d= |  
|	¡ |  
|¡ | j
S )NTr   r   Úvalr   r   z	/rec_lossF)r…   r†   r‡   rˆ   Ú	sync_distz/aelossz1.4.0)r~   r   r   ru   r‰   Úlogr	   ÚparseÚplÚ__version__rŠ   )r6   r}   r‹   r•   rW   r   rŽ   rg   r   r   r‘   r’   Zrec_lossr<   r<   r=   r–   ±   s4    ü
ü

ÿ
ÿ

zVQModel._validation_stepc                 C   sô   | j }| j| j  }td|ƒ td|ƒ tjjt| j ¡ ƒt| j	 ¡ ƒ t| j
 ¡ ƒ t| j ¡ ƒ t| j ¡ ƒ |dd}tjj| jj ¡ |dd}| jd urèt| jƒ}tdƒ t||jdddd	œt||jdddd	œg}||g|fS ||gg fS )
NÚlr_dÚlr_g)g      à?gÍÌÌÌÌÌì?)ÚlrÚbetasz Setting up LambdaLR scheduler...)Ú	lr_lambdaÚstepr   )Ú	schedulerÚintervalÚ	frequency)Úlearning_rater5   r+   r    ÚoptimÚAdamr1   r   r?   r   r   r#   r$   r   Údiscriminatorr4   r   r   Úschedule)r6   rŸ   r    Úopt_aeÚopt_discr¥   r<   r<   r=   Úconfigure_optimizersÌ   s@    

ÿþýüûÿ

ýýúzVQModel.configure_optimizersc                 C   s
   | j jjS rS   )r   Úconv_outÚweight)r6   r<   r<   r=   r‰   í   s    zVQModel.get_last_layerc           
      K   sÚ   i }|   || j¡}| | j¡}|r.||d< |S | |ƒ\}}|jd dkrn|jd dksZJ ‚|  |¡}|  |¡}||d< ||d< |rÖ|  ¡ < | |ƒ\}	}|jd dkr°|  |	¡}	|	|d< W d   ƒ n1 sÌ0    Y  |S )NÚinputsr   r   ÚreconstructionsÚreconstructions_ema)r~   r   rr   Údevicerp   Úto_rgbrC   )
r6   r}   Úonly_inputsZplot_emarU   r›   rW   r   rf   Úxrec_emar<   r<   r=   Ú
log_imagesð   s(    



&zVQModel.log_imagesc              	   C   st   | j dksJ ‚t| dƒs<|  dt d|jd dd¡ |¡¡ tj|| j	d}d|| 
¡   | ¡ | 
¡   d }|S )NÚsegmentationr   r   r   )r±   g       @r   )r   Úhasattrr'   r    r(   rp   rr   rz   Úconv2dr   ÚminÚmax)r6   rW   r<   r<   r=   r¶     s    
$$zVQModel.to_rgb)NNr   NNNNr   NFF)N)N)F)r˜   )FF)r-   Ú
__module__Ú__qualname__r   r   rC   r3   rV   r\   r_   ra   rb   rh   r~   r“   r—   r–   r¯   r‰   r¹   r¶   Ú__classcell__r<   r<   r;   r=   r
      s:              ñ0


!
r
   c                       s.   e Zd Z‡ fdd„Zdd„ Zddd„Z‡  ZS )	ÚVQModelInterfacec                    s    t ƒ j|d|i|¤Ž || _d S )Nr   )r   r   r   )r6   r   rT   rU   r;   r<   r=   r     s    zVQModelInterface.__init__c                 C   s   |   |¡}|  |¡}|S rS   r]   r^   r<   r<   r=   r\     s    

zVQModelInterface.encodeFc                 C   s2   |s|   |¡\}}}n|}|  |¡}|  |¡}|S rS   )r   r$   r   )r6   rX   Úforce_not_quantizerY   rZ   r[   r`   r<   r<   r=   ra     s    

zVQModelInterface.decode)F)r-   r¿   rÀ   r   r\   ra   rÁ   r<   r<   r;   r=   rÂ     s   rÂ   ) Únumpyrv   r    Úpytorch_lightningr   Ztorch.nn.functionalr!   Ú
functionalrz   Ú
contextlibr   Ztorch.optim.lr_schedulerr   Úldm.modules.emar   Zvqvae_quantizer   r   Ú"ldm.modules.diffusionmodules.modelr   r   Úldm.utilr   Úldm.models.autoencoderÚldmÚ	packagingr	   ÚLightningModuler
   rÂ   ÚmodelsÚautoencoderr<   r<   r<   r=   Ú<module>   s     }
