a
    Ady                     @   sr  d dl Z d dlmZ d dlmZ d dlZd dlZd dlZd dlZd dl	Z	d dl
Zd dlZd dlmZmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlZd dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z% d d	l&m'Z' ed
ddgZ(i Z)dd Z*G dd dZ+G dd dZ,G dd dZ-d*ddZ.dd Z/dd Z0dd Z1dd Z2d d! Z3d+d#d$Z4d%d& Z5d,d(d)Z6dS )-    N)
namedtuple)closing)ImagePngImagePlugin)SummaryWriter)	shareddevices	sd_hijack	sd_modelsimagessd_samplerssd_hijack_checkpointerrorshashes)LearnRateScheduler)embedding_to_b64embedding_from_b64insert_image_data_embedextract_image_data_embedcaption_image_overlay)save_settings_to_fileTextualInversionTemplatenamepathc                  C   sL   t   ttjjD ]0\} }}|D ] }tj| |}t	||t |< q$qt S N)
textual_inversion_templatesclearoswalkr   cmd_optstextual_inversion_templates_dirr   joinr   )root_fnsfnr    r&   S/var/www/html/stable-diffusion-webui/modules/textual_inversion/textual_inversion.py list_textual_inversion_templates   s    r(   c                   @   s.   e Zd Zd
ddZdd Zdd Zdd	 ZdS )	EmbeddingNc                 C   sL   || _ || _|| _d | _d| _d | _d | _d | _d | _d | _	d | _
d | _d S Nr   )vecr   stepshapevectorscached_checksumsd_checkpointsd_checkpoint_nameoptimizer_state_dictfilenamehash	shorthash)selfr+   r   r,   r&   r&   r'   __init__)   s    zEmbedding.__init__c                 C   sh   ddid| j i| j| j| j| jd}t|| tjj	rd| j
d urd|  | j
d}t|| d d S )N*i	  )Zstring_to_tokenstring_to_paramr   r,   r0   r1   )r4   r2   .optim)r+   r   r,   r0   r1   torchsaver   optssave_optimizer_stater2   checksum)r6   r3   Zembedding_dataoptimizer_saved_dictr&   r&   r'   r<   7   s    	zEmbedding.savec                 C   s<   | j d ur| j S dd }|| jdd d@ d| _ | j S )Nc                 S   s*   d}| D ]}|d t |d A d@ }q|S )Nr   i  i  l    )int)arvr&   r&   r'   
const_hashN   s    z&Embedding.checksum.<locals>.const_hashd   i  Z04x)r/   r+   reshape)r6   rE   r&   r&   r'   r?   J   s
    
zEmbedding.checksumc                 C   s   || _ | j dd | _d S )Nr      )r4   r5   )r6   rD   r&   r&   r'   set_hashW   s    zEmbedding.set_hash)N)__name__
__module____qualname__r7   r<   r?   rJ   r&   r&   r&   r'   r)   (   s   
r)   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )!DirWithTextualInversionEmbeddingsc                 C   s   || _ d | _d S r   )r   mtimer6   r   r&   r&   r'   r7   ]   s    z*DirWithTextualInversionEmbeddings.__init__c                 C   s<   t j| jsdS t j| j}| jd u s4|| jkr8dS d S )NFTr   r   isdirgetmtimerO   )r6   mtr&   r&   r'   has_changeda   s
    z-DirWithTextualInversionEmbeddings.has_changedc                 C   s&   t j| jsd S t j| j| _d S r   rQ   r6   r&   r&   r'   updatei   s    z(DirWithTextualInversionEmbeddings.updateN)rK   rL   rM   r7   rU   rW   r&   r&   r&   r'   rN   \   s   rN   c                   @   s^   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dddZdd ZdS )EmbeddingDatabasec                 C   s(   i | _ i | _i | _d| _i | _d| _d S )NrF   r&   )
ids_lookupword_embeddingsskipped_embeddingsexpected_shapeembedding_dirspreviously_displayed_embeddingsrV   r&   r&   r'   r7   q   s    zEmbeddingDatabase.__init__c                 C   s   t || j|< d S r   )rN   r]   rP   r&   r&   r'   add_embedding_diry   s    z#EmbeddingDatabase.add_embedding_dirc                 C   s   | j   d S r   )r]   r   rV   r&   r&   r'   clear_embedding_dirs|   s    z&EmbeddingDatabase.clear_embedding_dirsc                 C   s   |  |||jS r   )register_embedding_by_namer   )r6   	embeddingmodelr&   r&   r'   register_embedding   s    z$EmbeddingDatabase.register_embeddingc                    s   |j  gd }|d }|| jvr.g | j|<  | jv rR fdd| j| D }n
| j| }|d urr|||fg7 }t|dd dd| j|< |d u r | jv r| j = t| j| dkr| j|= d S || j < |S )Nr   c                    s   g | ]}|d  j  kr|qS )   r   .0xrf   r&   r'   
<listcomp>       z@EmbeddingDatabase.register_embedding_by_name.<locals>.<listcomp>c                 S   s   t | d S r*   )len)ri   r&   r&   r'   <lambda>   rk   z>EmbeddingDatabase.register_embedding_by_name.<locals>.<lambda>T)keyreverse)cond_stage_modeltokenizerY   rZ   sortedrl   )r6   rb   rc   r   idsZfirst_idlookupr&   rf   r'   ra      s$    





z,EmbeddingDatabase.register_embedding_by_namec                 C   s   t jjdd}|jd S )N,re   )r   sd_modelrp   encode_embedding_init_textr-   )r6   r+   r&   r&   r'   get_expected_shape   s    z$EmbeddingDatabase.get_expected_shapec                 C   s  t j|\}}| }|dv rt j|\}}| dkr@d S t|}t|drzd|jv rzt|jd }|	d|}qt
|}|r|	d|}qd S n6|dv rtj|dd}n|d	v rtjj|dd
}nd S d|v rF|d }	t|	d|	}	t|	dksJ dtt|	 d }
|
 jtjtjd}|jd }|jd }n
t|tkrd|v rd|v rdd | D }|d jd |d jd  }|d jd }nt|tkr@ttt| tjkr@t| dksJ dtt| }
t|
jdkr|
d}
|
 jtjtjd}|jd }|jd }nt d| dt!||}|	dd |_"|	dd |_#|	dd |_$||_%||_||_&|'t()|j&d| pd | j*dks| j*|jkr| +|t,j- n
|| j.|< d S )N)z.PNGz.WEBPz.JXLz.AVIFz.PREVIEWtextsd-ti-embeddingr   )z.BINz.PTcpumap_location)z.SAFETENSORSdevicer9   _parametersre   z'embedding file has multiple terms in itdtyperF   r   clip_gclip_lc                 S   s(   i | ] \}}||  jtjtjd qS )r   )detachtor   r   r;   float32)rh   krD   r&   r&   r'   
<dictcomp>   rk   z4EmbeddingDatabase.load_from_file.<locals>.<dictcomp>zCouldn't identify z= as neither textual inversion embedding nor diffuser concept.r,   r0   r1   ztextual_inversion/ )/r   r   splitextupperr   openhasattrry   r   getr   r;   loadsafetensors	load_filegetattrrl   nextiteritemsr   r   r   r   r   r-   typedictvaluesTensorkeys	unsqueeze	Exceptionr)   r,   r0   r1   r.   r3   rJ   r   sha256r\   rd   r   rv   r[   )r6   r   r3   r   extr#   Z
second_extZembed_imagedataZ
param_dictembr+   r-   r.   rb   r&   r&   r'   load_from_file   sf    


"*


z EmbeddingDatabase.load_from_filec              
   C   s   t j|jsd S t j|jddD ]r\}}}|D ]b}z2t j||}t |jdkrXW q0| || W q0 ty   t	j
d| dd Y q0Y q00 q0q"d S )NT)followlinksr   zError loading embedding exc_info)r   r   rR   r   r!   statst_sizer   r   r   report)r6   embdirr"   r#   r$   r%   fullfnr&   r&   r'   load_from_dir   s    zEmbeddingDatabase.load_from_dirFc                 C   s@  |s0d}| j  D ]}| rd} q(q|s0d S | j  | j  | j  |  | _| j  D ]}| 	| |
  qbdd t| j dd dD }| j  | j
| t| j t| j f}tjjr<| j|kr<|| _tdt| j d	d
| j   | jr<tdt| j d	d
| j   d S )NFTc                 S   s   i | ]}|j |qS r&   rf   )rh   er&   r&   r'   r     rk   zGEmbeddingDatabase.load_textual_inversion_embeddings.<locals>.<dictcomp>c                 S   s
   | j  S r   )r   lower)r   r&   r&   r'   rm     rk   zEEmbeddingDatabase.load_textual_inversion_embeddings.<locals>.<lambda>)rn   z$Textual inversion embeddings loaded(z): z, z%Textual inversion embeddings skipped()r]   r   rU   rY   r   rZ   r[   rx   r\   r   rW   rr   tupler   r   r=   textual_inversion_print_at_loadr^   printrl   r!   )r6   force_reloadZneed_reloadr   Zsorted_word_embeddingsZdisplayed_embeddingsr&   r&   r'   !load_textual_inversion_embeddings   s0    





 
&z3EmbeddingDatabase.load_textual_inversion_embeddingsc                 C   s\   || }| j |d }|d u r"dS |D ]0\}}|||t|  |kr&|t|f  S q&dS )N)NN)rY   r   rl   )r6   tokensoffsettokenZpossible_matchesrs   rb   r&   r&   r'   find_embedding_at_position  s    z,EmbeddingDatabase.find_embedding_at_positionN)F)rK   rL   rM   r7   r_   r`   rd   ra   rx   r   r   r   r   r&   r&   r&   r'   rX   p   s   C
!rX   r8   c           
      C   s   t jj}t  |dg W d    n1 s00    Y  ||pDd|}tj||jd ftj	d}|rt
|D ]"}||t|jd  |  ||< qpddd | D } tjt jj|  d}|stj|rJ d	| d
t|| }	d|	_|	| |S )Nr   r8   re   r~   r   c                 s   s"   | ]}|  s|d v r|V  qdS )z._- N)isalnumrg   r&   r&   r'   	<genexpr>0  rk   z#create_embedding.<locals>.<genexpr>.ptzfile z already exists)r   rv   rp   r   autocastrw   r;   zerosr-   r   rangerA   r!   r   r   r   embeddings_direxistsr)   r,   r<   )
r   num_vectors_per_tokenoverwrite_old	init_text
cond_modelembeddedr+   ir%   rb   r&   r&   r'   create_embedding   s     
( 

r   c           
      C   s   t jjdkrd S |t jj dkr$d S tjtj| |r>dnd}ttj| |dddf}tj	|ddd	g|
 d
}|r|  |d | }|d | }	||||	d| W d    n1 s0    Y  d S )Nr   FTza+r   )newliner,   epoch
epoch_step)
fieldnamesre   )r,   r   r   )r   r=   training_write_csv_everyr   r   r   r!   r   csv
DictWriterr   writeheaderwriterow)
log_directoryr3   r,   Z	epoch_lenr   Zwrite_csv_headerfoutZ
csv_writerr   r   r&   r&   r'   
write_loss<  s$    r   c                 C   s2   t jt j| ddd tt j| dtjjdS )NtensorboardTexist_ok)log_dirZ
flush_secs)r   makedirsr   r!   r   r   r=    training_tensorboard_flush_every)r   r&   r&   r'   tensorboard_setupT  s
    r   c                 C   sH   t | d|| t | d| || t | d|| t | d| || d S )Nz
Loss/trainzLoss/train/epoch-zLearn rate/trainzLearn rate/train/epoch-)tensorboard_add_scaler)tensorboard_writerlossglobal_stepr,   
learn_rate	epoch_numr&   r&   r'   tensorboard_addZ  s    r   c                 C   s   | j |||d d S )N)tagscalar_valuer   )
add_scalar)r   r   valuer,   r&   r&   r'   r   `  s    r   c                 C   sT   t tj|dd}||jd |jd t| }|d}| j	|||d d S )NT)copyre   r   )   r   re   )r   )
r;   	as_tensornparrayviewsizerl   getbandspermute	add_image)r   r   	pil_imager,   Z
img_tensorr&   r&   r'   tensorboard_add_imaged  s    

r   rb   c                 C   sl  | sJ | d|sJ dt |ts0J d|dks@J dt |tsRJ d|dksbJ d|snJ dtj|sJ d	t|sJ d|sJ d
|sJ d| dtj|jsJ d| d|sJ dt |tsJ d|dksJ dt |tsJ d|dks(J dt |	ts<J d|	dksNJ d|sZ|	rh|
shJ dd S )Nz not selectedzLearning rate is empty or 0zBatch size must be integerr   zBatch size must be positivez*Gradient accumulation step must be integerz+Gradient accumulation step must be positivezDataset directory is emptyzDataset directory doesn't existz!Prompt template file not selectedzPrompt template file z
 not foundz doesn't existzMax steps is empty or 0zMax steps must be integerzMax steps must be positivezSave {name} must be integerz!Save {name} must be positive or 0zCreate image must be integerz"Create image must be positive or 0zLog directory is empty)
isinstancerA   r   r   rR   listdirisfile)
model_namer   
batch_sizegradient_step	data_roottemplate_filetemplate_filenamestepsZsave_model_everycreate_image_everyr   r   r&   r&   r'   validate_train_inputsm  s*    r   c           [      C   sL
  ddl m} |pd}|pd}t|d }t||||||||
|||dd |j}dtj_dtj_	|
tj_
tjtjj| d} tj|tj d|}tjj}!|dkrtj|d	}"tj|"d
d nd }"|dkrtj|d}#tj|#d
d nd }#|dkr$|r$tj|d}$tj|$d
d nd }$tj}%|%jj| }&t }'|&jpLd}(|(|
krhdtj_	|&| fS t||
|(})|dkrtjjj n|dkrtjjj!nd }*|*rt||
|(dd}+dt"#| dtj_	tj$},tjj%rt&|}-tjj'}.t j(j)j*|||tjj+|tj,tj,j-t.j/|||||||	|d}/tjj0r^t1|i t2|'j3|'j4t5|/t5|&j6dt7  |/j8}t j(j)j9|/||/j:|.d}0|!rdt_$tj,j;<t.j= d
|&j6_>tj?j@|&j6g|)jAdd}1tjjBr6d }2tjC|  drtjD|  ddd}3|&E |3dd kr|3dd }2|2d ur.|1F|2 tGd ntGd  tjHjIJ }4|/j:}|/jK}t5|/| | }5t5|/| t5|/| |  }6d}7d}8d!}9d!}:d!};d}<tj,jLjMd"v }=d }>tNjN|
|( d#}?z^ztOP  tQ|
|( | D ]}@|)jRr 	qtjjSr 	qtT|0D ]\}A}B|A|6kr q|)U|1|&j |)jRr6 qtjjSrF q|*rX|+|&j t.V  |BjWj<t.j/|.d$}C|r|BjXj<t.j/|.d$}Dtj,-|BjY}E|=r|>d u r|Ztj,|E||}>|>g|Egd%}Fn|E}F|rtj,[|C|F|Dd | }G~Dntj,\|C|Fd | }G~C|8|G] 7 }8W d    n1 s,0    Y  |4^|G_  |Ad& | dkrZq |*rn|*|&j6|+jA |4|1 |4`  |& jd&7  _|?`  |1jad
d' |8}7d}8|&jd& }H|&j|5 }I|&j|5 }Jd(|I d)|Jd&  d*|5 d+|7d,}K|?b|K |"d urF|H| dkrF| d-|H }Ltj|"|L d}9tc|&|1|'|L|9d
d. d
}<td|d/|&j|5|7d,|)jAd0 |#d u	r@|H| dk	r@| d-|H };tj|#|;}:tj,j;<t.j/ |jetj,d
d
d
d1}M|r||M_f||M_g||M_htijj| jk|M_l||M_m||M_n||M_o||M_pn|BjYd |M_fd2|M_h||M_o||M_p|Mjf}Ntq|M8 |r|M}Ot5|OjsdkrN|Ojsd nd }PW d    n1 sh0    Y  |!rtj,j;<t.j= |Pd urtjt|P tsju|P|#d3|Mjn|Mjftjjv|Ojwd |M|;dd4
\}:}Q|:d5|N 7 }:tjj%rtjjxrty|-d6|I |P|&j |	rtjC|9	r|<	rtj|$| d-|H d7}Rtz{ }StD|9}T|S|d8t}|T d9|Td:d; d<}Uzt~|Td=  d jd }VW n ty   d>}VY n0 t }'|'j3}Wd?|'j4 d@}X|V dA|H dB}Yt|P|U|W|X|Y}Zt|Z|T}Z|Zj|RdC|SdD d}<tsju|P|#d3|Mjn|Mjftjjv|Ojwd |M|;dd4
\}:}Q|:d5|N 7 }:|&jtj_dE|7d,dF|H dGt"#|BjYd  dHt"#|9 dIt"#|: dJtj_	q qtjtjj| d} tc|&|1|'|| d
d. W n" t	y   tjdKd
dL Y n0 W d|?_|?  tj,j;<t.j/ |,t_$tO  n.d|?_|?  tj,j;<t.j/ |,t_$tO  0 |&| fS )MNr   )
processingrb   rf   ztrain-embeddingz*Initializing textual inversion training...r   z%Y-%m-%d
embeddingsTr   r   Zimage_embeddingsz9Model has already been trained beyond specified max stepsr   normF)verbosezPreparing dataset from z...)r   widthheightrepeatsplaceholder_tokenrc   r   r   r   r   r   shuffle_tagstag_drop_outlatent_sampling_methodvarsize
use_weight)r   
model_hashnum_of_dataset_imagesr   )r  r   
pin_memoryg        )lrweight_decayr:   r{   r|   r4   r2   z)Loaded existing optimizer from checkpointz'No saved optimizer exists in checkpointz<none>>   hybridconcat)total)non_blocking)c_concatc_crossattnre   )set_to_nonez"Training textual inversion [Epoch z: /z] loss: z.7f-)remove_cached_checksumztextual_inversion_loss.csv)r   r   )rv   do_not_save_griddo_not_save_samplesdo_not_reload_embeddings   r   )pforced_filenamesave_to_dirsz
, prompt: zValidation at epoch z.pngrz   <r   z???>r9   ?[]zv sPNG)pnginfoz
<p>
Loss: z<br/>
Step: z<br/>
Last prompt: z<br/>
Last saved embedding: z<br/>
Last saved image: z<br/>
</p>
zError training embeddingr   )modulesr   r   r   r   r   r   statejobtextinfo	job_countr   r!   r   r   datetimenowstrftimer=   unload_models_when_trainingr   r	   model_hijackembedding_dbrZ   r
   select_checkpointr,   r   r;   nnutilsclip_grad_value_clip_grad_norm_htmlescapeparallel_processing_allowedtraining_enable_tensorboardr   r  textual_inversiondatasetPersonalizedBase training_image_repeats_per_epochrv   rp   r   r   save_training_settings_to_txtr   r   r   r5   rl   r+   localsr  PersonalizedDataLoaderr   first_stage_modelr   r{   requires_gradoptimAdamWr   r>   r   r   r?   load_state_dictr   cudaamp
GradScalerr   rc   conditioning_keytqdmr   addr   finishedinterrupted	enumerateapplyr   latent_sampleweight	cond_texttxt2img_image_conditioningweighted_forwardforwarditemscalebackwardrW   	zero_gradset_descriptionsave_embeddingr    StableDiffusionProcessingTxt2Imgpromptnegative_promptr   r   samplersr   sampler_name	cfg_scaleseedr   r  r   process_imagesr   assign_current_image
save_imagesamples_format	infotexts training_tensorboard_save_imagesr   r   PngInfoadd_textr   listr   r-   r   r   r   r<   job_nor   r   leavecloseremove)[id_taskembedding_namer   r   r   r   r   training_widthtraining_heightr  r   clip_grad_modeclip_grad_valuer  r  r  r  r   Zsave_embedding_everyr   Z save_image_with_stored_embeddingpreview_from_txt2imgpreview_promptpreview_negative_promptpreview_stepspreview_sampler_indexpreview_cfg_scalepreview_seedpreview_widthpreview_heightr   r   r3   unloadZembedding_dir
images_dirZimages_embeds_dirhijackrb   
checkpointinitial_step	scheduler	clip_gradclip_grad_schedold_parallel_processing_allowedr   r  dsdl	optimizerr2   r@   scalersteps_per_epochmax_steps_per_epoch	loss_step
_loss_steplast_saved_filelast_saved_imager  Zembedding_yet_to_be_embeddedZis_training_inpainting_modelimg_cpbarr#   jbatchri   wccondr   
steps_doner   r   descriptionZembedding_name_everyr  preview_text	processedimagelast_text_infoZlast_saved_image_chunksinfor   titleZ
vectorSizeZfooter_leftZ
footer_midZfooter_rightZcaptioned_imager&   r&   r'   train_embedding  s    

:
.








,



"


>
0


0

	

r  Tc           
      C   s   | j }t| dr| jnd }t| dr*| jnd }t| dr>| jnd }	z8|j| _|j| _|r^d | _|| _ | | _| 	| W n&   || _|| _|| _ |	| _ Y n0 d S )Nr0   r1   r/   )
r   r   r0   r1   r/   r5   r   
state_dictr2   r<   )
rb   r  r  rr  r3   r  Zold_embedding_nameold_sd_checkpointold_sd_checkpoint_nameZold_cached_checksumr&   r&   r'   r\    s$    
r\  )r8   )rb   )T)7r   collectionsr   
contextlibr   r;   rK  r7  r,  r   safetensors.torchr   numpyr   PILr   r   torch.utils.tensorboardr   r'  r   r   r	   r
   r   r   r   r   r   !modules.textual_inversion.dataset(modules.textual_inversion.learn_scheduler   Z)modules.textual_inversion.image_embeddingr   r   r   r   r   Z!modules.textual_inversion.loggingr   r   r   r(   r)   rN   rX   r   r   r   r   r   r   r   r  r\  r&   r&   r&   r'   <module>   sD   ,4 1
	
  !