a
    Adr                     @   s|  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZ	d dl
mZ d dlmZ d dlmZ d dlm  mZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'Z'dZ(ej)*ej)+ej,e(Z-i Z.i Z/e/Z0e 1 Z2d	d
 Z3G dd dZ4zd dl5m6Z6m7Z7 e68  W n e9yX   Y n0 dd Z:dMddZ;dd Z<e=dZ>dd Z?dd Z@dd ZAddddZBd d! ZCd"d# ZDd$d% ZEdNd&d'ZFe4d(d)d*ZGG d+d, d,ZHe4d(d-d.ZId/d0 ZJd1d2 ZKd3ZLd4ZMd5ZNd6ZOG d7d8 d8ZPeP ZQd9d: ZRd;d< ZSd=d> ZTd?d@ ZUdAdB ZVdOdCdDZWdEdF ZXdPdGdHZYdQdIdJZZdKdL Z[dS )R    N)	OmegaConf)mkdir)request)instantiate_from_config)pathssharedmodelloaderdevicesscript_callbackssd_vaesd_disable_initializationerrorshashessd_models_configsd_unetsd_models_xlcacheextra_networks
processinglowvram	sd_hijack)TimerzStable-diffusionc                    sZ   t   }| |< ||vr  S ||}|||<  fdd|D }    |  S )Nc                    s   i | ]}| | qS  r   ).0kdr   9/var/www/html/stable-diffusion-webui/modules/sd_models.py
<dictcomp>)       zreplace_key.<locals>.<dictcomp>)listkeysindexclearupdate)r   keynew_keyvaluer!   r"   new_dr   r   r   replace_key   s    

r)   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )CheckpointInfoc              
      sD   _ tj }tj d  dk_tjj	d urX|
tjj	rX|tjj	d}n$|
trp|td}ntj }|
ds|
dr|dd  } fdd}i _jrztdd	|  |_W n8 ty
 } zt|d
   W Y d }~n
d }~0 0 |_tjtj d _tj|ddddd _t _tj d	| _jrjdd nd _jd u r|n| dj d_jd u rjnj dj d_jjj|j| dj dg_jr@ jjjj dj dj dj dg7  _d S )N   .safetensors \/c                     s   t  } | dd _| S )Nzmodelspec.thumbnail)read_metadata_from_safetensorspopZmodelspec_thumbnail)metadatafilenameselfr   r   read_metadataA   s    z.CheckpointInfo.__init__.<locals>.read_metadatazsafetensors-metadatacheckpoint/zreading metadata for r   _
    []) r4   ospathabspathsplitextloweris_safetensorsr   cmd_optsckpt_dir
startswithreplace
model_pathbasenamer2   r   cached_data_for_file	Exceptionr   displaynamename_for_extra
model_name
model_hashhashr   Zsha256_from_cachesha256	shorthashtitleshort_titleids)r5   r4   r>   rK   r6   er   r3   r   __init__1   s8    
("
$((zCheckpointInfo.__init__c                 C   s"   | t | j< | jD ]}| t|< qd S N)checkpoints_listrR   rT   checkpoint_aliases)r5   idr   r   r   register]   s    

zCheckpointInfo.registerc              	   C   s   t | jd| j | _| jd u r&d S | jdd }| j| jdd krN| jS || _| j| jvr|  j| j| j| j d| j d| j d| j dg7  _| j}| j d| j d| _| j d| j d| _t	t
|| j|  |   | jS )Nr7   r   r9   r:   r;   )r   rP   r4   rK   rQ   rT   rL   rR   rS   r)   rX   r[   )r5   rQ   	old_titler   r   r   calculate_shorthashb   s    
:z"CheckpointInfo.calculate_shorthashN)__name__
__module____qualname__rV   r[   r]   r   r   r   r   r*   0   s   ,r*   )logging	CLIPModelc                   C   s   t jtdd t  d S )NT)exist_ok)r<   makedirsrF   enable_midas_autodownloadr   r   r   r   setup_model   s    rf   Fc                    s    fddt  D S )Nc                    s   g | ]} r|j n|jqS r   )rS   rR   r   xZ	use_shortr   r   
<listcomp>   r   z$checkpoint_tiles.<locals>.<listcomp>)rX   valuesri   r   ri   r   checkpoint_tiles   s    rl   c                  C   s   t   t  tjj} tjjs6| tjks6tj	
| r<d }nd}tjt|tjjddgdddgd}tj	
| rt| }|  |jtjjd< n,| d ur| tjkrtd	t d
|  tjd |D ]}t|}|  qd S )Nzbhttps://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensorsz.ckptr,   zv1-5-pruned-emaonly.safetensorsz	.vae.ckptz.vae.safetensors)rF   	model_urlcommand_path
ext_filterdownload_nameext_blacklistsd_model_checkpointzBCheckpoint in --ckpt argument not found (Possible it was moved to : file)rX   r#   rY   r   rB   ckptno_download_sd_modelsd_model_filer<   r=   existsr   load_modelsrF   rC   r*   r[   rR   optsdatadefault_sd_model_fileprintsysstderr)Zcmd_ckptrm   Z
model_listcheckpoint_infor4   r   r   r   list_models   s     "r   z\s*\[[^]]+]\s*$c                    s    sd S t  d }|d ur |S t fddt D dd d}|rN|d S ttd tfddt D d	d d}|r|d S d S )
Nc                    s   g | ]} |j v r|qS r   rR   r   info)search_stringr   r   rj      r   z/get_closet_checkpoint_match.<locals>.<listcomp>c                 S   s
   t | jS rW   lenrR   rh   r   r   r   <lambda>   r   z-get_closet_checkpoint_match.<locals>.<lambda>)r%   r   r-   c                    s   g | ]} |j v r|qS r   r   r   )search_string_without_checksumr   r   rj      r   c                 S   s
   t | jS rW   r   r   r   r   r   r      r   )rY   getsortedrX   rk   resubre_strip_checksum)r   r   foundr   )r   r   r   get_closet_checkpoint_match   s    ""r   c                 C   s   zjt | dL}ddl}| }|d ||d | dd W  d   W S 1 s^0    Y  W n ty~   Y dS 0 dS )zOold hash that only looks at a small part of the file and is prone to collisionsrbr   Ni   i      ZNOFILE)openhashlibrP   seekr$   read	hexdigestFileNotFoundError)r4   ru   r   mr   r   r   rN      s    
4rN   c                  C   s   t jj} t| d}|dur |S ttdkrd}t jjdurV|dt	j
t jj 7 }|dt 7 }t jjdur|dt	j
t jj 7 }|d7 }t|ttt }| durtd|  d|j tjd	 |S )
z7Raises `FileNotFoundError` if no checkpoints are found.Nr   z@No checkpoints found. When searching for checkpoints, looked at:z	
 - file z
 - directory zhCan't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations.zCheckpoint z not found; loading fallback rt   )r   r{   rr   rY   r   r   rX   rB   rv   r<   r=   r>   rF   rC   r   nextiterrk   r~   rR   r   r   )model_checkpointr   error_messager   r   r   select_checkpoint   s"    r   z3cond_stage_model.transformer.text_model.embeddings.z0cond_stage_model.transformer.text_model.encoder.z9cond_stage_model.transformer.text_model.final_layer_norm.)z(cond_stage_model.transformer.embeddings.z%cond_stage_model.transformer.encoder.z.cond_stage_model.transformer.final_layer_norm.c                 C   s4   t  D ]&\}}| |r|| t|d   } q| S rW   )checkpoint_dict_replacementsitemsrD   r   )r   textreplacementr   r   r   transform_checkpoint_dict_key   s    
r   c                 C   s\   |  d| } |  dd  i }|  D ] \}}t|}|d ur$|||< q$|   | | | S )N
state_dict)r1   r   r   r#   r$   )pl_sdsdr   vr&   r   r   r   get_state_dict_from_checkpoint   s    

r   c           
   
   C   s   dd l }t| dd}|d}t|d}|d}|dkrF|dv sTJ |  d|||d  }||}i }|d	i  D ]P\}}	|	||< t|	t	r|	dd
 dkrz||	||< W q t
y   Y q0 q|W  d    S 1 s0    Y  d S )Nr   r   )moder   little   )s   {"s   {'z is not a safetensors file__metadata__r+   {)jsonr   r   int
from_bytesloadsr   r   
isinstancestrrI   )
r4   r   ru   Zmetadata_lenZ
json_startZ	json_dataZjson_objresr   r   r   r   r   r0     s"    


r0   c                    s   t j| \}}| dkrv|p,tjp,t  tjj	sHt
jj|  d}qt
jt| d } fdd| D }ntj| |ptjd}|rd|v rtd|d   t|}|S )	Nr,   devicer   c                    s   i | ]\}}||  qS r   )to)r   r   r   r   r   r   r   &  r   z#read_state_dict.<locals>.<dictcomp>)map_locationglobal_stepzGlobal Step: )r<   r=   r?   r@   r   weight_load_locationr	   get_optimal_device_namer{   disable_mmap_load_safetensorssafetensorstorch	load_fileloadr   r   r   r~   r   )checkpoint_fileZprint_global_stater   r8   	extensionr   r   r   r   r   read_state_dict  s    r   )r   c                 C   s`   |   }|d | tv r2td| d t|  S td| d| j  t| j}|d |S )Ncalculate hashzLoading weights [z] from cachez] from zload weights from disk)r]   recordcheckpoints_loadedr~   r4   r   )r   timersd_model_hashr   r   r   r   get_checkpoint_state_dict1  s    


r   c                   @   s(   e Zd ZdZdZdZdd Zdd ZdS )SkipWritingToConfigzqThis context manager prevents load_model_weights from writing checkpoint name to the config when it loads weight.FNc                 C   s   t j| _dt _| S )NT)r   skippreviousr5   r   r   r   	__enter__G  s    zSkipWritingToConfig.__enter__c                 C   s   | j t_d S rW   )r   r   r   )r5   exc_type	exc_valueexc_tracebackr   r   r   __exit__L  s    zSkipWritingToConfig.__exit__)r^   r_   r`   __doc__r   r   r   r   r   r   r   r   r   A  s
   r   c           	      C   s0  |  }|d tjs&|jtjjd< |d u r8t||}t	| d| _
| j
 oVt	| jd| _| j
 oh| j | _| j
r|t|  | j|dd |d tjjdkr|t|< ~tjjr| jtjd	 |d
 tjjr|   tjt_|d nd| j}t| dd }tjjrd | _tjj r&|r&d | _!| "  || _|r@|| _!tj#t_|d tjj ottj$tj#kottjtj#kt_%| jtj& |d t'ttjjkrtj(dd q|| _)|j*| _+|| _,|j-tjjd< t	| dr| j.tj/| _.t01  t02  t03|j*4 \}}t05| || |d d S )Nr   rr   conditionermodelF)strictzapply weights to modelr   )memory_formatzapply channels_lastzapply float()depth_modelzapply half()zapply dtype to VAE)lastsd_checkpoint_hashlogvarzload VAE)6r]   r   r   r   rR   r   r{   r|   r   hasattris_sdxlcond_stage_modelis_sd2is_sd1r   Zextend_sdxlload_state_dictsd_checkpoint_cacher   rB   opt_channelslastr   r   channels_lastno_halffloatfloat32r	   
dtype_unetfirst_stage_modelgetattrno_half_vaeupcast_samplingr   halffloat16dtypeunet_needs_upcast	dtype_vaer   popitemr   r4   rr   sd_checkpoint_inforP   r   r   r   Zdelete_base_vaeZclear_loaded_vaeZresolve_vaetupleZload_vae)	r   r   r   r   r   vaer   Zvae_fileZ
vae_sourcer   r   r   load_model_weightsP  sf    






&
r   c                     s|   t jtjd tjj D ]*\} }t j	|}t j |tjj| < qdddddtjj
tj_ fdd}|tj_
d	S )
aT  
    Gives the ldm.modules.midas.api.load_model function automatic downloading.

    When the 512-depth-ema model, and other future models like it, is loaded,
    it calls midas.api.load_model to load the associated midas depth model.
    This function applies a wrapper to download the model to the correct
    location automatically.
    midaszRhttps://github.com/intel-isl/DPT/releases/download/1_0/dpt_large-midas-2f21e586.ptzShttps://github.com/intel-isl/DPT/releases/download/1_0/dpt_hybrid-midas-501f0c75.ptzShttps://github.com/AlexeyAB/MiDaS/releases/download/midas_dpt/midas_v21-f6b98070.ptzYhttps://github.com/AlexeyAB/MiDaS/releases/download/midas_dpt/midas_v21_small-70d6b9c8.pt)Z	dpt_large
dpt_hybridZ	midas_v21Zmidas_v21_smallc                    sj   t jj|  }tj|s^tj s,t  td|  d|  t	|  | t|  d t j
| S )Nz$Downloading midas model weights for z to z downloaded)r   api	ISL_PATHSr<   r=   ry   r   r~   r   urlretrieveload_model_inner)
model_typer=   Z
midas_pathZ
midas_urlsr   r   load_model_wrapper  s    z5enable_midas_autodownload.<locals>.load_model_wrapperN)r<   r=   joinr   models_pathr   r   r   r   rG   
load_modelr   )r   r   	file_namer  r   r   r   re     s    
re   c                 C   s   t | jjdsd| jj_t | jjdrTtjjr>d| jjjj_ntjj	rTd| jjjj_t
| jjjjjdd dkrtjsd| jjjjj_t | jjdrt | jjjjd	rtjtjd
}| jjjjjd|| jjjj_d S )Nuse_emaFunet_configT	attn_typezvanilla-xformersvanillanoise_aug_configclip_stats_pathZkarlozcheckpoints/karlo_models)r   r   paramsr  r   rB   r   r  use_fp16r   r   first_stage_configddconfigxformers_availabler  r
  r<   r=   r  r   r  r  rE   )	sd_configZ
karlo_pathr   r   r   repair_config  s    
  r  zIcond_stage_model.transformer.text_model.embeddings.token_embedding.weightzBcond_stage_model.model.transformer.resblocks.0.attn.in_proj_weightz-conditioner.embedders.1.model.ln_final.weightz-conditioner.embedders.0.model.ln_final.weightc                   @   s&   e Zd Zdd Zdd Zd	ddZdS )
SdModelDatac                 C   s    d | _ g | _d| _t | _d S )NF)sd_modelloaded_sd_modelswas_loaded_at_least_once	threadingLocklockr   r   r   r   rV     s    zSdModelData.__init__c                 C   s   | j r| jS | jd u r| j | jd us.| j rB| jW  d    S z
t  W nV ty } z>tj|ddd tdtj	d tdtj	d d | _W Y d }~n
d }~0 0 W d    n1 s0    Y  | jS )Nzloading stable diffusion modelTfull_tracebackr-   rt   z%Stable diffusion model failed to load)
r  r  r  r  rI   r   rJ   r~   r   r   )r5   rU   r   r   r   get_sd_model  s    

:zSdModelData.get_sd_modelFc                 C   sl   || _ |r.t|dd t_t|dd t_|jt_z| j| W n t	yP   Y n0 |d urh| j
d| d S )Nbase_vaeloaded_vae_filer   )r  r   r   r  r  r   r   r  remove
ValueErrorinsert)r5   r   already_loadedr   r   r   set_sd_model  s    zSdModelData.set_sd_modelN)F)r^   r_   r`   rV   r  r#  r   r   r   r   r    s   r  c                 C   sB   t  }t|i  t| dr2| dg}|d S | dgS d S )Nr   r-   	crossattn)r    StableDiffusionProcessingTxt2Imgr   activater   get_learned_conditioningr   )r  pr   r   r   r   get_empty_cond  s    
r)  c                 C   s(   | j rt   n| tj t  d S rW   )r   send_everything_to_cpur   r	   cputorch_gcr   r   r   r   send_model_to_cpu  s    
r.  c                 C   s   t | rtjS tjS d S rW   )r   	is_neededr	   r+  r   r-  r   r   r   model_target_device%  s    
r0  c                 C   s    t |  | j s| tj d S rW   )r   applyr   r   r   r-  r   r   r   send_model_to_device,  s    
r2  c                 C   s   | j dd t  d S )Nmetar   )r   r	   r,  r-  r   r   r   send_model_to_trash3  s    r4  c           
   
      s  ddl m} | pt } t }tjr:ttj d t_t  |	d |d urR| n
t
| | t | }t fddttttfD }|	d t|}t| |	d td|  d }zltj|ptjjd	B t  t|j}W d    n1 s0    Y  W d    n1 s0    Y  W n6 ty` } ztj |d
dd W Y d }~n
d }~0 0 |d u rtdt!j"d t  t|j}W d    n1 s0    Y  ||_#|	d tjj$rd }	nd t%j&d}	tj' t(||	d t)||  | W d    n1 s0    Y  |	d t*| |	d |j+,| |	d |-  t.| dt_/|j+j0j1dd |	d t23| |	d t4 D t%5  t6||_7W d    n1 s0    Y  W d    n1 s0    Y  |	d td|8  d |S )Nr   )r   zunload existing modelc                 3   s   | ]}| v r|V  qd S rW   r   rg   r   r   r   	<genexpr>K  r   zload_model.<locals>.<genexpr>find configzload configzCreating model from config: )Zdisable_clipzcreating model quicklyTr  z=Failed to create model quickly; will retry using slow method.rt   zcreate model)r   r-   )r   weight_dtype_conversionzload weights from state dictmove model to devicehijack)force_reloadz!load textual inversion embeddingszscripts callbackszcalculate empty promptzModel loaded in .)9modulesr   r   r   
model_datar  r4  r	   r,  r   r   r   find_checkpoint_configanysd1_clip_weightsd2_clip_weightsdxl_clip_weightsdxl_refiner_clip_weightr   r   r  r~   r   ZDisableInitializationr   rB   do_not_download_clipZInitializeOnMetar   r   rI   r   rJ   r   r   used_configr   r   r   ZLoadStateDictOnMetar0  r   r2  model_hijackr:  evalr#  r  embedding_db!load_textual_inversion_embeddingsr
   model_loaded_callbackautocastno_gradr)  Zcond_stage_model_empty_promptsummary)
r   already_loaded_state_dictr   r   checkpoint_configZclip_is_included_into_sdr  r  rU   r8  r   r5  r   r  8  sn    







L&

*

.






J
r  c                 C   s  d}t tttjD ]}tj| }|jj|jkr8|}qttjtjj	  krVdkrn nDt
dttj dtjj	 d|jj  tj  t| |d tjjrt|  |d q|dur8t| |d tj|d	d
 tjs|jjtjjd< |jjtjjd< t
d|jj d|   t| tjS tjj	dkrttjtjj	k rt
d|j dttjd  dtjj	 d dt_t| tjS ttjdkrtj } | t_t| ddt_t| ddt_| jt_t
d| jj d|j  | S dS dS )a  
    Checks if the desired checkpoint from checkpoint_info is not already loaded in model_data.loaded_sd_models.
    If it is loaded, returns that (moving it to GPU if necessary, and moving the currently loadded model to CPU if necessary).
    If not, returns the model that can be used to load weights from checkpoint_info's file.
    If no such model exists, returns None.
    Additionaly deletes loaded models that are over the limit set in settings (sd_checkpoints_limit).
    Nr   zUnloading model z over the limit of rs   zsend model to trashzsend model to cpuzsend model to deviceT)r"  rr   r   zUsing already loaded model z
: done in r+   zLoading model z (z out of )r  r  zReusing loaded model z	 to load ) reversedranger   r>  r  r   r4   r   r{   sd_checkpoints_limitr~   rR   r1   r4  r   sd_checkpoints_keep_in_cpur.  r2  r#  r   r   r|   rP   rN  r   reload_vae_weightsr  r  r   r  r  r   )r  r   r   r"  iZloaded_modelr   r   r   reuse_model_from_already_loaded  sL    	
"(




",
rX  c              	   C   s  |pt  }t }| stj} | d u r(d }n| j}| j|jkr>| S t| ||} | d urd| jj|jkrd| S | d urt	d t
|  tj|  t||}t||}|d | d u s|| jkr| d urt|  t||d tjS zzt| ||| W n, ty   td t| |d |  Y n0 W tj|  |d t|  |d | js| tj |d nJtj|  |d t|  |d | js| tj |d 0 td|  d	 t |  t	  | S )
NNoner7  )rO  z-Failed to load checkpoint, restoring previousr:  zscript callbacksr9  zWeights loaded in r<  )!r   r   r>  r  r   rr   r4   rX  r   
apply_unetr.  r   rG  undo_hijackr   r   r?  r   rF  r4  r  r   rI   r~   r:  r
   rK  r   r   r	   r   rN  r#  )r  r   r   r   Zcurrent_checkpoint_infor   rP  r   r   r   reload_model_weights  s`    











r\  c                 C   sZ   t  }tjrBtjtj tjtj d t_d } t	
  t  td|  d | S )NzUnloaded weights r<  )r   r>  r  r   r	   r+  r   rG  r[  gccollectr,  r~   rN  )r  r   r   r   r   r   unload_model_weights  s    r_  c                 C   sR   t | dd}||krdS |dkr*t|  |dkrHtj| |ddddd || _dS )z=
    Applies speed and memory optimizations from tomesd.
    applied_token_merged_ratior   NFT)ratioZuse_randZ
merge_attnZmerge_crossattnZ	merge_mlp)r   tomesdZremove_patchapply_patchr`  )r  token_merging_ratioZcurrent_token_merging_ratior   r   r   apply_token_merging  s    
	re  )F)FN)NN)NN)NN)\collectionsos.pathr<   r   r]  r  r   r   safetensors.torchr   	omegaconfr   r   urllibr   Zldm.modules.midasr=  r   ldm.utilr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   modules.timerr   rb  	model_dirr=   r>   r  r  rF   rX   rY   Zcheckpoint_alisasesOrderedDictr   r)   r*   transformersra   rb   set_verbosity_errorrI   rf   rl   r   compiler   r   rN   r   r   r   r   r0   r   r   r   r   re   r  rA  rB  rC  rD  r  r>  r)  r.  r0  r2  r4  r  rX  r\  r_  re  r   r   r   r   <module>   s   LJ


M++	
Z;
>
