a
    d8                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZ d dlZddlmZmZ ddlmZmZmZmZmZmZ ddlmZ dd	lmZmZmZ dd
lm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z, dZ-ee.j/d gZ0i a1dd Z2dd Z3e3  dd Z4dd Z5dd Z6dd Z7d1e8dddZ9d2d!d"Z:d3e8ee8 e8ee8ej;f e<e<e<ee= eee>ee>e>f f  e<e<ee8 ee< e<d%d&d'Z?d(d) Z@d4e8ee8 e8ee8ej;f e<e<e<ee= eee>ee>e>f f  e<e<eee=d*f  eee=d*f  eee
e8e	f e)f  ee8 ee< d+d,d-ZAd5e8ee8 e8ee8ej;f e<e<e<eee>ee>e>f f  e<eee=d*f  eee=d*f  ee8 d.d/d0ZBdS )6    N)deepcopy)Path)AnyDictOptionalTupleUnion   )OPENAI_DATASET_MEANOPENAI_DATASET_STD)CLIPCustomTextCLIPconvert_weights_to_lp!convert_to_custom_text_state_dictresize_pos_embedget_cast_dtype)CoCa)ClipLossDistillClipLossCoCaLoss)load_openai_model)is_pretrained_cfgget_pretrained_cfgdownload_pretrainedlist_pretrained_tags_by_modeldownload_pretrained_from_hf)image_transformAugmentationCfg)HFTokenizertokenizezhf-hub:zmodel_configs/c                 C   s   dd t d|  D S )Nc                 S   s    g | ]}|  rt|n|qS  )isdigitint).0sr    r    Z/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/open_clip/factory.py
<listcomp>       z _natural_key.<locals>.<listcomp>z(\d+))resplitlower)Zstring_r    r    r%   _natural_key   s    r+   c               	      s   d} g }t D ]J}| r.|j| v r.|| q| r| D ]}||d|  q:q|D ]X}t|d:}t	| t
 fdddD r t|j< W d    q\1 s0    Y  q\dd tt d	d
 dD ad S )N)z.json*rc                 3   s   | ]}| v V  qd S Nr    )r#   a	model_cfgr    r%   	<genexpr>0   r'   z(_rescan_model_configs.<locals>.<genexpr>)Z	embed_dim
vision_cfgtext_cfgc                 S   s   i | ]\}}||qS r    r    r#   kvr    r    r%   
<dictcomp>3   r'   z)_rescan_model_configs.<locals>.<dictcomp>c                 S   s   t | d S )Nr   )r+   )xr    r    r%   <lambda>3   r'   z'_rescan_model_configs.<locals>.<lambda>)key)_MODEL_CONFIG_PATHSis_filesuffixappendis_dirextendglobopenjsonloadall_MODEL_CONFIGSstemsorteditems)Z
config_extconfig_filesconfig_pathextcffr    r0   r%   _rescan_model_configs!   s    
*rP   c                   C   s   t t S )z? enumerate available model architectures based on config files )listrG   keysr    r    r    r%   list_models9   s    rS   c                 C   s&   t | tst| } t|  t  dS )z3 add model config path or file and update registry N)
isinstancer   r<   r?   rP   )pathr    r    r%   add_model_config>   s    

rV   c                 C   s   | t v rtt |  S d S d S r.   )rG   r   )
model_namer    r    r%   get_model_configF   s    rX   c                 C   sL   |  tr t| ttd  }n(t| }d|d v rDt|d d nt}|S )NZhf_tokenizer_namer4   )
startswithHF_HUB_PREFIXr   lenrX   r   )rW   	tokenizerconfigr    r    r%   get_tokenizerM   s    


r^   cpu)checkpoint_pathc                 C   s^   t j| |d}t|tr*d|v r*|d }n|}tt| d drZdd | D }|S )N)map_location
state_dictr   modulec                 S   s   i | ]\}}|d d |qS )   Nr    r5   r    r    r%   r8   ^   r'   z#load_state_dict.<locals>.<dictcomp>)torchrE   rT   dictnextiterrJ   rY   )r`   ra   
checkpointrb   r    r    r%   load_state_dictW   s    
rj   Tc                 C   s>   t |}d|v r"t| ds"t|}t||  | j ||d}|S )NZpositional_embedding)strict)rj   hasattrr   r   )modelr`   rk   rb   Zincompatible_keysr    r    r%   load_checkpointb   s    
rn   fp32F)rW   
pretrained	precisiondevicejitforce_quick_geluforce_custom_textforce_patch_dropoutforce_image_sizepretrained_imagepretrained_hf	cache_diroutput_dictrequire_pretrainedc                    s4  |  t}|r| ttd  }t||d}t|d|d}t|ddd}t|}W d    n1 sf0    Y  |d }|d }n| d	d
} d }i }d }t|t	rt
|}|r| dkrtd|  d t| |||d}n"|pt| }|d urtd|  d n*td|  dt  d td|  d|rFd|d< |d ur\||d d< |d urr||d d< d|di v }|	r|rd|d d< ndsJ dt|}d|di v }|d dp|p|}|r,|r|
|d d!< d"| v rtf i |d#|i}ntf i |d#|i}ntf i |d#|i}|d$v rd%|v r\t
jnt
j}|r|j||d& d'd(lm   fd)d*}|| n|j|d+ t ||d, n<|d-v rd%|v rt
jnt
j}|j||d& n|j|d+ d}|rd.}t!| |}|rt"||d}nt#j$%|r.|}|rXtd|  d/| d0 t&|| n.d1| d2|  d3t'|  d}t(| t|d}n,|rtd|  d/| d0 t&|| d}|r|std4|  d5| d6|d7d pt)|j*_+|d8d p t,|j*_-|rt.|d9rd|_/|r0t
j01|}|S ):N)rz   zopen_clip_config.json)filenamerz   r-   zutf-8)encodingZpreprocess_cfgr1   /-openaizLoading pretrained z from OpenAI.)rq   rr   rz   zLoaded z model config.zModel config for z not found; available models .z not found.TZ
quick_gelur3   Zpatch_dropout
image_sizeZtimm_model_nameZtimm_model_pretrainedFz@pretrained image towers currently only supported for timm modelsZhf_model_namer4   custom_textZhf_model_pretrainedcoca
cast_dtype)fp16Zbf16r   )rr   dtyper	   LayerNormFp32c                    s6   t |  r2| jjtj| j_| jjtj| j_d S r.   )rT   Zweightdatatore   float32Zbias)mr   r    r%   _convert_ln   s    
z!create_model.<locals>._convert_ln)rr   )r   )Z	pure_fp16Z	pure_bf16 z
 weights (z).zPretrained weights (z) not found for model z.Available pretrained tags (z-Pretrained weights were required for (model: z, pretrained: z) but not loaded.meanstdr{   )2rY   rZ   r[   r   rC   rD   rE   replacerT   strre   rr   r*   logginginfor   rX   errorrS   RuntimeErrorgetr   popr   r   r   float16Zbfloat16r   Ztransformerr   applyr   r   r   osrU   existsrn   r   warningr
   visual
image_meanr   	image_stdrl   r{   rs   script)rW   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   Zhas_hf_hub_prefixZmodel_idr`   rL   rO   r]   Zpretrained_cfgr1   rm   Zis_timm_modelr   Zis_hf_modelr   r   r   Zpretrained_loadedZ	error_strr    r   r%   create_modell   s    
(












r   c              
   C   sv   | j r$t| j| jd| j| j| jdS d| j v rXt	| j
| j| j| jd| j| j| jdS t| j| jd| j| j| jdS )NT)
local_lossgather_with_gradcache_labelsrank
world_sizeuse_horovodr   )Zcaption_loss_weightZclip_loss_weightr   r   r   r   r   r   )Zdistillr   r   r   r   r   Zhorovodrm   r*   r   Zcoca_caption_loss_weightZcoca_contrastive_loss_weightr   )argsr    r    r%   create_loss  s8    
r   .)rW   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   r   r   aug_cfgrz   r{   c                 C   sz   t | |||||||||	|
||d}|p2t|jdd }|pDt|jdd }t|jjd|||d}t|jjd||d}|||fS )N)rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r   r   T)is_trainr   r   r   Fr   r   r   r   getattrr   r   r   )rW   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   r   r   r   rz   r{   rm   Zpreprocess_trainZpreprocess_valr    r    r%   create_model_and_transforms"  s>    r   )rW   rp   rq   rr   rs   rt   ru   rw   return_transformr   r   rz   c                 C   sd   t | ||||||||dd
}|s$|S |	p4t|jdd }	|
pFt|jdd }
t|jjd|	|
d}||fS )NT)rq   rr   rs   rt   ru   rw   rz   r|   r   r   Fr   r   )rW   rp   rq   rr   rs   rt   ru   rw   r   r   r   rz   rm   
preprocessr    r    r%   create_model_from_pretrainedW  s.    r   )r_   )T)Nro   r_   FFFNNFTNNF)Nro   r_   FFFNNFTNNNNN)Nro   r_   FFFNTNNN)CrD   r   r   pathlibr(   copyr   r   typingr   r   r   r   r   re   	constantsr
   r   rm   r   r   r   r   r   r   Z
coca_modelr   Zlossr   r   r   r   r   rp   r   r   r   r   r   Z	transformr   r   r\   r   r   rZ   __file__parentr<   rG   r+   rP   rS   rV   rX   r^   r   rj   rn   rr   boolfloatr"   r   r   r   r   r    r    r    r%   <module>   s    

              !               7           