a
    Ad}!                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZmZmZmZmZmZ dZdZed	g d
ZedZdd Zdd ZG dd dZdS )    N)
namedtuple)Path)
transforms)InterpolationMode)devicespathssharedlowvrammodelloadererrorsi  zViT-L/14Categorynametopnitemsz\.top(\d+)\.c                   C   s   dd t tjjdD S )Nc                 S   s   g | ]
}|j qS  )stem).0fr   r   ;/var/www/html/stable-diffusion-webui/modules/interrogate.py
<listcomp>       z"category_types.<locals>.<listcomp>*.txt)r   r   interrogatorcontent_dirglobr   r   r   r   category_types   s    r   c              
   C   s   t d |  d}g d}zzNtj|dd |D ]*}tjd| dtj|| d q0t||  W n0 t	y } zt
|d W Y d }~n
d }~0 0 W tj|rt| ntj|rt| 0 d S )	NzDownloading CLIP categories..._tmp)artistsZflavorsZmediumsZ	movementsT)exist_okz`https://raw.githubusercontent.com/pharmapsychotic/clip-interrogator/main/clip_interrogator/data/z.txtz/downloading default CLIP interrogate categories)printosmakedirstorchhubdownload_url_to_filepathjoinrename	Exceptionr   displayexists
removedirs)r   tmpdirr   category_typeer   r   r   ,download_default_clip_interrogate_categories   s    
($r0   c                   @   s   e Zd ZdZdZdZdZdZdd Zdd Z	dd Z
dd	 Zd
d Zdd Zdd Zdd Zdd ZdddZdd Zdd ZdS )InterrogateModelsNc                 C   s(   d | _ g | _|| _tjtdk| _d S )Ncpu)loaded_categoriesskip_categoriesr   r   device_interrogater#   devicerunning_on_cpu)selfr   r   r   r   __init__4   s    zInterrogateModels.__init__c              	   C   s  t j| jst| j | jd ur6| jtjj	kr6| jS g | _t j| jrtjj	| _g }t
| jdD ]}||j |j| jv rqjt|j}|d u rdnt|d}t|ddd"}dd | D }W d    n1 s0    Y  | jt|j||d qj| jS )	Nr      rutf8)encodingc                 S   s   g | ]}|  qS r   )strip)r   xr   r   r   r   M   r   z0InterrogateModels.categories.<locals>.<listcomp>r   )r!   r&   r+   r   r0   r3   r4   r   opts interrogate_clip_skip_categoriesr   r   appendr   re_topnsearchintgroupopen	readlinesr   )r8   r   filenamemr   filelinesr   r   r   
categories:   s$    

0zInterrogateModels.categoriesc                 C   s   G dd d}|t jd< d S )Nc                   @   s   e Zd Zdd ZdS )z>InterrogateModels.create_fake_fairscale.<locals>.FakeFairscalec                 S   s   d S Nr   r8   r   r   r   checkpoint_wrapperU   s    zQInterrogateModels.create_fake_fairscale.<locals>.FakeFairscale.checkpoint_wrapperN)__name__
__module____qualname__rP   r   r   r   r   FakeFairscaleT   s   rT   z.fairscale.nn.checkpoint.checkpoint_activations)sysmodules)r8   rT   r   r   r   create_fake_fairscaleS   s    z'InterrogateModels.create_fake_fairscalec              	   C   sf   |    dd l}tjtjtjdddgdd}|j	j
|d tdtjtjd dd	d
}|  |S )Nr   BLIPzlhttps://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pthz.pthz$model_base_caption_capfilt_large.pth)
model_path	model_urlZ
ext_filterZdownload_namebaseconfigszmed_config.json)
pretrained
image_sizevitZ
med_config)rW   Zmodels.blipr
   Zload_modelsr!   r&   r'   r   models_pathblipZblip_decoderblip_image_eval_sizeeval)r8   modelsfiles
blip_modelr   r   r   load_blip_modelZ   s    *z!InterrogateModels.load_blip_modelc                 C   sZ   dd l }| jr(|jtdtjjd\}}n|jttjjd\}}|  |t	j
}||fS )Nr   r2   )r6   download_root)rh   )clipr7   loadclip_model_namer   cmd_optsclip_models_pathrc   tor   r5   )r8   ri   model
preprocessr   r   r   load_clip_modelj   s    z!InterrogateModels.load_clip_modelc                 C   s   | j d u r.|  | _ tjjs.| js.| j  | _ | j tj	| _ | j
d u rr|  \| _
| _tjjsr| jsr| j
 | _
| j
tj	| _
t| j
 j| _d S rN   )rf   rg   r   rl   no_halfr7   halfrn   r   r5   
clip_modelrq   clip_preprocessnext
parametersdtyperO   r   r   r   rj   w   s    


zInterrogateModels.loadc                 C   s&   t jjs"| jd ur"| jtj| _d S rN   )r   r@   !interrogate_keep_models_in_memoryrt   rn   r   r2   rO   r   r   r   send_clip_to_ram   s    
z"InterrogateModels.send_clip_to_ramc                 C   s&   t jjs"| jd ur"| jtj| _d S rN   )r   r@   ry   rf   rn   r   r2   rO   r   r   r   send_blip_to_ram   s    
z"InterrogateModels.send_blip_to_ramc                 C   s   |    |   t  d S rN   )rz   r{   r   torch_gcrO   r   r   r   unload   s    zInterrogateModels.unloadr:   c           	         s  dd l }t  tjjdkr0 dttjj  t|t }|j	t
 ddtj}| j|| j}||jddd }tdt ftj}t|jd D ](}|d|| d |j jdd7 }q||jd  }| j|dd\ fd	d
t|D S )Nr   T)truncatedimkeepdimr:   g      Y@)r   c                    s4   g | ],} d  |    d  |   d fqS )r   d   )numpy)r   i
text_arrayZ
top_labelsZ	top_probsr   r   r      r   z*InterrogateModels.rank.<locals>.<listcomp>)ri   r   r|   r   r@   interrogate_clip_dict_limitrE   minlentokenizelistrn   r5   rt   encode_texttyperx   normr#   zerosrangeshape	unsqueezeTsoftmaxr2   topk)	r8   image_featuresr   	top_countri   Ztext_tokenstext_features
similarityr   r   r   r   rank   s    &zInterrogateModels.rankc                 C   s   t t jttftjdt  t ddg|d	| j
tj}t 2 | jj|dtjjtjjtjjd}W d    n1 s0    Y  |d S )N)interpolation)g3<4'?gwgM?gy{ ?)gB91?gwt.?g	U?r   F)sample	num_beams
min_length
max_length)r   ComposeResizerb   r   BICUBICToTensor	Normalizer   r   rx   rn   r   r5   r#   no_gradrf   generater   r@   interrogate_clip_num_beamsinterrogate_clip_min_lengthinterrogate_clip_max_length)r8   	pil_imageZ	gpu_imagecaptionr   r   r   generate_caption   s     

@z"InterrogateModels.generate_captionc           
   
   C   s  d}t jjdd z8t  t  |   | |}| 	  t  |}| 
|d| jtj}t  t  | j|| j}||jddd }|  D ]Z}| j||j|jd}|D ]<\}}	t jjr|d	| d
|	d dd7 }q|d| 7 }qqW d    n1 s 0    Y  W d    n1 s@0    Y  W n* tyv   tjddd |d7 }Y n0 |   t j   |S )N interrogate)jobr   r   Tr   )r   z, (:r   z.3f)z, zError interrogating)exc_infoz<error>)!r   statebeginr	   Zsend_everything_to_cpur   r|   rj   r   r{   ru   r   r   rx   rn   r5   r#   r   autocastrt   encode_imager   rM   r   r   r   r@   interrogate_return_ranksr)   r   reportr}   end)
r8   r   resr   Z
clip_imager   catmatchesmatchscorer   r   r   r      s4    
 V
zInterrogateModels.interrogate)r:   )rQ   rR   rS   rf   rt   ru   rx   r7   r9   rM   rW   rg   rq   rj   rz   r{   r}   r   r   r   r   r   r   r   r1   -   s"   
r1   )r!   rU   collectionsr   pathlibr   rer#   Z	torch.hubtorchvisionr   Z!torchvision.transforms.functionalr   rV   r   r   r   r	   r
   r   rb   rk   r   compilerC   r   r0   r1   r   r   r   r   <module>   s     
