a
    dH                     @   s  d Z ddlmZ ddlZddlZddlmZmZmZ ddl	Z
ddlZddlm  mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlm Z  eG dd dZ!eG dd dZ"e#dddZ$e#dddZ%d2e&e!e'eej( dddZ)d3e&e"e'eej( dddZ*G dd dej+Z,G dd  d ej+Z-ej.fej+d!d"d#Z/e/Z0e1d$d%d&Z2d'ej.fe1d$d(d)Z3d*e4d+fd,d-Z5d4e#e'd/d0d1Z6dS )5zm CLIP Model

Adapted from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI.
    )	dataclassN)OptionalTupleUnion)nn)
checkpoint   )HFTextEncoder)ModifiedResNet)	TimmModel)LayerNormFp32	LayerNorm	QuickGELU	AttentionVisionTransformerTextTransformer)	to_2tuplec                   @   s6  e Zd ZU dZeeeeeef ef ed< dZeed< dZ	eed< dZ
eed< d	Zeed
< dZeeeef ef ed< dZee ed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed < dZeed!< dZee ed"< dS )#CLIPVisionCfg   layersi   width@   
head_widthg      @	mlp_ratio   
patch_size   
image_sizeNls_init_valueg        patch_dropoutFinput_patchnormglobal_average_poolattentional_pool   	n_queries   attn_pooler_headsoutput_tokenstimm_model_nametimm_model_pretrainedavg	timm_poolZlinear	timm_projtimm_proj_bias	timm_droptimm_drop_path) __name__
__module____qualname__r   r   r   int__annotations__r   r   r   floatr   r   r   r   r   r    boolr!   r"   r$   r&   r'   r(   strr)   r+   r,   r-   r.   r/    r8   r8   X/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/open_clip/model.pyr      s*   
 r   c                   @   s   e Zd ZU dZeed< dZeed< dZeed< dZeed< d	Z	eed
< dZ
ee ed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dS )CLIPTextCfgM   context_lengthi   
vocab_sizei   r   r%   headsr   r   Nr   hf_model_namehf_tokenizer_nameThf_model_pretrainedZmlpprojZmean_poolerpooler_typeF	embed_clsr   pad_idr'   )r0   r1   r2   r<   r3   r4   r=   r   r>   r   r   r   r5   r?   r7   r@   rA   r6   rB   rC   rD   rE   r'   r8   r8   r8   r9   r:   2   s   
r:   )	precisionc                 C   s&   d }| dkrt j}n| dkr"t j}|S )Nbf16fp16torchbfloat16float16)rF   
cast_dtyper8   r8   r9   get_cast_dtypeD   s    rN   c                 C   s&   d }| dv rt j}n| dv r"t j}|S )N)rG   Z	pure_bf16)rH   Z	pure_fp16rI   )rF   Zinput_dtyper8   r8   r9   get_input_dtypeM   s    rO   F)	embed_dim
vision_cfg
quick_gelurM   c                 C   s  t |trtf i |}|r tntj}|jrjt|j|j|j	|j
|j|j|j|jdkrZ|jnd | |jd
}nt |jttfr|jd |j }t|j| ||j|jd}nh|j|j }|tjtjfv rtnt}t|j|j|j|j||j|j|j|j|j |j!|j"|j#|j$| ||d}|S )Nr   )	
pretrainedpoolrB   Z	proj_biasZdropZ	drop_pathZ
patch_droprP   r       )r   
output_dimr>   r   r   )r   r   r   r   r>   r   r   r   r    r!   r"   r$   r&   r'   rV   	act_layer
norm_layer)%
isinstancedictr   r   r   GELUr(   r   r)   r+   r,   r-   r.   r/   r   r   r   tuplelistr   r   r
   rJ   rL   rK   r   r   r   r   r   r   r    r!   r"   r$   r&   r'   )rP   rQ   rR   rM   rW   visualZvision_headsrX   r8   r8   r9   _build_vision_towerV   s^    
r_   )rP   text_cfgrR   rM   c                 C   s   t |trtf i |}|jr>t|j| |j|j|j|jd}nX|rFt	nt
j}|tjtjfv r`tnt}t|j|j|j|j|j|j| |j|j|j||d}|S )N)rV   rB   rC   rS   r'   )r<   r=   r   r>   r   r   rV   rD   r'   rE   rW   rX   )rY   rZ   r:   r?   r	   rB   rC   rA   r'   r   r   r[   rJ   rL   rK   r   r   r   r<   r=   r   r>   r   r   rD   rE   )rP   r`   rR   rM   textrW   rX   r8   r8   r9   _build_text_tower   s8    
	rb   c                       s   e Zd ZU ejje ed< dee	e
eeej ed fddZddd	ZejjdddZdedddZdedddZdeej eej dddZ  ZS )CLIPoutput_dictFNrP   rQ   r`   rR   rM   rd   c                    s   t    || _t||||| _t||||}|j| _|j| _|j| _|j	| _	|j
| _
|j| _|j| _| jd|jdd ttg td | _d S )N	attn_maskF)
persistent$I$I,@)super__init__rd   r_   r^   rb   transformerr<   r=   token_embeddingpositional_embeddingln_finaltext_projectionZregister_bufferrf   r   	ParameterrJ   onesnploglogit_scale)selfrP   rQ   r`   rR   rM   rd   ra   	__class__r8   r9   rj      s    	
zCLIP.__init__r   c                 C   s   | j j||d d S N)unlocked_groupsfreeze_bn_statsr^   lockru   ry   rz   r8   r8   r9   lock_image_tower   s    zCLIP.lock_image_towerTc                 C   s   | j | || j_d S N)r^   set_grad_checkpointingrk   Zgrad_checkpointingru   enabler8   r8   r9   r      s    zCLIP.set_grad_checkpointing	normalizec                 C   s    |  |}|rtj|ddS |S NZdimr^   Fr   ru   imager   featuresr8   r8   r9   encode_image   s    
zCLIP.encode_imagec                 C   s   | j  }| ||}|| j| }|ddd}| j || jd}|ddd}| |}|t	|j
d |jddf | j }|rtj|ddS |S )Nr   r      )rf   r   r   )rk   rN   rl   torm   permuterf   rn   rJ   ZarangeshapeZargmaxro   r   r   )ru   ra   r   rM   xr8   r8   r9   encode_text   s    

&zCLIP.encode_textr   ra   c                 C   s\   |d ur| j |ddnd }|d ur0| j|ddnd }| jrL||| j dS ||| j fS NTr   )image_featurestext_featuresrt   r   r   rd   rt   expru   r   ra   r   r   r8   r8   r9   forward   s    zCLIP.forward)FNF)r   F)T)F)F)NN)r0   r1   r2   rJ   jitFinalr6   r4   r3   r   r:   r   dtyperj   r~   ignorer   r   r   Tensorr   __classcell__r8   r8   rv   r9   rc      s.   
   
  rc   c                       s   e Zd ZU ejje ed< dee	e
eeej ed fddZddd	ZdeedddZejjdddZdedddZdedddZdeej eej dddZ  ZS )CustomTextCLIPrd   FNre   c                    sd   t    || _t||||| _t||||| _| jj| _| jj| _t	
tg td | _d S )Nrh   )ri   rj   rd   r_   r^   rb   ra   r<   r=   r   rp   rJ   rq   rr   rs   rt   )ru   rP   rQ   r`   rR   rM   rd   rv   r8   r9   rj     s    	


zCustomTextCLIP.__init__r   c                 C   s   | j j||d d S rx   r{   r}   r8   r8   r9   r~     s    zCustomTextCLIP.lock_image_towerT)unlocked_layersfreeze_layer_normc                 C   s   | j || d S r   )ra   r|   )ru   r   r   r8   r8   r9   lock_text_tower  s    zCustomTextCLIP.lock_text_towerc                 C   s   | j | | j| d S r   )r^   r   ra   r   r8   r8   r9   r     s    z%CustomTextCLIP.set_grad_checkpointingr   c                 C   s    |  |}|rtj|ddS |S r   r   r   r8   r8   r9   r   "  s    
zCustomTextCLIP.encode_imagec                 C   s    |  |}|rtj|ddS |S r   )ra   r   r   )ru   ra   r   r   r8   r8   r9   r   &  s    
zCustomTextCLIP.encode_textr   c                 C   s\   |d ur| j |ddnd }|d ur0| j|ddnd }| jrL||| j dS ||| j fS r   r   r   r8   r8   r9   r   *  s    zCustomTextCLIP.forward)FNF)r   F)r   T)T)F)F)NN)r0   r1   r2   rJ   r   r   r6   r4   r3   r   r:   r   r   rj   r~   r   r   r   r   r   r   r   r   r8   r8   rv   r9   r     s0   
   
  r   )modelc                    s    fdd}|  | dS )zCConvert applicable model parameters to low-precision (bf16 or fp16)c                    s   t | tjtjtjfrD| jj | j_| jd urD| jj | j_t | tj	t
frg dd dD dddD ]$}t| |}|d urr|j |_qrt | ttfrt| dd }|d ur|j |_t | trt| dd }|d ur|j |_d S )	Nc                 S   s   g | ]}| d qS )Z_proj_weightr8   ).0sr8   r8   r9   
<listcomp>D      zCconvert_weights_to_lp.<locals>._convert_weights.<locals>.<listcomp>)inqkvZin_proj_biasZbias_kZbias_vro   rB   )rY   r   ZConv1dZConv2dZLinearZweightdatar   ZbiasZMultiheadAttentionr   getattrrc   r   r   )lattrZtensorr   r8   r9   _convert_weights=  s"    
"

z/convert_weights_to_lp.<locals>._convert_weightsN)apply)r   r   r   r8   r   r9   convert_weights_to_lp:  s    r   
state_dictc                    sL   d| v rHi }|   D ].\ }t fdddD r:d   || < q|S | S )Nro   c                 3   s   | ]}  |V  qd S r   )
startswith)r   pr   r8   r9   	<genexpr>a  r   z4convert_to_custom_text_state_dict.<locals>.<genexpr>)ro   rm   rl   rk   rn   ztext.)itemsany)r   Znew_state_dictr   r8   r   r9   !convert_to_custom_text_state_dict\  s    
r   Tc                    s  d v }|rb d j d }tdd   D } d j d }t d j d d d	 }|| }nl fd
ddD }	t|	} d j d }t d j d d d	 }
d }|
d d  d j d ksJ |
d } d j d } d j d } d j d } d j d }|d }ttdd  D }t||||d}t|||||d}t|||||d}dD ]} 	|d  q\t
| |  | S )Nzvisual.projzvisual.conv1.weightr   c                 S   s$   g | ]}| d r|dr|qS )zvisual.z.attn.in_proj_weight)r   endswithr   r   r8   r8   r9   r   x  r   z6build_model_from_openai_state_dict.<locals>.<listcomp>r   visual.positional_embeddingr   g      ?c                    s&   g | ] t t fd dD qS )c                 3   s,   | ]$}| d   r|dd V  qdS )zvisual.layer.r   Nr   splitr   br8   r9   r   ~  r   z@build_model_from_openai_state_dict.<locals>.<listcomp>.<genexpr>)lenset)r   r   r   r9   r   }  s   )r   r         zvisual.layer1.0.conv1.weightz$visual.attnpool.positional_embeddingr   rU   ro   rm   ztoken_embedding.weightzln_final.weightr   c                 s   s&   | ]}| d r|dd V  qdS )ztransformer.resblocksr   r   Nr   r   r8   r8   r9   r     r   z5build_model_from_openai_state_dict.<locals>.<genexpr>)r   r   r   r   )r<   r=   r   r>   r   )rQ   r`   rR   rM   )Zinput_resolutionr<   r=   )r   r   keysroundr\   r   r   r:   rc   popconvert_weights_to_fp16Zload_state_dicteval)r   rR   rM   ZvitZvision_widthZvision_layersZvision_patch_size	grid_sizer   countsZoutput_widthrP   r<   r=   Ztransformer_widthZtransformer_headsZtransformer_layersrQ   r`   r   keyr8   r   r9   "build_model_from_openai_state_dictn  sb    


r   r#   cpuc                 C   sl   |    | jj}tj|d||f|d}tj|| jftj|d}tjj	| t
||f|f|fdd} || j_| S )Nr   )device)r   r   )r   r   r   )inputs)r   r^   r   rJ   rq   zerosr<   r3   r   Ztrace_modulerZ   )r   Z
batch_sizer   r   Zexample_imagesZexample_textr8   r8   r9   trace_model  s    r   bicubic)interpolation	antialiasc                 C   s8  |  dd }|d u s t|jds$d S t|jj}d}|d |d  | }||jd krZd S |rz|d | ||d   }}	n
d | }}	tttt	|	}
t
d|
| |	d|
d |
d ddddd}	tj|	|||d	d
}	|	ddddd|d |d  dd }	|d ur(tj||	gdd}n|	}|| d< d S )Nr   r   r   r   z3Resizing position embedding grid-size from %s to %sr   r   r   F)sizemoder   Zalign_cornersr   )gethasattrr^   r   r   r   r3   mathsqrtr   logginginfoZreshaper   r   ZinterpolaterJ   cat)r   r   r   r   Zold_pos_embedr   Zextra_tokensZnew_seq_lenZpos_emb_tokZpos_emb_imgZold_grid_sizeZnew_pos_embedr8   r8   r9   resize_pos_embed  s4    
$*
r   )FN)FN)r   T)7__doc__Zdataclassesr   r   r   typingr   r   r   numpyrr   rJ   Ztorch.nn.functionalr   Z
functionalr   Ztorch.utils.checkpointr   Zhf_modelr	   Zmodified_resnetr
   Z
timm_modelr   rk   r   r   r   r   r   r   utilsr   r   r:   r7   rN   rO   r3   r6   r   r_   rb   Modulerc   r   rL   r   r   rZ   r   r   r   r   r   r8   r8   r8   r9   <module>   s`    	  A  'G8<