a
    d                  
   @   s8  d Z ddlZddlZddlmZ ddlmZ z8ddlZddlmZmZm	Z	m
Z
 ddlmZmZmZ W nD ey Z z,dZG dd dZG dd	 d	Z
W Y dZ[n
dZ[0 0 d
dlmZ dd Zi Zdd ZeG dd dejZeG dd dejZeG dd dejZeG dd dejZG dd dejZdS )z huggingface model adapter

Wraps HuggingFace transformers (https://github.com/huggingface/transformers) models for use as a text tower in CLIP model.
    N)
TensorType)	AutoModelAutoTokenizer
AutoConfigPretrainedConfig)BaseModelOutputBaseModelOutputWithPooling,BaseModelOutputWithPoolingAndCrossAttentionsc                   @   s   e Zd ZdS )r   N__name__
__module____qualname__ r   r   [/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/open_clip/hf_model.pyr      s   r   c                   @   s   e Zd ZdS )r   Nr
   r   r   r   r   r      s   r      )	arch_dictc                 C   s   t dd|  S )Nz(?<!^)(?=[A-Z])_)resublower)sr   r   r   _camel2snake   s    r   c                 C   s   | t t| j< | S )z"Decorator registering pooler class)_POOLERSr   r   )clsr   r   r   register_pooler'   s    r   c                   @   s    e Zd ZdZeedddZdS )
MeanPoolerzMean poolingxattention_maskc                 C   s*   |j |d }|jdd|jddd S )Nr   )ZdimT)Zkeepdim)last_hidden_state	unsqueezesumselfr   r   Zmasked_outputr   r   r   forward1   s    zMeanPooler.forwardNr   r   r   __doc__r   r   r%   r   r   r   r   r   -   s   r   c                   @   s    e Zd ZdZeedddZdS )	MaxPoolerzMax poolingr   c                 C   s$   |j |dtj }|djS )Nr   r   )r    Zmasked_fillr!   torchinfmaxvaluesr#   r   r   r   r%   :   s    zMaxPooler.forwardNr&   r   r   r   r   r(   6   s   r(   c                       s2   e Zd ZdZd fdd	ZeedddZ  ZS )		ClsPoolerzCLS token poolingTc                    s   t    d| _|| _d S Nr   )super__init__cls_token_positionuse_pooler_output)r$   r2   	__class__r   r   r0   C   s    
zClsPooler.__init__r   c                 C   s>   | j r$t|ttfr$|jd ur$|jS |jd d | jd d f S N)r2   
isinstancer   r	   Zpooler_outputr    r1   r$   r   r   r   r   r   r%   H   s    zClsPooler.forward)T	r   r   r   r'   r0   r   r   r%   __classcell__r   r   r3   r   r-   ?   s   r-   c                       s0   e Zd ZdZ fddZeedddZ  ZS )ClsLastHiddenStatePoolerzcCLS token pooling
    NOTE: this is equivalent to ClsPooler above with use_pooler_output=False
    c                    s   t    d| _d S r.   )r/   r0   r1   r$   r3   r   r   r0   X   s    
z!ClsLastHiddenStatePooler.__init__r   c                 C   s   |j d d | jd d f S r5   )r    r1   r7   r   r   r   r%   \   s    z ClsLastHiddenStatePooler.forwardr8   r   r   r3   r   r:   R   s   r:   c                	       s   e Zd ZU dZejje ed< de	e
ee	e	eed fddZed	d
dZde
edddZejjdddZdd Z  ZS )HFTextEncoderzHuggingFace model adapteroutput_tokensNTF)model_name_or_path
output_dimconfigpooler_typeproj
pretrainedr=   c              	      s  t    || _|| _|dk}td u r.td|d u rt|| _|rPt	j|fn
t	j
| jf\}	}
t| jdr| jjr|	|
| _| jj| _q|	|
|d| _n|| _t	
|| _|d u rt| jj d }t| jdd| _t| jdd| _t|  | _t| jt| jj d	 d
 }||kr,|d u r,t | _n`|dkrJtj||dd| _nB|dkr|| d }ttj||ddt tj||dd| _d S )NZ
cls_poolerzGPlease `pip install transformers` to use pre-trained HuggingFace modelsis_encoder_decoder)Zadd_pooling_layerpooler
vocab_sizer   Zmax_position_embeddingsconfig_nameswidthZlinearF)ZbiasZmlp   )r/   r0   r=   r?   transformersRuntimeErrorr   Zfrom_pretrainedr@   r   Zfrom_confighasattrrD   transformerencoderr   
model_typegetattrrF   Zcontext_lengthr   rE   nnZIdentityrB   ZLinearZ
SequentialZGELU)r$   r>   r?   r@   rA   rB   rC   r=   Zuses_transformer_poolerZcreate_funcZ
model_argsZd_modelZhidden_sizer3   r   r   r0   d   sD    




zHFTextEncoder.__init__)r   c                 C   s   || j jk }| j||d}| ||}| |}|jjd }t| jt	krt|jd d t
|| jjkd d f n|j}| jr||fS |S )N)Z	input_idsr   r   )r@   Zpad_token_idlongrM   rE   rB   r    shapetyper-   r)   Zaranger1   r=   )r$   r   Z	attn_maskoutZ
pooled_outZ	projectedZseq_lentokensr   r   r   r%      s    
(zHFTextEncoder.forwardr   )unlocked_layersfreeze_layer_normc           
      C   s   |s6| j  D ]"\}}d|dv r*| nd|_qd S t| j drJ| j jn| j }t|t| jj	 d d }t
d| dt|d	  d
 t| j t| jj	 d d }|g|d |  }|D ]0}	|	 D ]"\}}d|dv r| nd|_qqd S )NZ	LayerNorm.FrN   rG   Z
layer_attrz
Unlocking /r   z layers of hf modelZtoken_embeddings_attr)rM   Znamed_parameterssplitZrequires_gradrL   rN   rP   r   r@   rO   printlen)
r$   rW   rX   nprN   Z
layer_listZ
embeddingsmodulesmoduler   r   r   lock   s    zHFTextEncoder.lockc                 C   s   | j   d S r5   )rM   Zgradient_checkpointing_enable)r$   enabler   r   r   set_grad_checkpointing   s    z$HFTextEncoder.set_grad_checkpointingc                 C   s   d S r5   r   r;   r   r   r   init_parameters   s    zHFTextEncoder.init_parameters)NNNTF)r   T)T)r   r   r   r'   r)   ZjitFinalbool__annotations__strintr   r0   r   r%   rb   ignorerd   re   r9   r   r   r3   r   r<   `   s*   
     6r<   )r'   r   r)   Ztorch.nnrQ   r   rJ   r   r   r   r   Ztransformers.modeling_outputsr   r   r	   ImportErroreZ
hf_configsr   r   r   r   Moduler   r(   r-   r:   r<   r   r   r   r   <module>   s2   $