a
    d`                     @   s   d dl mZ d dlZd dlmZ d dlmZmZ d dlmZ dd ZG dd	 d	ejZ	ej
jG d
d deZG dd dejZdS )    )OptionalN)nn)r   Tensor)_get_activation_fnc                 C   s   t | dr>t | dr>t | _| `| j}| j}t||d| _nt | dr|t | dr|t | _| `| j}| j}t||d| _nHd| 	 v sd| 	 v r| `
| j}| j}t||d| _
ntd td t | d	rd
| _| S )Nglobal_poolfc)num_classesinitial_num_features
classifierZRegNetZTResNetz;Model code-writing is not aligned currently with ml-decoder	drop_rater   )hasattrr   ZIdentityr   r   r   num_features	MLDecoderr
   Z	_get_nameheadprintexitr   )modelr   r    r   _/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/layers/ml_decoder.pyadd_ml_decoder_head	   s,    


r   c                	       s\   e Zd Zddd fdd	Z fd
dZdeeee ee ee ee edddZ  ZS )TransformerDecoderLayerOptimal      皙?reluh㈵>N)returnc                    s   t t|   tj||d| _t|| _t|| _t|| _	t|| _
tj|||d| _t||| _t||| _tj||d| _tj||d| _t|| _d S )N)Zeps)dropout)superr   __init__r   Z	LayerNormnorm1ZDropoutr   dropout1dropout2dropout3ZMultiheadAttentionmultihead_attnLinearlinear1linear2norm2norm3r   
activation)selfd_modelZnheaddim_feedforwardr   r+   Zlayer_norm_eps	__class__r   r   r    $   s    z'TransformerDecoderLayerOptimal.__init__c                    s*   d|vrt jjj|d< tt| | d S )Nr+   )torchr   
functionalr   r   r   __setstate__)r,   stater/   r   r   r3   8   s    z+TransformerDecoderLayerOptimal.__setstate__)tgtmemorytgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskr   c              	   C   sz   ||  | }| |}| |||d }|| | }| |}| | | | |}|| 	| }| 
|}|S )Nr   )r"   r!   r%   r#   r)   r(   r   r+   r'   r$   r*   )r,   r5   r6   r7   r8   r9   r:   Ztgt2r   r   r   forward=   s    


z&TransformerDecoderLayerOptimal.forward)r   r   r   r   r   )NNNN)	__name__
__module____qualname__r    r3   r   r   r;   __classcell__r   r   r/   r   r   #   s         r   c                   @   s2   e Zd ZedddZejejejdddZdS )GroupFCembed_len_decoderc                 C   s
   || _ d S NrA   )r,   rB   r   r   r   r    ]   s    zGroupFC.__init__)hduplicate_pooling
out_extrapc                 C   s^   t | jD ]N}|d d |d d f }||d d d d f }t|||d d |d d f< q
d S rC   )rangerB   r1   matmul)r,   rD   rE   rF   iZh_iZw_ir   r   r   __call__`   s    zGroupFC.__call__N)r<   r=   r>   intr    r1   r   rJ   r   r   r   r   r@   [   s   r@   c                       s&   e Zd Zd fdd	Zdd Z  ZS )	r   r      r   c           
         s  t t|   |dk rdn|}||kr*|}|dk r6dn|}t||| _d}d}d}t|||d}	tj|	|d| _t	||| _
| j
d	 || _t|| d
 | _tjt||| j| _tjt|| _tjj| j tjj| jd t|| _d S )Nr   d   rL   r      r   )r-   r.   r   )Z
num_layersFg+?)r   r   r    r   r&   embed_standartr   ZTransformerDecoderdecoderZ	Embeddingquery_embedZrequires_grad_r   rK   duplicate_factorr1   	Parameterr   rE   duplicate_pooling_biasinitZxavier_normal_Z	constant_r@   group_fc)
r,   r   Znum_of_groupsZdecoder_embeddingr	   rB   Zdecoder_dropoutZnum_layers_decoderr.   Zlayer_decoder/   r   r   r    h   s0    zMLDecoder.__init__c                 C   s   t |jdkr"|ddd}n|}| |}tjjj|dd}|jd }| j	j
}|dd|d}| ||dd}|dd}tj|jd |jd | j|j|jd}| || j| |dd d d | jf }	|	| j7 }	|	}
|
S )	N      rN   T)Zinplacer   r   )devicedtype)lenshapeflattenZ	transposerO   r1   r   r2   r   rQ   ZweightZ	unsqueezeexpandrP   zerosrR   rY   rZ   rV   rE   r   rT   )r,   xZembedding_spatialZembedding_spatial_786bsrQ   r5   rD   rF   Zh_outZlogitsr   r   r   r;      s     

&
zMLDecoder.forward)r   rL   r   )r<   r=   r>   r    r;   r?   r   r   r/   r   r   g   s    r   )typingr   r1   r   r   Ztorch.nn.modules.transformerr   r   Moduler   Zjitscriptobjectr@   r   r   r   r   r   <module>   s   8