a
    Ad*                     @   s   d dl Z d dlZd dlmZmZ d dlm  mZ d dlmZ d dl	m
Z
mZ d dlmZ dddZd	d
 ZG dd dejZdd ZG dd dejZG dd dejZe G dd de
ZdS )    N)nnTensor)Optional)VQAutoEncoderResBlock)ARCH_REGISTRYh㈵>c                 C   s   |   }t|dksJ d|dd \}}| ||djdd| }| ||dd}| ||djdd||dd}||fS )zCalculate mean and std for adaptive_instance_normalization.

    Args:
        feat (Tensor): 4D tensor.
        eps (float): A small value added to the variance to avoid
            divide-by-zero. Default: 1e-5.
       z&The input feature should be 4D tensor.N   dim   )sizelenviewvarsqrtmean)featepsr   bcZfeat_varZfeat_stdZ	feat_mean r   J/var/www/html/stable-diffusion-webui/modules/codeformer/codeformer_arch.pycalc_mean_std   s    "r   c                 C   sP   |   }t|\}}t| \}}| || || }||| || S )a  Adaptive instance normalization.

    Adjust the reference features to have the similar color and illuminations
    as those in the degradate features.

    Args:
        content_feat (Tensor): The reference feature.
        style_feat (Tensor): The degradate features.
    )r   r   expand)Zcontent_featZ
style_featr   Z
style_meanZ	style_stdZcontent_meanZcontent_stdZnormalized_featr   r   r   adaptive_instance_normalization   s
    
r   c                       s,   e Zd ZdZd
 fdd	Zddd	Z  ZS )PositionEmbeddingSinez
    This is a more standard version of the position embedding, very similar to the one
    used by the Attention is all you need paper, generalized to work on images.
    @   '  FNc                    sP   t    || _|| _|| _|d ur4|du r4td|d u rFdtj }|| _d S )NFz+normalize should be True if scale is passedr
   )	super__init__num_pos_featstemperature	normalize
ValueErrormathpiscale)selfr#   r$   r%   r)   	__class__r   r   r"   4   s    

zPositionEmbeddingSine.__init__c              	   C   s  |d u r4t j|d|d|df|jt jd}| }|jdt jd}|jdt jd}| jrd}||d d dd d d f |  | j }||d d d d dd f |  | j }t j	| j
t j|jd	}| jd|d  | j
  }|d d d d d d d f | }|d d d d d d d f | }	t j|d d d d d d dd df  |d d d d d d dd df  fd
dd}t j|	d d d d d d dd df  |	d d d d d d dd df  fd
dd}	t j|	|fdddddd}
|
S )Nr   r
      )devicedtyper   )r/   gư>r   )r/   r.   r	   r   )torchzerosr   r.   boolcumsumfloat32r%   r)   aranger#   r$   stacksincosflattencatpermute)r*   xmaskZnot_masky_embedx_embedr   dim_tpos_xpos_yposr   r   r   forward?   s2    ,((  LLzPositionEmbeddingSine.forward)r   r    FN)N)__name__
__module____qualname____doc__r"   rD   __classcell__r   r   r+   r   r   .   s   r   c                 C   s>   | dkrt jS | dkrt jS | dkr*t jS td|  ddS )z,Return an activation function given a stringrelugelugluz$activation should be relu/gelu, not .N)FrJ   rK   rL   RuntimeError)
activationr   r   r   _get_activation_fnX   s    rQ   c                       sP   e Zd Zd fdd	Zee ddd	Zdee ee ee dddZ  ZS )TransformerSALayer              rK   c                    s   t    tj|||d| _t||| _t|| _t||| _	t
|| _t
|| _t|| _t|| _t|| _d S )N)dropout)r!   r"   r   MultiheadAttention	self_attnLinearlinear1DropoutrV   linear2	LayerNormnorm1norm2dropout1dropout2rQ   rP   )r*   	embed_dimnheaddim_mlprV   rP   r+   r   r   r"   d   s    
zTransformerSALayer.__init__)rC   c                 C   s   |d u r|S || S )Nr   )r*   tensorrC   r   r   r   with_pos_embeds   s    z!TransformerSALayer.with_pos_embedN)tgt_masktgt_key_padding_mask	query_posc              	   C   sx   |  |}| || }}| j|||||dd }|| | }| |}| | | | |}|| 	| }|S )N)value	attn_maskkey_padding_maskr   )
r^   rf   rX   r`   r_   r\   rV   rP   rZ   ra   )r*   tgtrg   rh   ri   Ztgt2qkr   r   r   rD   v   s    

zTransformerSALayer.forward)rS   rT   rU   rK   )NNN)	rE   rF   rG   r"   r   r   rf   rD   rI   r   r   r+   r   rR   c   s      rR   c                       s&   e Zd Z fddZdddZ  ZS )Fuse_sft_blockc              
      s   t    td| || _ttj||dddtddtj||ddd| _ttj||dddtddtj||ddd| _	d S )Nr
   r-   r   )kernel_sizepaddingg?T)
r!   r"   r   
encode_encr   
SequentialConv2d	LeakyReLUr)   shift)r*   in_chout_chr+   r   r   r"      s    


zFuse_sft_block.__init__r   c                 C   sH   |  tj||gdd}| |}| |}||| |  }|| }|S )Nr   r   )rs   r0   r:   r)   rw   )r*   Zenc_featZdec_featwr)   rw   residualoutr   r   r   rD      s    

zFuse_sft_block.forward)r   )rE   rF   rG   r"   rD   rI   r   r   r+   r   rp      s   rp   c                       s0   e Zd Zd fdd		Zd
d ZdddZ  ZS )
CodeFormer   rS   	         3264128256quantize	generatorc              	      sF  t tddg ddddg| |d urN|D ]}t| D ]
}	d|	_q@q.|_|_ _ d _	t
t|j_t
dj_t
j fd	d
tjD  _t
t
 t
j |dd_ddddddd_ddddddd_ddddddd_t
 _jD ] }
j|
 }t||j|
< q d S )Nr~   r   )r   r
   r
   r	   r	   rS   nearestr
      Fr   c                    s   g | ]}t  jd dqS )rU   )rb   rc   rd   rV   )rR   rd   ).0_dim_embdn_headr*   r   r   
<listcomp>   s   z'CodeFormer.__init__.<locals>.<listcomp>)bias   )16r   r   r   r   512   rS            )r   r   r   r   r   r      r            )r!   r}   r"   getattr
parametersrequires_gradconnect_listn_layersr   rd   r   	Parameterr0   r1   position_embrY   feat_embrt   range	ft_layersr]   idx_pred_layerchannelsfuse_encoder_blockfuse_generator_block
ModuleDictfuse_convs_dictrp   )r*   r   r   r   codebook_sizelatent_sizer   Zfix_modulesmoduleparamf_sizerx   r+   r   r   r"      s>    "






zCodeFormer.__init__c                 C   sr   t |tjtjfrH|jjjddd t |tjrn|jd urn|jj  n&t |tj	rn|jj  |jj
d d S )NrU   g{Gz?)r   stdg      ?)
isinstancer   rY   	Embeddingweightdatanormal_r   zero_r]   fill_)r*   r   r   r   r   _init_weights   s    zCodeFormer._init_weightsr   TFc                    s  i } fdd j D }t jjD ].\}}	|	|}||v r$| |t|jd < q$|}
 jd	d|jd d} 
|
dddd}|} jD ]}|||d}q |}|ddd}|r||
fS tj|dd}tj|ddd\}} jj||jd d	d	d
gd}|r| }|r,t||
}|} fdd j D }t jjD ]N\}}	|	|}||v rPt|jd }|dkrP j| ||  ||}qP|}|||
fS )Nc                    s   g | ]} j | qS r   )r   r   r   r*   r   r   r          z&CodeFormer.forward.<locals>.<listcomp>r   r   r   r
   )ri   r   r   r   )shapec                    s   g | ]} j | qS r   )r   r   r   r   r   r   
  r   )r   	enumerateencoderblocksclonestrr   r   	unsqueezerepeatr   r9   r;   r   r   rN   softmaxr0   topkr   Zget_codebook_featdetachr   r   r   )r*   r<   rz   Z	detach_16Z	code_onlyadainZenc_feat_dictout_listiblockZlq_featpos_embr   Z	query_emblayerlogitsZsoft_one_hotr   Ztop_idxZ
quant_featZ	fuse_listr   r|   r   r   r   rD      sB    




zCodeFormer.forward)r~   rS   r   r   r   r   r   )r   TFF)rE   rF   rG   r"   r   rD   rI   r   r   r+   r   r}      s       0	r}   )r   )r'   r0   r   r   Ztorch.nn.functional
functionalrN   typingr   Zmodules.codeformer.vqgan_archr   r   Zbasicsr.utils.registryr   r   r   Moduler   rQ   rR   rp   registerr}   r   r   r   r   <module>   s   
*%