a
    dL                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlm  mZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dl	mZ d dlmZmZ d d	lmZ d
dlmZ d
dlmZ d
dlm Z  e!e"Z#G dd deZ$e$Z%ej&j'dddZ(ej&j'dddZ)e*dddZ+ej&j'dddZ,dMej&j'dddZ-dd Z.dd  Z/G d!d" d"Z0G d#d$ d$Z1ej&j2d%d&d'Z3ej&j'ej&j'd(d)d*Z4ej&j'ej&j'd(d+d,Z5ej6ej6ej6ej6d-d.d/Z7ej&j'ej&j'd(d0d1Z8ej&j'ej&j'd(d2d3Z9ej6ej6ej6ej6d-d4d5Z:ej6ej6e;e;d6d7d8Z<ej=d9ej>j?e.e/d:d;Z@d<d= ZAd>d? ZBej=d@ej>j?eAeBd:d;ZCdAdB ZDG dCdD dDZEG dEdF dFejFjGZHej&jIdNdIdJZJej&jIdKdL ZKej
jjLeJejKeKiZMeJeKhZNdS )O    N)_prims)fake_mode_from_tensors)matches_module_patternreplace_node_module)ProxyTorchDispatchMode)	ShapeProp)
functional)fuse_conv_bn_evalfuse_conv_bn_weights)TorchFunctionMode   )config)matches_module_function_pattern)mkldnn_fuse_fxc                   @   s   e Zd ZdddZdS )AutogradMonkeypatch Nc                 C   sB   |si }|t v r4tjr"t | tv s4t | |i |S ||i |S N)replacementsr   fallback_random replacements_using_triton_random)selffunctypesargskwargsr   r   b/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_inductor/overrides.py__torch_function__   s    
z&AutogradMonkeypatch.__torch_function__)r   N)__name__
__module____qualname__r   r   r   r   r   r      s   r   )gmc              	   C   s   t t| jjD ]}|jdkr|jtv rtjr>t|j t	v r>q| j
|0 || jt|j |j|j W d    n1 s0    Y  | j| q| j  |   | S Ncall_function)reversedlistgraphnodesoptargetr   r   r   r   inserting_beforereplace_all_uses_withr"   r   r   
erase_nodelint	recompile)r    noder   r   r   
replace_fx,   s"    "
r/   c                 C   s   t dd |D }t|}t| } tjrV|sVt| |dj|  t| } t| } t	| } t
 rb| S |sj| S t| } t| } tjst| |} | S )Nc                 s   s*   | ]"}t |tjr|jtd kV  qdS )cpuN)
isinstancetorchTensordevice).0Zexample_inputr   r   r   	<genexpr>B   s   zfuse_fx.<locals>.<genexpr>)	fake_mode)allr   sink_cat_after_pointwiser   Zpermute_fusionr   	propagatelinear_permute_fusionpermute_linear_fusionpermute_matmul_fusionr2   Zis_grad_enabledremove_identityfuse_conv_bndynamo_configZdynamic_shapesr   )r    Zexample_inputsZis_cpur7   r   r   r   fuse_fxA   s&    

rA   )r(   c                 C   sT   |  d}|}t|D ]8\}}t||sDtdd|d |  t||}q|S )N.z#Node referenced nonexistant target )split	enumeratehasattrRuntimeErrorjoingetattr)r(   modZtarget_atomsZattr_itriZatomr   r   r   
fetch_attrd   s    

rK   c                 C   s    G dd dt jj}||  S )z6
    Removes all identity layers from the module.
    c                       s   e Zd Z fddZ  ZS )z(remove_identity.<locals>.IdentityRemoverc                    s>   t | j| tjr*t|dks"J |d S t |||S d S Nr   r   )r1   Z
submodulesnnZIdentitylensupercall_module)r   r(   r   r   	__class__r   r   rP   v   s    z4remove_identity.<locals>.IdentityRemover.call_module)r   r   r   rP   __classcell__r   r   rQ   r   IdentityRemoveru   s   rT   )r2   fxZTransformerZ	transform)r    rT   r   r   r   r>   p   s    r>   Fc              
   C   s  t jjt jjft jjt jjft jjt jjfg}t jjtj	ft jjtj	ft jjtj	fg}t
|  }|D ]}| jjD ]}t|||rrt|jd jdkrqr||jd j }||j }tdd ||fD }	|	sqr|jsqrt||}
t|jd ||
 ||jd  | j| qrqf| j  |D ]`}| jjD ]N}t|||r.t|jdkrVq.||jd j }|jd }|jd }|js.|rq.t|turq.tdd |jdd D }|sĐq.t|jd j| }t|jd	 j| }t|jd
 j| }t|jd j| }|du s.|du r$q.t |}
t!|
j"|
j#|||||\|
_"|
_#t|jd ||
 ||jd  | j| q.q | j  | $  | S )z=
    Fuses Convolution/BN layers for inference purposes.
    r   r   c                 s   s   | ]}|j  V  qd S r   )trainingr5   nr   r   r   r6          zfuse_conv_bn.<locals>.<genexpr>         c                 s   s&   | ]}|j d kot|jdkV  qdS )Zget_attrr   N)r'   rN   usersrW   r   r   r   r6      s            N)%r2   rM   ZConv1dZBatchNorm1dZConv2dZBatchNorm2dZConv3dZBatchNorm3dFZ
batch_normdictZnamed_modulesr%   r&   r   rN   r   r]   r(   r8   Ztrack_running_statsr	   r   r*   r+   r,   r   rV   typefloatrK   copydeepcopyr
   weightbiasr-   )r    inplaceZmodules_patternsZmodule_function_patternsmodulespatternr.   convZbnZ	eval_modeZ
fused_convZbn_trainingZbn_epsZbn_args_is_constantZbn_running_meanZbn_running_varZ	bn_weightZbn_biasr   r   r   r?      s    






	
r?   c                 C   s
   t | S r   )r   
TensorMetainputseedoffsetr   r   r   _philox_rand_like_meta   s    rr   c                 C   s
   t | S r   )r2   	rand_likern   r   r   r   _philox_rand_like   s    rt   c                   @   sV   e Zd ZejjddddZejjdddZejjddd	Zejjdd
dZ	dS )NormalizedLinearNodeNr.   returnc                 C   s.   |j dksJ |jtjjjfv s$J || _d S r!   )r'   r(   r2   rM   r   linearr.   r   r.   r   r   r   __init__   s    zNormalizedLinearNode.__init__rw   c                 C   s,   t | jjdkr| jjd S | jjd S d S Nr   ro   rN   r.   r   r   r   r   r   r   	get_input   s    zNormalizedLinearNode.get_inputc                 C   s,   t | jjdkr| jjd S | jjd S d S )Nr   rg   r}   r~   r   r   r   
get_weight   s    zNormalizedLinearNode.get_weightc                 C   s,   t | jjdkr| jjd S | jjd S d S )Nr^   rh   r}   r~   r   r   r   get_bias   s    zNormalizedLinearNode.get_bias)
r   r   r   r2   rU   Noderz   r   r   r   r   r   r   r   ru      s   ru   c                   @   sD   e Zd ZejjddddZejjdddZejjddd	ZdS )
NormalizedMatmulNodeNrv   c                 C   s.   |j dksJ |jtjtjfv s$J || _d S r!   )r'   r(   r2   bmmmatmulr.   ry   r   r   r   rz      s    zNormalizedMatmulNode.__init__r{   c                 C   s,   t | jjdkr| jjd S | jjd S d S r|   r}   r~   r   r   r   r      s    zNormalizedMatmulNode.get_inputc                 C   s,   t | jjdkr| jjd S | jjd S d S )Nr   otherr}   r~   r   r   r   	get_other   s    zNormalizedMatmulNode.get_other)	r   r   r   r2   rU   r   rz   r   r   r   r   r   r   r      s   r   )r.   c                    s   t  jd jt  jdkr> fddtdd D }nHd jv r jd d urt  jd dkrfdd jd D }nd	S tt}d |d
< d |d< ||kS )NZtensor_metar_   c                    s   g | ]} j |  qS r   r   r5   rJ   r.   ranksr   r   
<listcomp>  rY   z!check_permute.<locals>.<listcomp>r   permutationr^   c                    s   g | ]}|  qS r   r   r   )r   r   r   r   	  rY   F)rN   metashaper   ranger   r$   )r.   r   Zallowed_permutationr   r   r   check_permute   s     r   )modulerw   c              	      s   dd }dd }dd }| j   jD ]}|jdks$|jtjkr@q$|}||r`|sZq`}qDr$|r$ |T  fdd	|jd
 D }|f|jdd   |_|  	 W d    q$1 s0    Y  q$ 
  |   | S )Nc                 S   s"   t | j}t|dkr|d S d S rL   )r$   r]   rN   )r.   r]   r   r   r   one_user  s    
z*sink_cat_after_pointwise.<locals>.one_userc                 S   s   dh}| j dko| j|v S )Nviewcall_method)r'   r(   )r.   r   r   r   r   is_view  s    z)sink_cat_after_pointwise.<locals>.is_viewc                 S   s$   t jt jddh}| jdv o"| j|v S )Nrelutanh>   r   r"   )r2   r   r   r'   r(   )r.   Z	pointwiser   r   r   is_pointwise_unary  s    z4sink_cat_after_pointwise.<locals>.is_pointwise_unaryr"   c                    s&   g | ]} j jj|fjd qS ))r   r   )Zcreate_noder'   r(   r   )r5   argguserr   r   r   -  s   z,sink_cat_after_pointwise.<locals>.<listcomp>r   r   )r%   r&   r'   r(   r2   catr)   r   r*   r+   r,   r-   )r   r   r   r   r.   Zcat_or_viewZnew_tensorsr   r   r   r9     s.    

*r9   c              	   C   s  | j jD ]}|jdkr|jdkrt|rt|jdkrB|jd }n
|jd }|jdkr|jtj	j
jkrt|}| }| }| }| j |V | j jt|||fd}|| | j | t|jdkr| j | W d    q1 s0    Y  q| j   |   | S )Nr   permuter   ro   r"   r   )r%   r&   r'   r(   r   rN   r   r   r2   rM   r   rx   ru   r   r   r   r)   r"   linear_transposer*   r+   r]   r,   r-   r   r.   Z
input_node
normalizedro   rg   rh   
fused_noder   r   r   r;   9  s:    


,
r;   )ro   rg   rh   rw   c                 C   s   t || dd|d S Nr   r   )r2   r   	transposeZ	unsqueezero   rg   rh   r   r   r   r   ^  s    r   c              	   C   s2  | j jD ]}|jdkr|jtjjjkrt|j	dkrB|j	d }n
|j
d }|jdkr|jdkrt|rt|}t|j	dkr|j	d }n
|j
d }| }| }| j |X | j jt|||fd}|| | j | t|jdk r| j | W d    q1 s0    Y  q| j   |   | S )Nr"   r   ro   r   r   r   )r%   r&   r'   r(   r2   rM   r   rx   rN   r   r   r   ru   r   r   r)   r"   transpose_linearr*   r+   r]   r,   r-   r   r   r   r   r<   d  s8    



.
r<   c           
   	   C   s  | j jD ]z}|jdkr|jtjks0|jtjkrt|}| }|	 }|}|}d }}|jdkr|jdkrt
|rd}t|jdkr|jd }n
|jd }|jdkr|jdkrt
|rd}t|jdkr|jd }n
|jd }|s|r| j |( | j jt||||fd}	W d    n1 s$0    Y  ||	 | j | |rft|jdkrf| j | |rt|jdkr| j | q| j   |   | S )	Nr"   Fr   r   Tr   ro   r   )r%   r&   r'   r(   r2   r   r   r   r   r   r   rN   r   r   r)   r"   transpose_matmulr*   r+   r]   r,   r-   )
r   r.   r   Zinput_A_nodeZinput_B_nodeZinput_AZinput_BAtransBtransr   r   r   r   r=     s^    





&

r=   c                 C   s   t | dd| | S r   )r2   r   r   tr   r   r   r   r     s    r   ABr   r   c                 C   s,   |r|  dd} |r | dd}t| |S r   )r   r2   r   r   r   r   r   r     s
    r   zAphilox_rand_like(Tensor input, Tensor seed, int offset) -> Tensor )Zschemareturn_typer   Z	impl_atendocc                 C   s   t t| S r   )r   rm   _philox_seed_likexr   r   r   _philox_seed_like_meta  s    r   c                 C   s   t jtd| jt jdS )Nl        )r4   dtype)r2   Ztensorrandom	randranger4   int32r   r   r   r   r     s    r   z(philox_seed_like(Tensor other) -> Tensorc                   C   s   d S r   r   r   r   r   r   null_ref  s    r   c                   @   s2   e Zd ZdZi ZeZedddZedd Z	dS )PhiloxRandomStater   Nc                 C   s(   d| _ i | _|d urt|nt| _d S )Nr   )next_offsetrp   weakrefrefr   last_tracer_ref)clstracerr   r   r   reset  s    zPhiloxRandomState.resetc                 C   s   t jjj }dd |D }|rB|d j}|  |urJ| | n|   |j}|| j	vrht
|| j	|< | j	| }| j}|  j| 7  _||fS )Nc                 S   s   g | ]}t |tr|qS r   )r1   r   )r5   mr   r   r   r     rY   z5PhiloxRandomState.get_seed_offset.<locals>.<listcomp>r   )r2   rU   ZexperimentalZproxy_tensorZget_torch_dispatch_modesr   r   r   r4   rp   philox_seed_liker   Znumel)r   r   modesZproxy_modesr   r4   rp   rq   r   r   r   get_seed_offset  s    


z!PhiloxRandomState.get_seed_offset)N)
r   r   r   r   rp   r   r   classmethodr   r   r   r   r   r   r     s   r   c                   @   s$   e Zd Zedd Zedd ZdS )LowmemDropoutc                 C   sX   || _ tdd|  }t|\}}| | || _t||||k}||j| | S Ng      ?)	prd   r   r   Zsave_for_backwardrq   philox_rand_liketor   )ctxr   r   scalerp   rq   	bool_maskr   r   r   forward  s    
zLowmemDropout.forwardc                 C   sH   | j }tdd|  }| j\}t||| j|k}||j| | d fS r   )r   rd   Zsaved_tensorsr   rq   r   r   )r   Zgrad_outputr   r   rp   r   r   r   r   backward  s
    zLowmemDropout.backwardN)r   r   r   staticmethodr   r   r   r   r   r   r     s   
	r         ?Tc                 C   sT   t | tjjr&| jdt| ||fi S |r2|dkr6| S t| |}|rP| 	| |S )Nr"   r   )
r1   r2   rU   Proxyr   create_proxylowmem_dropoutr   applyZcopy_)ro   r   rV   ri   resultr   r   r   r   "  s    
r   c                 K   sb   t | tjjr | jdt| |S |d| j| jks8J t	
| \}}t| |||dtjS )Nr"   r4   r   )r1   r2   rU   r   r   r   rs   getr4   r   r   r   r   float32)r   r   rp   rq   r   r   r   rs   4  s
    rs   )F)r   TF)Ore   loggingr   r   r2   Ztorch._dynamo.configZ_dynamor   r@   Ztorch.nnrM   r   Ztorch._dynamo.utilsr   Z"torch.fx.experimental.optimizationr   r   Z"torch.fx.experimental.proxy_tensorr   Ztorch.fx.passes.shape_propr   r   ra   Ztorch.nn.utils.fusionr	   r
   Ztorch.overridesr   r   Zfx_utilsr   Zmkldnnr   	getLoggerr   logr   Zpatch_functionsrU   ZGraphModuler/   rA   strrK   r>   r?   rr   rt   ru   r   r   r   r9   r;   r3   r   r<   r=   r   boolr   Z
_make_primZRETURN_TYPEZNEWr   r   r   r   r   r   ZautogradZFunctionr   wrapr   rs   Zdropoutr   r   r   r   r   r   <module>   s   
#K'&!8		$
	