a
    d
n                     @   s  d dl Z d dlZd dlZd dlmZ d dlmZ d dlZd dlm	Z	 d dl
m	  mZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d	d
lmZ d	dlmZ G dd dZdd Zdd Zdd Zdd Z dd Z!G dd de	j"Z#G dd de	j"Z$G dd de	j%Z&G dd de	j%Z'G d d! d!e	j%Z(G d"d# d#e	j)Z*e	j+e,d$d%d&Z-e	j+e,d'd(d)Z.e	j+e	j+e,d*d+d,Z/e	j+e0e,d-d.d/Z1e	j+e	j+e,d0d1d2Z2e	j+e,d3d4d5Z3e	j+e	j+e,d6d7d8Z4e	j+e0e,d9d:d;Z5e	j+e	j+e,d<d=d>Z6ej7j8d?d@dAZ9ej7j:dBdCdDZ;ej7j8d?dEdFZ<dGdH Z=dIdJ Z>ej7j8d?dKdLZ?ej7j8d?dMdNZ@ej7j8d?dOdPZAe	j"e/e	j%e4e$e2e	j)e6iZBe	jCedQe	jDedRe	jEedSe	jFedTe	jGedUdVgdWe	jHedXdYdZgdWe	jIed[d\d]e	jJedXdYdZgdWe	jKed^e	jLed_i
ZMe	jCe	jDe	jEe	jFe	jGe	jHe	jIe	jJe	jKe	jLejNejOejPejQejRejSejTejUejVejWejNejOejPdQdRdSgZXejYd`d`d`dadbejYd`ejZdbej[dcdcdcdddcej[dcej\dci
Z]e	j"e1e	j%e5iZ^e	j%e3e	j"e-e	j)e.iZ_d d	ded	d degd d	degd d	degdfZ`dS )g    N)reduce)Optional)fake_mode_from_tensors)matches_module_patternreplace_node_module	guard_int)	ShapeProp)_pair   )config)matches_module_function_patternc                       s4   e Zd Zded fddZejdddZ  ZS )		UnaryAttrN)op_namec                    s0   || _ |r|ng | _|r|nd| _t   d S )N )r   scalars_attralgorithm_attrsuper__init__)selfr   r   r   	__class__ _/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_inductor/mkldnn.pyr      s    zUnaryAttr.__init__unary_modulec                    s~   t  tju rtjddd t fdd| jD s8J  fdd| jD }d}| jrrt | jsfJ t | j}| j	||fS )	Nr      )min_valmax_valc                 3   s   | ]}t  |V  qd S N)hasattr.0itemr   r   r   	<genexpr>"       z%UnaryAttr.__call__.<locals>.<genexpr>c                    s   g | ]}t  |qS r   )getattrr!   r   r   r   
<listcomp>#   r%   z&UnaryAttr.__call__.<locals>.<listcomp>r   )
typennReLU6Hardtanhallr   r   r    r&   r   )r   r   scalars	algorithmr   r   r   __call__   s    zUnaryAttr.__call__)NN)	__name__
__module____qualname__strr   r)   Moduler/   __classcell__r   r   r   r   r      s   r   c                 C   s.   | j jtjk}| jd u p$| jjtjk}|o,|S r   )weightdtypetorchZbfloat16bias)mZweight_is_bf16Zbias_is_bf16r   r   r   is_bfloat16_module-   s    r;   c                 C   s&   t | tjfv o$| jdko$| j| jkS )Nr   )r(   r)   ConvTranspose2dgroupsin_channels)r:   r   r   r   !is_group_depthwise_conv_transpose3   s    $r?   c                 C   sX   t | tjjsdS | jdkr dS t | jts0dS | j|vr>dS t|| j |urTdS dS )NFcall_moduleT)
isinstancer8   fxNodeoptargetr3   r(   )Zcurrent_nodemodules	node_kindr   r   r   check_node_kind9   s    

rH   c                 C   sT   | j dkr| jtjtjfv pR| j dkr@| jtjtjtjtjfv pR| j dkoR| jdv S )Ncall_functioncall_method)addadd_subsub_)rD   rE   r8   rK   rM   operatoriaddisubnoder   r   r   check_node_is_binaryG   s    
rT   c                 C   s\   t | jdkrdS t | jdkrXd| jv r<| jd d ur<dS d| jv rX| jd dkrXdS dS )N   Fr   outalphag      ?T)lenargskwargsrR   r   r   r   !check_binary_op_kwargs_is_defaultS   s    r[   c                       sF   e Zd Zejeej ed fddZdd Zdd Z	dd	 Z
  ZS )
ConvUnary2dconvunary
input_sizec                    sP   t  |j|j|j|j|j|j|j|j	d u|j
|jj|jj | ||| d S r   r   r   r>   out_channelskernel_sizestridepaddingdilationr=   r9   padding_moder6   devicer7   _update_module_paramsr   r^   r_   r`   r   r   r   r   a   s    zConvUnary2d.__init__c                 C   s   t |j| _d| _g | _d| _|d urBt|j |\| _| _| _tj	j
tjj| j | j| j| j| jtdd |D | jjd| _d S )Nnoner   c                 s   s   | ]}t |V  qd S r   r   r"   xr   r   r   r$      r%   z4ConvUnary2d._update_module_params.<locals>.<genexpr>requires_grad)copydeepcopy__dict__attrr-   r.   unary_modules_mapr   r8   r)   	Parameter_C_nnmkldnn_reorder_conv2d_weightr6   	to_mkldnnre   rd   rf   r=   tuplero   rj   r   r   r   ri   v   s*    z!ConvUnary2d._update_module_paramsc                 C   sx   | j dkrJtjjtj|| j| j d||td| j	| j
| j| j| j| j
S tjj|||| j| j	| j
| j| j| j| j
S Nzeros)moder   )rg   r8   opsmkldnn_convolution_pointwiseFpad _reversed_padding_repeated_twicer
   rd   rf   r=   rs   r-   r.   re   r   inputr6   r9   r   r   r   _conv_forward   s6    

zConvUnary2d._conv_forwardc                 C   s   |  || j| jS r   r   r6   r9   r   r   r   r   r   forward   s    zConvUnary2d.forward)r0   r1   r2   r)   r4   r   listr   ri   r   r   r5   r   r   r   r   r\   `   s   r\   c                       sH   e Zd Zejeed fddZdd Zdd Z	dd	 Z
d
d Z  ZS )ConvBinary2dr^   binary_op_namer`   c                    sP   t  |j|j|j|j|j|j|j|j	d u|j
|jj|jj | ||| d S r   ra   r   r^   r   r`   r   r   r   r      s    zConvBinary2d.__init__c                 C   sv   t |j| _|| _d | _d | _g | _d | _tj	j
tjj| j | j| j| j| jtdd |D | jjd| _d S )Nc                 s   s   | ]}t |V  qd S r   r   rl   r   r   r   r$      r%   z5ConvBinary2d._update_module_params.<locals>.<genexpr>rn   )rp   rq   rr   binary_attrbinary_alpha
unary_attrunary_scalarsunary_algorithmr8   r)   ru   rv   rw   rx   r6   ry   re   rd   rf   r=   rz   ro   r   r   r   r   ri      s"    z"ConvBinary2d._update_module_paramsc                 C   s   t |j |\| _| _| _d S r   )rt   r   r   r   r   )r   r_   r   r   r   _update_unary_params   s
    z!ConvBinary2d._update_unary_paramsc                 C   s   | j dkrTtjjtj|| j| j d|||td| j	| j
| j| j| j| j| j| jS tjj||||| j| j	| j
| j| j| j| j| j| jS r{   )rg   r8   r~   r   r   r   r   r   r
   rd   rf   r=   r   r   r   r   r   re   )r   r   otherr6   r9   r   r   r   r      sB    

zConvBinary2d._conv_forwardc                 C   s   |  ||| j| jS r   r   )r   r   r   r   r   r   r      s    zConvBinary2d.forward)r0   r1   r2   r)   r4   r3   r   r   ri   r   r   r   r5   r   r   r   r   r      s   #r   c                       s6   e Zd Zejed fddZdd Zdd Z  Z	S )PackedLinearlinearr`   c                    s6   t  |j|j|jd u|jj|jj | || d S r   	r   r   Zin_featuresZout_featuresr9   r6   rh   r7   ri   r   r   r`   r   r   r   r     s    zPackedLinear.__init__c                 C   sT   t |j| _tdd |d d | _tjjtjj	
| j | j| jjd| _d S )Nc                 S   s   | | S r   r   )rm   yr   r   r   <lambda>  r%   z4PackedLinear._update_module_params.<locals>.<lambda>rn   )rp   rq   rr   r   
batch_sizer8   r)   ru   r~   mklZ_mkl_reorder_linear_weightr6   ry   ro   packed_weightr   r   r   r   ri     s    z"PackedLinear._update_module_paramsc                 C   s"   t jj|| j| j| j| j}|S r   )r8   r~   r   Z_mkl_linearr   r6   r9   r   r   r   r   r   r   r   r     s    zPackedLinear.forward)
r0   r1   r2   r)   r4   r   r   ri   r   r5   r   r   r   r   r     s   

r   c                       s8   e Zd Zejejd fddZdd Zdd Z  ZS )LinearUnary)r   r_   c                    s6   t  |j|j|jd u|jj|jj | || d S r   r   r   r   r_   r   r   r   r     s    zLinearUnary.__init__c                 C   s,   t |j| _t|j |\| _| _| _d S r   )rp   rq   rr   rt   r   rs   r-   r.   r   r   r   r   ri   ,  s    z!LinearUnary._update_module_paramsc                 C   s&   t jj|| j| j| j| j| j}|S r   )	r8   r~   r   _linear_pointwiser6   r9   rs   r-   r.   r   r   r   r   r   2  s    zLinearUnary.forward)	r0   r1   r2   r)   r4   r   ri   r   r5   r   r   r   r   r     s
   r   c                       s6   e Zd Zejed fddZdd Zdd Z  Z	S )LinearBinary)r   r   c                    s6   t  |j|j|jd u|jj|jj | || d S r   r   r   r   r   r   r   r   r   :  s    zLinearBinary.__init__c                 C   s   t |j| _|| _d S r   )rp   rq   rr   rs   r   r   r   r   ri   D  s    z"LinearBinary._update_module_paramsc                 C   s    t jj||| j| j| j}|S r   )r8   r~   r   r   r6   r9   rs   )r   r   r   r   r   r   r   r   I  s    zLinearBinary.forward)
r0   r1   r2   r)   r4   r3   r   ri   r   r5   r   r   r   r   r   9  s   
r   c                       sF   e Zd Zejeej ed fddZdd Zdd Z	dd	 Z
  ZS )
ConvTransposeUnary2dconv_transposer_   r`   c                    sT   t  |j|j|j|j|j|j|j|j	d u|j
|j|jj|jj | ||| d S r   )r   r   r>   rb   rc   rd   re   output_paddingr=   r9   rf   rg   r6   rh   r7   ri   )r   r   r_   r`   r   r   r   r   Q  s    zConvTransposeUnary2d.__init__c              	   C   sz   t |j| _|r t|j |ndg df\| _| _| _tj	j
| j | j| j| j| j| j|}tjj|| jjd| _d S )Nrk   r   rn   )rp   rq   rr   rt   r   rs   r-   r.   r8   r~   r   Z%_reorder_convolution_transpose_weightr6   ry   re   r   rd   rf   r=   r)   ru   ro   )r   r   r_   r`   r   r   r   r   ri   g  s     	z*ConvTransposeUnary2d._update_module_paramsc                 C   s   | j dkrNtjjtj|| j| j d||td| j	| j
| j| j| j| j| jS tjj|||| j| j	| j
| j| j| j| j| jS r{   )rg   r8   r~   r   Z _convolution_transpose_pointwiser   r   r   r
   r   rd   rf   r=   rs   r-   r.   re   r   r   r   r   _conv_transpose_forwardz  s:    

z,ConvTransposeUnary2d._conv_transpose_forwardc                 C   s   |  || j| jS r   )r   r6   r9   r   r   r   r   r     s    zConvTransposeUnary2d.forward)r0   r1   r2   r)   r4   r   r   r   ri   r   r   r5   r   r   r   r   r   P  s   r   r^   r`   c                 C   s   | j rJ dt| d |S NzFusion only for eval!trainingr\   r   r   r   r   packed_conv_eval  s    r   r   r`   c                 C   s   | j rJ dt| d |S r   r   r   r   r   r   r   packed_conv_transpose_eval  s    r   r]   c                 C   s   | j rJ dt| ||S r   r   r]   r   r   r   fused_conv_unary_eval  s    r   r   c                 C   s   | j rJ dt| ||S r   )r   r   r   r   r   r   fused_conv_binary_eval  s    r   Zconv_binaryr_   r`   c                 C   s   | j rJ d| | | S r   )r   r   r   r   r   r   fused_conv_binary_unary_eval  s    
r   r   c                 C   s   | j rJ dt| |S r   )r   r   r   r   r   r   packed_linear_eval  s    r   r   r_   r`   c                 C   s   | j rJ dt| |S r   )r   r   r   r   r   r   fused_linear_unary_eval  s
    r   )r   rs   r`   c                 C   s   | j rJ dt| |}|S r   )r   r   )r   rs   r`   Zlinear_binaryr   r   r   fused_linear_binary_eval  s    r   r   c                 C   s   | j rJ dt| ||S r   r   r   r   r   r   fused_conv_transpose_unary_eval  s    r   )gmc                 C   s   t dd |D }t r| S tjjjr4tjj s8| S |s@| S t|}t| |dj	|  t
| } t| } t
| } tjjrt| } | S )Nc                 s   s*   | ]"}t |tjr|jtd kV  qdS )cpuN)rA   r8   ZTensorrh   )r"   Zexample_inputr   r   r   r$     s   z!mkldnn_fuse_fx.<locals>.<genexpr>)	fake_mode)r,   r8   Zis_grad_enabledbackendsr   enabledis_availabler   r	   	propagate
fuse_unaryfuse_binaryr   cppZweight_prepackpack_module)r   Zexample_inputsZis_cpur   r   r   r   mkldnn_fuse_fx  s"    r   rR   c                  C   s   | j dks| j dksJ dtjtjtjtjtjtjtj	tj
tjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjdtjdtjdtji}|| j | jdd  i | jS )NrI   rJ   z1The current node should be a function/method noderelusigmoidtanhr   )rD   r   r   r)   ReLUr   Sigmoidr   Tanh	hardswish	Hardswish
leaky_relu	LeakyReLUhardtanhr+   geluGELUrelu6r*   siluSiLUhardsigmoidHardsigmoidr8   rE   rY   rZ   )rS   Z	unary_mapr   r   r   create_unary_module  s0    r   c                 C   s  t |  }ttt D ]\}\}}||f}| jjD ]}t	|||sXt
|||r:t|jd jdkrnq:||jd j }|jdks|jdkrtdd |jdd  D s:tdd |j D rq:t|}|  n
||j }tdd ||fD }	|	sq:t|tjfv r$t|jtr$q:t|tfv rHt|tjfvrHq:t|tjfv rft|sfq:t|rrq:|jd jd j !d	j"}
||||
}t#|jd || |$|jd  | j%| q:q| j&  | '  | S )
Nr   r   rI   rJ   c                 s   s   | ]}t |tjjV  qd S r   rA   r8   rB   rC   )r"   vr   r   r   r$   4  r%   zfuse_unary.<locals>.<genexpr>c                 s   s    | ]\}}t |tjjV  qd S r   r   )r"   _r   r   r   r   r$   4  s   c                 s   s   | ]}|j  V  qd S r   )r   )r"   nr   r   r   r$   <  r%   tensor_meta)(dictnamed_modules	itertoolsproduct	unary_ops"computation_op_unary_op_fusion_mapitemsgraphnodesr   r   rX   rY   usersrE   rD   anyrZ   r   evalr,   r(   r)   Conv2drA   re   r3   r   r   Linearr;   r?   metagetshaper   replace_all_uses_with
erase_nodelint	recompile)r   rF   Zunary_opZcomputation_module	fuse_funcpatternrS   computation_nodeZ
unary_nodeZ	eval_modecomputation_node_input_sizefused_moduler   r   r   r   !  sl    
$



r   c           	      C   sl   |j | j d jdj}|| ||}t|j | || |j | j |j | f |j | _ ||j |  d S )Nr   r   )rY   r   r   r   r   r   )	r   rS   r   rs   rF   
index_nodeindex_pointwiser   r   r   r   r   replace_and_fuse_for_binary`  s    
r   c                 C   s\   | j d jd}| j d jd}|r,|s0dS |j|jksT|j|jksT|j|jkrXdS dS )Nr   r   r   FT)rY   r   r   r   rd   r7   )Zbinary_nodeZtensor0_metaZtensor1_metar   r   r   binary_inputs_meta_is_samen  s    


r   c                 C   st  t |  }| jjD ]F}t|rt|rt D ]&\}}t|j	d t
jjr2t|j	d t
jjsfq2t|spq2t|j }t| }|D ]}|d }|d }	t|j	| ||rt|j	| jdkrq||j	| j }
|
jrqt|
tjfv rt|
jtrqt|
tjfv rt|
sqt|
|||dkr,|nd|||	 ||j	|  | j|  q2qq2q| j  |    | S )Nr   r   index_computationr   rP   rK   )!r   r   r   r   rT   r[   #computation_op_binary_op_fusion_mapr   rA   rY   r8   rB   rC   r   r   rE   supported_index_listrH   rX   r   r   r(   r)   r   re   r3   r   r;   r   appendr   r   r   )r   rF   rS   rG   r   rs   Z
index_listZ
index_dictr   r   r   r   r   r   r   }  s\    



r   c                 C   s   t jjjrt jj s| S | jjD ]}|jdkr"|jt j	jj
jfv r"|jd jdv rVq"t|jd jdkrlq"|jd |jd krq"|jd }|jd }|dks"|dvrq"t j	jjj|_q"| j  |   | S )	NrI   r   )placeholderoutputr      
   rK   )r   r   )r8   r   r   r   r   r   r   rD   rE   r~   r   binaryrY   rX   r   Z_convolution_pointwise_r   r   )r   rS   r   r   r   r   r   convert_outplace_to_inplace  s(    



r  c                 C   s"  t |  }| jjD ]}|jdkrt|jts2J ||j }t|t	v r|j
rPq|jd jd}|jtjkrpqt|tjjfv rtjjsq|j}t|tjjfv rt|dk rqt|tjfv rt|jtrqt|rqt	t| ||}t|tj sJ t||| q| j  |   | S )Nr@   r   r   rU   )r   r   r   r   rD   rA   rE   r3   r(   computation_op_packed_mapr   rY   r   r   r7   r8   float32r)   r   rv   Zhas_mklr   rX   r   re   r?   r4   r   r   r   )r   rF   rS   Z
cur_moduleZcomputation_node_input_metar   Z
new_moduler   r   r   r     sB    




r   r   r   r   r   r   Znegative_slope)r   r   r   r   r   Zapproximate)r   Zswishr   rK   rL   rP   rM   rN   )r   r   )rK   rP   rM   )arp   r   rO   	functoolsr   typingr   r8   Ztorch.nnr)   Ztorch.nn.functionalZ
functionalr   Ztorch._dynamo.utilsr   Z"torch.fx.experimental.optimizationr   r   Z%torch.fx.experimental.symbolic_shapesr   Ztorch.fx.passes.shape_propr	   Ztorch.nn.modules.utilsr
   r   r   Zfx_utilsr   r   r;   r?   rH   rT   r[   r   r\   r   r   r   r   r   r<   r   r4   r   r   r   r   r3   r   r   r   r   r   r   rB   ZGraphModuler   rS   r   r   r   r   r   r  r   r   r   r   r   r   r   r+   r   r*   r   r   rt   r   r   r   r   r   r   r   r   r   r   r   rK   rP   rM   rQ   r   r   r	  r  r   r   r   r   <module>   s   LUM			

	

?3&	





"

