a
    dK                    @   s  d dl Z d dlmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZmZmZ d dlmZ d dlmZmZ d dlmZmZmZmZmZmZmZ d dlmZ d d	lmZ d d
lmZ d dl m!Z! ej"j#Z#ej$%dddZ&dd Z'dd Z(e'e#j)j*e#j)j+ge dd Z,e'e#j-j*e#j-j+ge dd Z.e'e#j/j0ddddZ1e'e#j2j*ej3ddddddZ4e'e#j2j5ej3ddddddZ6e'e#j7j*ddddddd Z8e'e#j9j*e#j9j+ge d!d" Z:e'e#j;j*dId$d%Z<d&d' Z=e'e#j>j*d(d) Z?e'e#j@j*d*d+ ZAe'e#j@j+d,d- ZBe'e#jCj*e#jCjDge d.d/ ZEe'e#jCjFdJd0d1ZGe'e#jHj*gd2d3 ZIe'e#jJj*d4d5 ZKe'e#jJj+d6d7 ZLe	eMd8d9d:ZNdKe	eMeOd<d=d>ZPdLe	eMeMd@dAdBZQeMdCdDdEZRdMdGdHZSe'e#jTj*dNe	eOeOdIdJdKZTe'e#jUj*dOe	eOdLdMdNZVe'e#jWj*dPe	eOeOeMdOdPdQZXe'e#jYj*dRdS ZZe'e#j[j*e#j\j*gdTdU Z]e'e#j^j*dVdW Z_e'e#j`j*e#j`j+ge dddXdYZae'e#jbjcdQd[d\Zde'e#j`jedRd]d^Zfe'e#jgj*dSd_d`Zhdadb Zie'e#jjj*dcdd Zke'e#jlj*gdedf Zmdgdh ZndidjdkdlZodTej	ej	eeep epf eeep epf eeep epf eOepeeeep epf  dmdndoZqdpdq Zre'e#jsj*ej	ej	ej	eep eep eep eOeep epdr	dsdtZtejujvrej$%duddZwdvdw Zxe'ej"jyjzj*dxdy Z{e'ej"jyjzj|dzd{ Z}e'ej"jyj~j|d|d} Ze'ej"jyjj*d~d Ze'ej"jyjj|dd Zejujrej$%dddZe'ej"jjdd Zdd Ze'e#jj*dUddZdd Ze'e#jj*dd Ze'e#jj*dd Ze'e#jj*dd Ze'e#jj*dd Ze'e#jj	dVddZe'e#jj*e#jj+ge dd Ze'e#jj*dd Ze'e#jj	dd Ze'e#jj*gdd Ze'e#jj*e#jj+ge dddddZe'e#jj*dd Ze'e#jj*dWddZe'e#jj*dd ZdXddZe'e#jj*e#jj+ge dYddddZe'e#jj*dd Ze'e#jjFe#jjgedddZddZe'e#jj*dd Ze'e#jj*dd Ze'e#jj*dd Ze'e#jje#jje#jj	e#jj	e#jj*e#jj*e#jj*gdd Ze'e#jje#jje#jj	e#jj	gd[ddZe'e#jj*e#jjgdd Ze'e#jj*dd Ze'e#jj	e#jjgddĄ Ze'e#jj	e#jjgddƄ Ze'e#jj*ddȄ Ze'e#jj*d\ddʄZe'e#jjdd̄ Ze'e#jj*d]dd΄Ze'e#jj*ddЄ Zΐd^dd҄Ze'e#jj*ddԄ Zddք Zdd؄ Zddڄ Zdd܄ Zddބ Ze'e#jj*dd Ze'e#jj*d_ddZe'e#jj*dd Ze'e#jj*gdd Ze'e#jj*e#jje#jj*e#jj*e#jj*e#jj*gdd Ze'e#jj*d`ddZe'e#jjpdd Ze'e#jj*dd Ze'e#jj*daddZdbepepeOdddZdd Zdd Ze'e#jj*dcddZddddZdeddZdd  ZdfddZdgddZe'e#jj*dd Ze'e#jdd Ze'e#jje#jje#jje#jjge dhd	d
Ze'e#j je#j je#j je#j jgdiddZe'e#jgdje	e	e	eceOeOdddZe'e#jge	e	e	e	e	e	e	e	epepeceOepepdddZe'e#jgdke	e	e	eOeOdddZe'e#jgdle	e	e	e	e	e	eOdddZ	e'e#j
je#j
jge dmddZe'e#jjdnddZdd Zd d! Ze'e#jj*dod"d#Ze'e#jj*dpd$d%Ze'e#jj*dqd&d'Ze'e#jj*e#jjgdrd(d)Zd*d+ Ze'e#jj*dsd,d-Ze'e#jj*d.d/ Ze'e#jj*d0d1 Zd2d3 Zd4d5 Ze'e#jj*e#j j*gdtd6d7Z!e'e#j"j*dud8d9Z"e'e#j#j*dvd:d;Z$ej%Z&d<d= Z'e'e#j(j*d>d? Z(e'e#j)j*d@dA Z*e'e#j+j*dBdC Z+e'e#j,j	e#j,j-ge d#d#dDdEdFZ.d dlZd dl/Zd dl0ZdGdH Z1e1  dS (w      N)ListOptionalUnion)Tensor)_add_op_to_registryglobal_decomposition_table
meta_table)
OpOverload)_elementwise_meta$ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND)checkcorresponding_complex_dtypecorresponding_real_dtypeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDIntLikemake_contiguous_strides_for)out_wrapper)_broadcast_shapes)check_no_bool_index_tensorstree_mapatenZIMPLMetac                    s    fdd}|S )Nc                    s    fdd}t |  S )Nc                    s   t t|   d S N)r   r   opfn b/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_meta_registrations.pyregister"   s    z0register_meta.<locals>.wrapper.<locals>.registerr   )r   r!   r   r   r    wrapper!   s    
zregister_meta.<locals>.wrapperr   )r   r"   r   r   r    register_meta    s    r#   c                 C   s(   t jt jt jt jt jt ji}|| | S r   )torchZ	complex32halfcfloatfloatcdoubledoubleget)dtypeZfrom_complexr   r   r    toRealValueType+   s
    r,   c                 C   s   | j jsJ | |  S r   )r+   
is_complex	new_emptysize)selfdimnormalizationZforwardr   r   r    meta_fft_c2c4   s    r3   c                 C   sR   | j jsJ t|  }|r<|d }|| d d }|||< | j|t| j dS )N      r+   )r+   is_floating_pointlistr/   r.   utilsr   )r0   r1   r2   Zonesidedoutput_sizesZlast_dimZlast_dim_halfsizer   r   r    meta_fft_r2c;   s    r<   )	generatorc                C   s    |j dkr|d| ksJ |S Nr6   r   )ndimr/   )nr=   outr   r   r    meta_randpermK   s    rB   r+   layoutdevice
pin_memoryc                C   s   t j|||||dS NrC   r$   empty)highr/   r+   rD   rE   rF   r   r   r    meta_randintQ   s    
rK   c                C   s   t j|||||dS rG   rH   )lowrJ   r/   r+   rD   rE   rF   r   r   r    meta_randint_lowZ   s    
rM   c                C   s   t j| ||||dS rG   rH   )r/   r+   rD   rE   rF   r   r   r    meta_rand_defaultc   s    
rN   c                 C   s8   | j jsJ t|  }|||d < | j|t| j dS )Nr4   r7   )r+   r-   r9   r/   r.   r,   )r0   r1   r2   Zlastdimr;   r   r   r    meta_fft_c2rj   s    rO   Fc                 C   s   | S r   r   )r0   srcZnon_blockingr   r   r    
meta_copy_s   s    rQ   c                 C   sX   t |  }t |  }||  kr(dn|| ||  }||d ||| ||fS Nr6   )r9   r/   strider1   insert)tensorr1   Zresult_sizesZresult_strides
new_strider   r   r    inferUnsqueezeGeometryx   s     rW   c                 C   s0   t ||  d }t| |\}}| || | S rR   )maybe_wrap_dimr1   rW   as_strided_)r0   r1   Zg_sizesZ	g_stridesr   r   r    meta_unsqueeze_   s    rZ   c                 C   s.   t |  }|  dkr$| ||< | |S )Nr   )r9   r/   r1   numelr.   )r0   r1   indexZresult_sizer   r   r    meta_index_select   s    r]   c                 C   s(   t ||  | j |t | ||S r   )r$   _resize_output_r/   rE   copy_index_select)r0   r1   r\   rA   r   r   r    meta_index_select_out   s    ra   c                 C   s
   |  dS Nr   r.   r0   r   r   r    meta_max   s    re   c                 C   s6   t | j|f}t| ||}| || j|tjdfS Nr7   r:   reduction_dimsshape_compute_reduction_shaper.   r$   long)r0   r1   keepdimoutput_shaper   r   r    meta_max_dim   s
    rn   c                 C   s
   |  dS rb   rc   rd   r   r   r    meta_min   s    ro   c                 C   s4   |   rt| j}nt| tjd\}}tj| |dS )N)Ztype_promotion_kindr7   )r-   r   r+   r   r   ZINT_TO_FLOATr$   
empty_like)r0   Zresult_dtype_r   r   r    
meta_angle   s    
rr   c                 C   s$   t ||  | j |t | S r   )r$   r^   r/   rE   r_   angle)r0   rA   r   r   r    meta_angle_out   s    rt   r0   f_namec                 C   sX   |   dksJ | d| d| dksTJ | d| d d| d dd S )Nr5   z3: The input tensor must have at least 2 dimensions.r4   z5: A must be batches of square matrices, but they are z by z	 matrices)r1   r/   ru   r   r   r    squareCheckInputs   s    
 rx   Ttrv   allow_low_precision_dtypesc                    sV   | j  t|  p|   fdd |rRt tjtjtjtjfv  fdd d S )Nc                      s    d  S )Nz@, : Expected a floating point or complex tensor as input. Got , r   r   r+   rv   r   r    <lambda>       z(checkFloatingOrComplex.<locals>.<lambda>c                      s    d  S )Nz+ : Low precision dtypes not supported. Got r   r   r|   r   r    r}      r~   )	r+   r   r8   r-   r$   r'   r)   r&   r(   ry   r   r|   r    checkFloatingOrComplex   s    r   Ar   rv   arg_namec                    s    t |  dk fdd d S )Nr5   c                      s    d  dS )Nz: The input tensor z! must have at least 2 dimensions.r   r   r   rv   r   r    r}      r~   zcheckIsMatrix.<locals>.<lambda>r   r1   r   r   r   r    checkIsMatrix   s    
r   )uploc                 C   s6   |   }t| dkr|dks2|dks2J d|  d S )Nr6   ULz1Expected UPLO argument to be 'L' or 'U', but got )upperlen)r   Zuplo_uppercaser   r   r    	checkUplo   s    
r   r   c                 C   sd   t | d t| t| j}|  dks,J | j| j|d}|dd | | jd d }||fS )NZlinalg_eighr5   r7   rw   r4   )rx   r   r,   r+   r1   r.   ri   Z
transpose_)r0   r   Z
real_dtypevaluesZvectorsr   r   r    meta_linalg_eigh   s    

r   )r   r   check_errorsc                 C   sf   t | d t| d | j}t|}t|d}| |}||| | j|d|d  tjd}||fS )Nzlinalg.choleskyFr   r5   r7   )	rx   r   ri   r   r   r.   rY   r$   int32)r   r   r   ZA_shaper?   Z	L_stridesr   infosr   r   r    linalg_cholesky_ex   s    



r   )r   r   c                 C   s^   t | d t| ddd | | j}|| jt| jdd | j| jd d tjd}||fS )Nzlinalg.inv_exF)r{   Z	row_majorrw   r7   rx   r   r.   ri   rY   r   r$   r   )r   r   r   r   r   r   r    linalg_inv_ex_meta  s    
r   )r   full_matrices
compute_uvdriverc                 C   s   t | d t| d t| jd d }| jd }| jd }t||}|r|||rT|n|g }| |}	|	|t|dd ||r|n||g }
| |
}||
t|
dd n| dg}	| dg}| j||g t| j	d}|	||fS )Nz
linalg.svdrw   r4   Fr   r   r7   )
r   r   r9   ri   minr.   rY   r   r,   r+   )r   r   r   r   Z
batch_dimsmr@   kZU_shaper   ZV_shapeVSr   r   r    _linalg_svd_meta  s"    






r   c                 C   sp   t | d t| d | | jd d }| | j}|| jt| jdd | j| jd d tjd}|||fS )Nz
linalg.detrw   Fr   r4   r7   r   )r   ZdetZLUZpivotsr   r   r    _linalg_det_meta5  s    

r   c                    s   dd d}d}|j }| dkrB|d }d7  d7  |d7 }|d }|d }|d }|d }	|| }
|  }| }|| |	 || | tj  kfdd tj   k fdd ||j S )	Nr5   r6   r         c                      s   d dj    S )Nz'gradOutput width unexpected. Expected: , Got: ri   r   )dim_wgrad_outputoutput_wr   r    r}   a  r~   z%meta_pad2d_backward.<locals>.<lambda>c                      s   d dj    S )Nz(gradOutput height unexpected. Expected: r   r   r   )dim_hr   output_hr   r    r}   e  r~   )ri   r1   r   r.   )r   r0   paddingZ	dim_planenbatchZ
self_shapepad_lpad_rpad_tpad_bnplaneinput_hinput_wr   )r   r   r   r   r   r    meta_pad2d_backwardD  s8    r   c                    s     ddko  ddk}t jdkr,|pF jdkoF|oF  ddk fdd  jdkrn j\}}}}nd} j\}}}|\}}}	}
||	 |
 }|| | } jdkr |||fS  ||||fS d S )Nr6   r   r5   r   r   c                      s
   d  S )Nz:3D or 4D (batch mode) tensor expected for input, but got: r   r   rd   r   r    r}   p  r~   zmeta_pad2d.<locals>.<lambda>)r/   r   r?   ri   r.   )r0   r   
valid_dimsr   r   r   r   r   r   r   r   r   r   r   rd   r    
meta_pad2dj  s     


r   c                C   s   t |  S r   r$   rp   
contiguous)r0   r=   r   r   r    meta_bernoulli  s    r         ?c                 C   s   | S r   r   r0   pr=   r   r   r    meta_bernoulli_  s    r   c                 C   s   t |  S r   r   r   r   r   r    meta_bernoulli_p  s    r   c                 C   s4   t |
|  k dd  tj| tjd}t| |fS )Nc                   S   s   dS )NzJError in fused_moving_avg_obs_fake_quant_cpu: ch_axis must be < self.dim()r   r   r   r   r    r}     r~   z6meta__fused_moving_avg_obs_fq_helper.<locals>.<lambda>r7   )r   r1   r$   rp   bool)r0   Zobserver_onZfake_quant_onZrunning_minZrunning_maxscaleZ
zero_pointZaveraging_constZ	quant_minZ	quant_maxZch_axisZper_row_fake_quantZsymmetric_quantmaskr   r   r    $meta__fused_moving_avg_obs_fq_helper  s    
r   c                    s,   t  dko  dk fdd d S )Nr6   c                      s   d   d    dS )Nz1D tensors expected, but got zD and z	D tensorsr1   r   otherr0   r   r    r}     r~   zdot_check.<locals>.<lambda>r   r0   r   r   r   r    	dot_check  s    r   c                 C   s   t | | | dS rb   )r   r.   )r0   rU   r   r   r    meta_dot  s    
r   c                 C   s^   t |  dkdd  t | dkdd  | j\}}|j\}}t ||kdd  | ||S )Nr5   c                   S   s   dS )Nza must be 2Dr   r   r   r   r    r}     r~   zmeta_mm.<locals>.<lambda>c                   S   s   dS )Nzb must be 2Dr   r   r   r   r    r}     r~   c                   S   s   dS )Nz$a and b must have same reduction dimr   r   r   r   r    r}     r~   )r   r1   ri   r.   )abNZM1ZM2Pr   r   r    meta_mm  s    

r   c                    s0   |r"t  fddtjD S tj S )Nc                 3   s$   | ]}| vrj | nd V  qdS )r6   Nr   .0idimsr0   r   r    	<genexpr>  r~   z+_compute_reduction_shape.<locals>.<genexpr>)tupleranger?   r:   compute_reduction_output_shaperi   )r0   r   rl   r   r   r    rj     s    rj   str)returnc                 C   s   t | tjjr| jjS dS d S )Ncuda)
isinstancer$   Z_subclassesZ
FakeTensorZfake_devicetype)rU   r   r   r    device_hint  s    r   )input_tensorweightrS   r   dilationis_transposedgroupsoutput_paddingc                 C   s  t t t t t t ddd}t t t t t t t ddd}	|jdd  }
| jdd  }|rb||jd  }n*|jd	 }|jd | | jd krtd
| jd	 |g}t|tr|gt| }nt|dkr|d	 gt| }t|tr|gt| }n t|dkr|d	 gt| }t|tr(|gt| }n t|dkrH|d	 gt| }d }|rt|trn|gt| }n&t|dkr|d	 gt| }n|}tt|D ]h}|r||	|| || || |
| || ||  n*|||| || || |
| ||  q|S )N)lnr   dr   sr   c                 S   s$   | d|  ||d   d | d S )a  
        Formula to apply to calculate the length of some dimension of the output

        See: https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
        Returns:
            The output length
        r5   r6   r   )r   r   r   r   r   r   r   r    _formula  s    z+calc_conv_nd_return_shape.<locals>._formula)r   r   r   r   r   r   r   c                 S   s(   | d | d|  ||d   | d S )a  
        Formula to apply to calculate the length of some dimension of the output
        if transposed convolution is used.
        See: https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
            op: output padding in that dim

        Returns:
            The output length
        r6   r5   r   )r   r   r   r   r   r   r   r   r    _formula_transposed  s    z6calc_conv_nd_return_shape.<locals>._formula_transposedr5   r6   r   zInvalid channel dimensions)intri   RuntimeErrorr   r   r   r   append)r   r   rS   r   r   r   r   r   r   r   kernel_sizer   Zout_channelsZ	ret_shapeZoutput_padding_listr   r   r   r    calc_conv_nd_return_shape  sZ    



"r   c                 C   s   t j| t jkS r   r$   _prims_commonsuggest_memory_formatchannels_lastZtenr   r   r    is_channels_last;  s    r   )	r   r   biasrS   r   r   r   r   r   c	              	      sH    fdd}	t  ||||||r&|nd }
 |
}|j|	 d}|S )Nc                      s^   t  dkr$t str2tjS nt r2tjS  jtjdrFtjS  jtjdrZtjS d S Nr   memory_format)r   r   r$   r   is_contiguouscontiguous_formatpreserve_formatr   r   r   r   r    pick_memory_formatK  s    z%meta_conv.<locals>.pick_memory_formatr   )r   r.   to)r   r   r   rS   r   r   r   r   r   r   	shape_outrA   r   r   r    	meta_conv?  s    

r   mkldnnc                 C   sN   |j rtjS t| st|r"tjS | jtjdr6tjS | jtjdrJtjS d S Nr   )Z	is_mkldnnr$   r   r   r   r   r   r   r   r   r    pick_mkldnn_conv_memory_formatl  s    r   c
              	   C   s6   t | ||||d|g }
| |
}tj}|j|d}|S )NFr   )r   r.   r$   r   r   )r   r   r   r   rS   r   r   attrscalars	algorithmr   rA   Zout_memory_formatr   r   r    meta_mkldnn_convolution_defaultv  s    
r  c                 C   s    |  | }|jtjd}|S r   )r.   r/   r   r$   r   )r   r   r   r   r   rS   r   r   binary_attralpha
unary_attrunary_scalarsunary_algorithmrA   r   r   r    meta_mkldnn_convolution_binary  s    r  c                 C   s   |S r   r   )r   r   r   r   r   rS   r   r   r  r  r  r  r  r   r   r    &meta_mkldnn_convolution_binary_inplace  s    r	  c                 C   s$   |  g | jd d |jd R S Nr4   r   r.   ri   )r   r   r   r   r   r  r   r   r    meta_linear_pointwise_default  s    r  c                 C   s   |  | }|S r   r.   r/   )r   r   r   r   r   rA   r   r   r    meta_linear_pointwise_binary  s    r  mklc                 C   s$   |  g | jd d |jd R S r
  r  )r   Zpacked_weightZorig_weightr   
batch_sizer   r   r    meta_mkl_linear  s    r  c                    s2   t   koj k fdd d S )Nc                      s8   d  d d dd   d dj   S )NzExpected a tensor of dimension z and tensor.size[z] == , zbut got : dimension z] = r1   ri   r   r1   dim_sizer/   rU   r   r    r}     s   z check_dim_size.<locals>.<lambda>)r   r1   ri   )rU   r1   r  r/   r   r  r    check_dim_size  s    r  r   r   c                 C   s`  dd }|d|\}}	t t|dv dd  t|dkrD||	 }
}n.t|dkrd|d |d  }
}n|d	|\}
}|d
|\}}t |d u p|dkdd  |  dkr| dnd}| d}| d}| d}t||||
d|}t||	||d|}t| }t| ||	|
|||dd|||||| |  dkr>|||g}n||||g}tj	|| j
| j|dS )Nc                    sB   t t|dv  fdd |d }t|dkr2|n|d }||fS )Nr6   r5   c                      s   d  dS )Nzavg_pool2d: 4 must either be a single int, or a tuple of two intsr   r   namer   r    r}     r~   z1meta_avg_pool2d.<locals>.unpack.<locals>.<lambda>r   r6   r   r   r  valHWr   r  r    unpack  s    

zmeta_avg_pool2d.<locals>.unpackr   r   r6   r5   c                   S   s   dS NzOavg_pool2d: stride must either be omitted, a single int, or a tuple of two intsr   r   r   r   r    r}     r~   z!meta_avg_pool2d.<locals>.<lambda>r   r6   rS   r   c                   S   s   dS Nzdivisor must be not zeror   r   r   r   r    r}     r~   r   rw   r4   r   r+   rE   r   )r   r   r1   r/   pooling_output_shaper:   r   pool2d_shape_checkr$   rI   r+   rE   )inputr   rS   r   	ceil_modecount_include_paddivisor_overrider!  kHkWdHdWpadHpadWr   nInputPlaneinputHeight
inputWidthoutputHeightoutputWidthr   r/   r   r   r    meta_avg_pool2d  s\    
	




r9  c                 C   sj   t | ||||||dd|	|
|||| |  }|	}t|||d | t|||d | t|||d | d S )Nr6   r   r5   )r)  r1   r  )r*  Z
gradOutputr   r.  r/  r0  r1  r2  r3  r4  r5  r6  r7  r8  
mem_formatr?   nOutputPlaner   r   r    avg_pool2d_backward_shape_check"  s,    r<  c                 C   s  t t|dkpt|dkdd  |d }t|dkr:|n|d }	t t|dkpft|dkpft|dkdd  t|dkr|n|d }
t|dkr|	nt|dkr|
n|d }t t|dkpt|dkdd  |d }t|dkr|n|d }t |d u p|dkdd  |j}| d	kr*|d
 nd}|d }|d }|d }t||||
d|}t||	||d|}t|}t|| |||	|
||||||||| tj	||j
|j|dS )Nr6   r5   c                   S   s   dS )NzKavg_pool2d: kernel_size must either be a single int, or a tuple of two intsr   r   r   r   r    r}   \  r~   z*meta_avg_pool2d_backward.<locals>.<lambda>r   c                   S   s   dS r#  r   r   r   r   r    r}   b  r~   c                   S   s   dS )NzGavg_pool2d: padding must either be a single int, or a tuple of two intsr   r   r   r   r    r}   h  r~   c                   S   s   dS r$  r   r   r   r   r    r}   o  r~   r   r%  r&  rw   r4   r'  )r   r   ri   r1   r(  r:   r   r<  r$   rI   r+   rE   )ZgradOutput_r*  r   rS   r   r+  r,  r-  r.  r/  r0  r1  r2  r3  
input_sizer   r4  r5  r6  r7  r8  r:  r   r   r    meta_avg_pool2d_backwardN  sd    "(
r>  c                    sX   t  jdkp jdk fdd  jd d t| }t }tj| j j	|dS )Nr   r   c                      s   d j  S )Nz"Expected 3D or 4D tensor, but got r   r   rd   r   r    r}     r~   z*meta_adaptive_avg_pool2d.<locals>.<lambda>rw   r'  )
r   r?   ri   r   r:   r   r$   rI   r+   rE   )r0   output_sizerm   r   r   rd   r    meta_adaptive_avg_pool2d  s    

r@  c                    s>   t  jdkp jdk fdd   jd d t| S )Nr      c                      s   d j  S )Nz"Expected 4D or 5D tensor, but got r   r   rd   r   r    r}     r~   z*meta_adaptive_avg_pool3d.<locals>.<lambda>r&  )r   r?   r.   ri   r   )r0   r?  r   rd   r    meta_adaptive_avg_pool3d  s
    
rB  c                    sz    j }td|D ]"t dk fdd qt|dkpD|dkfdd tj jk fdd jS )	Nr6   r   c                      s   d j  d dS )Nz{adaptive_avg_pool2d_backward(): Expected grad_output to have non-zero                       size for non-batch dimensions, z with dimension z being emptyr   r   )grad_outr   r   r    r}     s   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>r   r   c                      s   d j  S )NzBadaptive_avg_pool2d_backward(): Expected 3D or 4D tensor, but got r   r   rd   r   r    r}     r~   c                      s   dj  d j  S )Nzexpected dtype z! for `grad_output` but got dtype r7   r   )rC  r0   r   r    r}     r~   )r?   r   r   r/   r+   r.   ri   )rC  r0   r?   r   )rC  r   r0   r    "meta__adaptive_avg_pool2d_backward  s    

rD  c                 C   s   |d u rt d| |S )Nz:cannot repeat_interleave a meta tensor without output_size)r   r.   )repeatsr?  r   r   r    meta_repeat_interleave_Tensor  s    rF  c                 C   s:   | j jsJ |j jsJ t| j|j}| j|t| j dS rf   )r+   r8   r   ri   r.   r   )realimag	out_shaper   r   r    meta_complex  s    rJ  c                 C   sx   | j st| |S |  rH| r6t| |  S t|  |S n| rdt| |  S t| | | dS rb   )r-   r$   dotZis_conjvdotZconjr   r.   r   r   r   r    rL    s    
rL  c              	      s  t tjj tdd  g }tD ]\ d urtjtjtj	tj
tjfv dd  jtj
tjfv r }t|tj jkfddt tjD ]Ftj j  k fddt ||d qn
| q*| q*|ttjkfdd dd lm} t|j tjk rhd  qJd}d	}D ]J|dkrd urd}n*|dkrd u rd
}nd urt qĐqtd}|sDg }g }tD ](\ d ur|  | qtD ](\ d u r|  | q||g }	g }
g }tD ]H\}d u r|r|
j|  n|	j|  n
tj}qX|	| |
 S )Nc                   S   s   dS )Nz#at least one index must be providedr   r   r   r   r    r}     r~   z#meta_index_Tensor.<locals>.<lambda>c                   S   s   dS )Nz?tensors used as indices must be long, int, byte or bool tensorsr   r   r   r   r    r}     r~   c                      s   d j  S )N)too many indices for tensor of dimension r?   r   rd   r   r    r}     r~   c                	      s$   dj  d  dj  d  S )NzThe shape of the mask z
 at index z0 does not match the shape of the indexed tensor r   r   )r   r\   jr   r0   r   r    r}     s   r6   c                      s   dj  dt  dS )NrM  z (got ))r?   r   r   )indicesr0   r   r    r}   	  r~   r   Fr5   T)r   r   r\   r   r   	enumerater+   r$   rk   r   Zint8r   nonzeror   r?   
IndexErrorr   ri   r   selecttorch._refsZ_refsr9   Z_maybe_broadcastZpermuter.   )r0   rQ  resultrS  refsstateZhas_contiguous_subspacer   Ztransposed_indicesZbefore_shapeZafter_shapeZreplacement_shaper1   r   )r   r\   rQ  rO  r   r0   r    meta_index_Tensor  s    













rZ  c                 C   sT   d }d }d }|
d r"|  | }|
d r8|  | }|
d rJ|  |}|||fS )Nr   r6   r5   r  )Zgrad_output_Zinput_Zweight_Zbias_sizes_optrS   r   r   Z
transposedr   r   output_maskZbackend_grad_inputZbackend_grad_weightZbackend_grad_biasr   r   r    meta_convolution_backwardL  s    
r\  r6   )betar  c                   s     d} d}| ||f} t  dkdd  t dkdd  t  d dk fdd t  d dk fd	d t|  d|ko|  d|kd
d  | |   S )Nr6   r5   r   c                   S   s   dS Nzbatch1 must be a 3D tensorr   r   r   r   r    r}   p  r~   zmeta_addbmm.<locals>.<lambda>c                   S   s   dS Nzbatch2 must be a 3D tensorr   r   r   r   r    r}   q  r~   r   c                      s   d  d d d S )Nz8batch1 and batch2 must have same number of batches, got r    and r/   r   batch1batch2r   r    r}   t  r~   c                
      s6   d  d d  d d d d d d	S )Nz#Incompatible matrix sizes for bmm (r6   xr5   r`  rP  ra  r   rb  r   r    r}   x  s
    c                   S   s   dS )Nz.self tensor does not match matmul output shaper   r   r   r   r    r}     r~   )r/   expandr   r1   r.   )r0   rc  rd  r]  r  Zdim1Zdim2r   rb  r    meta_addbmmj  s$    

rg  c           	         s  t  dkfdd t  dkfdd t ddkfdd t tjdd  t tjdd  t |d	kd
d  t  dv  fdd d}d}jd d }jd d }tt	||}|
||g |S )Nr5   c                      s   d    dS )Nz1cdist only supports at least 2D tensors, X1 got: Dr   r   )x1r   r    r}     r~   z$meta_cdist_forward.<locals>.<lambda>c                      s   d    dS )Nz1cdist only supports at least 2D tensors, X2 got: rh  r   r   )x2r   r    r}     r~   r4   c                      s   d  d d d S )Nz4X1 and X2 must have the same number of columns. X1: r4   z X2: ra  r   )ri  rj  r   r    r}     r~   c                   S   s   dS )Nz=cdist only supports floating-point dtypes, X1 got: {x1.dtype}r   r   r   r   r    r}     r~   c                   S   s   dS )Nz=cdist only supports floating-point dtypes, X2 got: {x2.dtype}r   r   r   r   r    r}     r~   r   c                   S   s   dS )Nz)cdist only supports non-negative p valuesr   r   r   r   r    r}     r~   )Nr6   r5   c                      s
   d  S )Nz%possible modes: None, 1, 2, but was: r   r   )compute_moder   r    r}     r~   rw   )r   r1   r/   r:   is_float_dtyper+   ri   r9   r$   Zbroadcast_shapesextendr.   )	ri  rj  r   rk  Zr1Zr2Zbatch_tensor1Zbatch_tensor2rm   r   )rk  ri  rj  r    meta_cdist_forward  s@    








rn  r4   c	                    s:  t  jtjtjfv  fdd t jtjtjfv fdd t tjfdd d}	|rt |	dkdd  |	d8 }	|	d}
t	d\}}}d urt ||kd	d  t jjkfd
d t j
dkfdd t    k fdd fdddd fdd}tdkr  d}  }||kr~ |	d}n
 d}n||
|}||ks||ks|sĈ d}n
d}|	}jd }||kr |rt |dkdd  |d8 }|jd }n| }|
|||fS )Nc                      s   d j  S )Nz(expected indices to be long or int, got r7   r   )rQ  r   r    r}     r~   z$meta_embedding_bag.<locals>.<lambda>c                      s   d j  S )Nz(expected offsets to be long or int, got r7   r   )offsetsr   r    r}     r~   c                      s   d j  S )Nz/expected weight to be floating point type, got r7   r   )r   r   r    r}     r~   r   r6   c                   S   s   dS Nz1include_last_offset: numBags should be at least 1r   r   r   r   r    r}     r~   r   c                   S   s   dS )Nz@embedding_bag: per_sample_weights only supported with mode='sum'r   r   r   r   r    r}     r~   c                      s   dj  d j  dS )Nzexpected weight (z) and per_sample_weights (z) to have same dtyper7   r   )per_sample_weightsr   r   r    r}     r~   c                      s   d j  dS )Nz1expected per_sample_weights to be 1D tensor, got rh  rN  r   )rq  r   r    r}     r~   c                      s   d   d    dS )Nz%expected per_sample_weights.numel() (z$ to be the same as indices.numel() (rP  r[   r   )rQ  rq  r   r    r}     s    c                    s    | ||o| ddkS Nr   r6   )rS   rP   r   outputpadding_idx)is_fast_path_index_selectr   r    is_fast_path_index_select_scale  s    z;meta_embedding_bag.<locals>.is_fast_path_index_select_scalec                 S   s<   | j tjks| j tjko:| ddko:|ddko:|dk S r>   )r+   r$   r'   r%   rS   )rP   ru  rv  r   r   r    rw    s    z5meta_embedding_bag.<locals>.is_fast_path_index_selectc                    s&   |d ur| |||S  | ||S d S r   r   rt  )rw  rx  r   r    is_fast_path  s    z(meta_embedding_bag.<locals>.is_fast_pathcpuc                   S   s   dS rp  r   r   r   r   r    r}     r~   )r   r+   r$   rk   r   r:   rl  r/   r.   r   r?   r[   r   ri   )r   rQ  ro  Zscale_grad_by_freqmodesparserq  Zinclude_last_offsetrv  Znum_bagsru  ZMODE_SUMZ	MODE_MEANZMODE_MAXry  
offset2bagbag_sizemax_indicesZfast_path_sumZnumBagsr   )rQ  rw  rx  ro  rq  r   r    meta_embedding_bag  s|    












r  c                 G   sB   t | ||g|R  \}}}}t|dkr6|| }||||fS )Nrz  )r  r   r.   r/   )r   rQ  ro  argsru  r}  r~  r  r   r   r    meta_embedding_bag_forward_only  s    r  c                 C   s.   |r|S | j js| j jr| j S |r(tjS | j S r   )r+   r8   r-   r$   rk   )r*  r+   promote_int_to_longr   r   r    _get_reduction_dtype  s    r  r7   c                C   s6   t | |dd}t| j|}t| ||}| j||dS )NT)r  r7   )r  r:   rh   ri   rj   r.   )r*  r   rl   r+   Zoutput_dtyperm   r   r   r    meta_nansum'  s    r  c                 C   s$   t | jtt|  }| |S r   )r:   r   ri   r   r   r1   r.   )r*  rm   r   r   r    meta_nanmedian0  s    r  r   rQ  c                 C   s6   t | j|f}t| ||}| || j|tjdfS rf   rg   )r*  r1   rl   rm   r   r   r    meta_nanmedian_dim8  s
    r  c                 C   s   | S r   r   rd   r   r   r    meta_logical_not_C  s    r  c                    sb   t t|  kdd  t|   }d| t| j   fddttD }| |S )Nc                   S   s   dS )NzZNumber of dimensions of repeat dims can not be smaller than number of dimensions of tensorr   r   r   r   r    r}   L  r~   zmeta_repeat.<locals>.<lambda>r6   c                    s   g | ]} | |  qS r   r   r   Zpadded_sizerE  r   r    
<listcomp>S  r~   zmeta_repeat.<locals>.<listcomp>)r   r   r1   r   ri   r   r.   )r0   rE  Znum_new_dimensionsZtarget_sizer   r  r    meta_repeatH  s    r  c                 C   s   | S r   r   rd   r   r   r    
meta_zero_W  s    r  c                 C   s   | S r   r   r   r   r   r    meta_binop_inplace\  s    r  c                 C   s   | S r   r   )r0   r   r  r   r   r    meta_binop_inplace_alphak  s    	r  c                 K   s   t | tjdS )N)Ztype_promotion)r
   r   DEFAULT)r0   kwargsr   r   r    
meta_roundw  s    r  c                 C   s   |  | jS r   r  rd   r   r   r    	meta_zero~  s    r  c                 C   s   | S r   r   r0   r  r   r   r    
meta_fill_  s    r  c                 C   s
   t | S r   r$   rp   r  r   r   r    	meta_fill  s    r  c                 C   s   | S r   r   rd   r   r   r    
meta_relu_  s    r  c                 C   s
   t | S r   r  r0   rQ  r   
accumulater   r   r    meta_index_put  s    r  c                 C   s   | S r   r   )r0   r   valuer   r   r    meta_masked_fill_  s    r  c                 C   s   | S r   r   r  r   r   r    meta_index_put_  s    r  c                 C   s   |  | jS r   )viewri   rd   r   r   r    
meta_alias  s    r  c           	         s   t |  dkdd  t | dkdd  |  }|  |d |d |d } d }||f}t  d ko~ d k fdd ||}|s|d urt | dkd	d  t | |kd
d  |S )Nr   c                   S   s   dS r^  r   r   r   r   r    r}     r~   z)common_meta_baddbmm_bmm.<locals>.<lambda>c                   S   s   dS r_  r   r   r   r   r    r}     r~   r   r5   r6   c                	      s&   d d d d  d d  d	S )Nz@Expected size for first two dimensions of batch2 tensor to be: [r  z] but got: [r   r6   z].r   r   Zbatch2_sizesbsZcontraction_sizer   r    r}     s   c                   S   s   dS )Nzself must be a 3D tensorr   r   r   r   r    r}     r~   c                   S   s   dS )NzTExpected an input tensor shape with shape {output_size} but got shape: {self.size()}r   r   r   r   r    r}     r~   )r   r1   r/   r.   )	rc  rd  Zis_bmmZself_baddbmmZbatch1_sizesZres_rowsZres_colsr?  ru  r   r  r    common_meta_baddbmm_bmm  s*    


r  c                 C   s   t | |dS )NT)r  )r0   Zmat2r   r   r    meta_bmm  s    r  c                 C   s<   | | }| | }|dkr8t |dk t |dk kr8|d8 }|S rs  )r   )re  yqrr   r   r    div_rtn  s
     r  c                 C   sZ   t | | | ||d   d |r(|d nd |d }|rV|d | | | krV|d8 }|S r>   )r  )	inputSize
kernelSizer   r   rS   r   r+  Z
outputSizer   r   r    pooling_output_shape_pad_lr  s*    
	r  c                    sX   t |dkdd  t dkfdd t  d k fdd t|  |||S )Nr   c                   S   s   dS )Nzstride should not be zeror   r   r   r   r    r}     r~   z&pooling_output_shape.<locals>.<lambda>c                      s
   d  S )Nz'pad must be non-negative, but got pad: r   r   )padr   r    r}     r~   r5   c                      s   d d  S )Nz7pad should be at most half of kernel size, but got pad=z and kernel_size=r   r   r  r  r   r    r}     r~   )r   r  )r  r  r  rS   r   r+  r   r  r    r(    s    
r(  c                    sD     }tdkodkdd  t|dko6|dkdd  t|dkoP|dkdd   ddkot ddk}|tjkrt|dko|o d	dkd
d  nBt|d	kr ddkr|p|dko|oڈ d	dk fdd td 
kod 	k	
fdd tdko(dkfdd d S )Nr   c                   S   s   dS )NzCkernel size should be greater than zero, but got kH: {kH}, kW: {kW}r   r   r   r   r    r}     r~   z$pool2d_shape_check.<locals>.<lambda>c                   S   s   dS )Nz>stride should be greater than zero, but got dH: {dH}, dW: {dW}r   r   r   r   r    r}     r~   c                   S   s   dS )Nz\dilation should be greater than zero, but got dilationH: {dilationH}, dilationW: {dilationW}r   r   r   r   r    r}     r~   r6   r5   r   r   c                   S   s   dS )NzExpected 4D (batch mode) tensor expected for input with channels_last layout with optional 0 dim batch size for input, but got: {input.size()}r   r   r   r   r    r}     r~   c                      s   d    S )NzYExpected 3D or 4D (batch mode) tensor with optional 0 dim batch size for input, but got: ra  r   )r*  r   r    r}   $  r~   c                      s   d d d d  S )NzKpad should be smaller than or equal to half of kernel size, but got padW = z	, padH = z, kW = z, kH = r   r   )r.  r/  r2  r3  r   r    r}   )  s   c                      s*   d d  d d d d dS )NzGiven input size: (re  z). Calculated output size: (z). Output size is too smallr   r   )r5  r6  r4  r;  r7  r8  r   r    r}   /  s   )r1   r   r/   r$   r   )r*  r.  r/  r0  r1  r2  r3  	dilationH	dilationWr4  r5  r6  r7  r8  r   r?   r   r   )r*  r5  r6  r.  r/  r4  r;  r7  r8  r2  r3  r    r)    sB    

r)  c                 C   s:  dd }|d|\}}t t|dv dd  t|dkrD|| }	}
n|d|\}	}
|d	|\}}|d
|\}}| d}| d}| d}t| }|tjkrt |  dkdd  n0|tjkrt |  dv dd  nt ddd  t	||||	||}t	||||
||}t
| |||	|
|||||||||| |||fS )Nc                    sB   t t|dv  fdd |d }t|dkr2|n|d }||fS )Nr  c                      s   d  dS )Nzmax_pool2d: r  r   r   r  r   r    r}   <  r~   zEmax_pool2d_checks_and_compute_shape.<locals>.unpack.<locals>.<lambda>r   r6   r  r  r   r  r    r!  9  s    

z3max_pool2d_checks_and_compute_shape.<locals>.unpackr   r"  c                   S   s   dS )NzOmax_pool2d: stride must either be omitted, a single int, or a tuple of two intsr   r   r   r   r    r}   F  r~   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>r   rS   r   r   r&  rw   r4   r   c                   S   s   dS )NzMnon-empty 4D (batch mode) tensor expected for input with channels_last layoutr   r   r   r   r    r}   W  r~   )r   r   c                   S   s   dS )Nz9non-empty 3D or 4D (batch mode) tensor expected for inputr   r   r   r   r    r}   \  r~   Fc                   S   s   dS )Nz?Unsupport memory format. Supports only ChannelsLast, Contiguousr   r   r   r   r    r}   a  r~   )r   r   r/   r:   r   r$   r   r1   r   r(  r)  )r*  r   rS   r   r   r+  r!  r.  r/  r0  r1  r2  r3  r  r  r4  r5  r6  r   r7  r8  r   r   r    #max_pool2d_checks_and_compute_shape5  sb    	








r  c                    s|   t ||||||\}t|j| jkdd  | |j fdd}	|	|  |	| t|}
tj|j|j|j	|
dS )Nc                   S   s   dS )NzNexpected dtype {self.dtype} for `gradOutput` but got dtype {grad_output.dtype}r   r   r   r   r    r}     r~   z7meta_max_pool2d_with_indices_backward.<locals>.<lambda>c                    s:   t | d   t | d  t | d  d S )Nr   r5   r6   )r  )rz   r;  r?   r7  r8  r   r    _check_dim_size  s    z>meta_max_pool2d_with_indices_backward.<locals>._check_dim_sizer'  )
r  r   r+   r?   r:   r   r$   rI   ri   rE   )r   r0   r   rS   r   r   r+  rQ  r4  r  r   r   r  r    %meta_max_pool2d_with_indices_backward|  s     


r  r  c                 C   s   t | |||||\}}}|  dkr.| dnd}	t| }
|  dkrT|||g}n|	|||g}tj|| j| j|
dtj|tj	| j|
dfS )Nr   r%  r6   r   r'  )
r  r1   r/   r:   r   r$   rI   r+   rE   int64)r*  r   rS   r   r   r+  r4  r7  r8  r   r   r/   r   r   r    meta_max_pool2d_with_indices  s    

r  c           
      C   s:   |d }|rt j|t jd}nd }t j|t jd}	||	fS )Nr   r   )r$   
zeros_liker   rp   )
r   r*  ZgridZinterpolation_modeZpadding_modeZalign_cornersr[  Zinput_requires_gradZ
grad_inputZ	grad_gridr   r   r    grid_sampler_2d_backward_meta  s    
r  c                 O   s   t j| g|R i |S r   rH   )r/   Z
fill_valuer  r  r   r   r    full  s    r  c                 O   s   t jj| fi |S r   )r   rp   default)r0   r  r  r   r   r    	meta_like  s    r  c                 C   s   |t jkrt|d u dd  t jd|d u r0| jn|||d u rB| jn||d}| jrn||  | 	 | 
  n||  |  d |d |S tjj| |||||dS )Nc                   S   s   dS )Nz9memory format option is only supported by strided tensorsr   r   r   r   r    r}     r~   zzeros_like.<locals>.<lambda>r   rC   T)r+   rD   rE   rF   r   )r$   Z
sparse_coor   rI   r+   rE   Z	is_sparseZsparse_resize_and_clear_r/   Z
sparse_dimZ	dense_dimr1   Z_coalesced_r   rp   r  )r0   r+   rD   rE   rF   r   resr   r   r    r    s6    

r  c                    s     }t|dkdd t  dkr( n |   }t |kpL|k  fddt dkrnn| t }t } |    }| = | = |||S )Nr   c                   S   s   dS )Nz-select() cannot be applied to a 0-dim tensor.r   r   r   r   r    r}     r~   zmeta_select.<locals>.<lambda>c                      s   d d   d  S )Nzselect(): index z! out of range for tensor of size z at dimension ra  r   r1   r\   r0   r   r    r}   
  s   )r1   r   rT  r/   r9   rS   Zstorage_offsetZ
as_strided)r0   r1   r\   r?   r/   Znew_sizerV   Znew_storage_offsetr   r  r    meta_select  s$    
r  c                 C   s
   t | S r   r:   Zclone_preserve_strides)r0   rP   r1   r\   r   r   r    meta_select_scatter  s    r  c                 C   s
   t | S r   r  )r0   rP   r1   startendstepr   r   r    meta_slice_scatter   s    r  )r1   dim_post_exprwrap_scalarc                 C   sb   |dkr|sJ d}| }|d }| |k s2| |krNJ d|  d| d| d| dk r^| |7 } | S )Nr   r6   zdim z out of bounds (r  rP  r   )r1   r  r  r   maxr   r   r    rX   &  s    ,rX   c                 C   s   |   dkrdS | j| S rs  r  )rz   r1   r   r   r    ensure_nonempty_size2  s    r  c                    sp   t  d}t  d}t||kdd  t|D ]4 kr6tttk fdd q6d S )Nr6   c                   S   s   dS )NzDIndex tensor must have the same number of dimensions as input tensorr   r   r   r   r    r}   <  r~   z$gather_shape_check.<locals>.<lambda>c                      s$   d dj  dj  d   S )Nz!Size does not match at dimension z expected index  to be smaller than self  apart from dimension r   r   r1   r   r\   r0   r   r    r}   B  s   )r  r1   r   r   r  )r0   r1   r\   	self_dimsZ
index_dimsr   r  r    gather_shape_check7  s    r  c                    sP   t ||  }  dk}|sDt jtjk fdd t| |  |  j	S )Nr   c                      s   d j  S )Nz2gather(): Expected dtype int64 for index, but got r7   r   r\   r   r    r}   N  r~   zmeta_gather.<locals>.<lambda>)
rX   r1   r[   r   r+   r$   rk   r  r.   ri   )r0   r1   r\   Zsparse_gradwrapped_dimZis_index_emptyr   r  r    meta_gatherG  s    

r  c                 C   s   |rR| dkrdS | dkrdS | dkr(dS | dkr4dS | d	kr@d
S t ddd  d S | dkr^dS | dkrjdS t ddd  d S d S )NsumZ
REDUCE_ADDprodZREDUCE_MULTIPLYmeanZREDUCE_MEANZamaxZREDUCE_MAXIMUMZaminZREDUCE_MINIMUMFc                   S   s   dS )Nz=reduce argument must be either sum, prod, mean, amax or amin.r   r   r   r   r    r}   c  r~   z#get_operator_enum.<locals>.<lambda>addmultiplyc                   S   s   dS )Nz/reduce argument must be either add or multiply.r   r   r   r   r    r}   k  r~   )r   )reduce_use_new_optionsr   r   r    get_operator_enumU  s,    r  c                    sL   |  dkr&t|jtjk fdd |d urHt|j|jk fdd d S )Nr   c                      s
     dS )Nz"(): Expected dtype int64 for indexr   r   method_namer   r    r}   t  r~   z,scatter_gather_dtype_check.<locals>.<lambda>c                      s
     dS )Nz0(): Expected self.dtype to be equal to src.dtyper   r   r  r   r    r}   z  r~   )r[   r   r+   r$   rk   )r  r0   r\   src_optr   r  r    scatter_gather_dtype_checkp  s    



r  c                 C   s
   t | dS rR   )r  r   r   r   r    ensure_nonempty_dim~  s    r  c                    s    dkrd S tt t kdd  d}t }t|D ].}t|}| krbqJ|t|krJd} qzqJ|sd urt|D ]$}t|}|t|krd} qqd urtt t kdd  t|  fdd nt|  fdd d S )	Nr   c                   S   s   dS NzCIndex tensor must have the same number of dimensions as self tensorr   r   r   r   r    r}     r~   z%scatter_shape_check.<locals>.<lambda>FTc                   S   s   dS r  r   r   r   r   r    r}     r~   c                      s&   dj  dj  d  dj   S )NExpected index r  r  z and to be smaller than src r   r   r1   r\   r0   r  r   r    r}     s   c                      s   dj  dj  d   S )Nr  r  r  r   r   r  r   r    r}     s   )r[   r   r  r1   r   r  )r0   r1   r\   r  Zis_wrong_shaper  r   Zindex_d_sizer   r  r    scatter_shape_check  sD    

r  c                 C   s@   t ||  }td| || t| ||| |d ur<t|| d S )Nscatter)rX   r1   r  r  r  )r0   r1   r\   rP   r  r  r  r   r   r    scatter_meta_impl  s
    r  c                 C   s   t | |||d | | jS Nr  r  r.   ri   r0   r1   r\   rP   r   r   r    meta_scatter_add  s    r  c                 C   s   t | |||d | S r  r  r  r   r   r    meta_scatter_add_  s    r  c                 C   s0   t |tjr|nd }t| |||| | | jS r   )r   r$   r   r  r.   ri   r0   r1   r\   Zsrc_or_valuereducerP   r   r   r    meta_scatter  s    
r  c                 C   s(   t |tjr|nd }t| |||| | S r   )r   r$   r   r  r  r   r   r    meta_scatter_  s    	r          )querykeyr  	dropout_p	is_causalreturn_debug_maskc              	      s  t  dk fdd | d}| d}| d}| d}	|d}
| dd} |dd}|dd}|| }tj|||	f| j| jd}|||||	dd}t	|d	 d	 }tj|||ftj
| jd}tj|d tjd
d}tj|d tjd
d}|r`|	dkrdnd}t	|| }|
dkr4d}n|
dkrBd}tj||||f| j| jd}ntjd| j| jd}||||||
dd|f	S )Nr  c                      s   d  dS )NzcCan only trace _scaled_dot_product_flash_attention when dropout is set to 0 but got a dropout_p of .r   r   r  r   r    r}     r~   z0meta__scaled_dot_product_flash.<locals>.<lambda>r   r6   r5   r   r+   rE      meta@         )r   r/   	transposer$   rI   r+   rE   r  mathceilr'   r   )r  r  r  r  r  r  r  	num_headsZmax_seqlen_batch_qhead_dimZmax_seqlen_batch_kNnz_qru  Zmax_seqlen_q	logsumexpZcumulative_sequence_length_qZcumulative_sequence_length_kZblocksize_cZmax_seqlen_kZ
debug_maskr   r  r    meta__scaled_dot_product_flash  sl    








r  )rC  r  r  r  rA   r  	cum_seq_q	cum_seq_kmax_qmax_kr  r  philox_seedphilox_offsetc                 C   s   | d}| d}| d}|| }||	 }|dd}|dd}|dd}||||}||||}||||}t|}t|}t|}|||||dd}|||	||dd}|||	||dd}|||fS )Nr   r6   r   r5   )r/   r  Zreshaper$   rp   r  )rC  r  r  r  rA   r  r  r  r  r  r  r  r  r  r  r  r  r   ZNnz_kvZquery_reshapedZkey_reshapedZvalue_reshapedgrad_qgrad_kgrad_vr   r   r    'meta__scaled_dot_product_flash_backward:  s$    





r  )r  r  r  compute_log_sumexpr  c                 C   s   |  dd} | dd}| dd}| d}| d}|d}| d}| d}	|d}
tj||||
| j| jd}|rt|d d nd}tj|||ftj| jd}| dd}||fS )Nr6   r5   r   rw   r4   r      )	r  r/   r$   rI   r+   rE   r  r  r'   )r  r  r  r  r  BMr   r  KZKvr  Zlogsumexp_dimZ
logsum_expr   r   r    "meta__scaled_dot_product_efficienti  s$    





r  )rC  r  r  r  rA   r  r  c                 C   sP  |  dd} | dd}| dd}| dd}|d}|d}	|d}
|d}|d}|ol|
|	k}|rtj||	d||f|j|jd}|dd}|dd}|dd}nxtj|j|j|jd}|rtj|j|j|jdntj|j|j|jd}|rtj|j|j|jdntj|j|j|jd}| dd| dd| ddfS )Nr6   r5   r   r   r  )	r  r/   r$   rI   r+   rE   rU  ri   zeros)rC  r  r  r  rA   r  r  Zchunk_grad_outputsr  r  r   ZnHr  Zgrad_kv_needs_initchunkr	  r
  r  r   r   r    +meta__scaled_dot_product_efficient_backward  s2    




r  c                 C   s    t | ||||dd | | jS NT)r  r  r0   r1   r\   rP   r  Zinclude_selfr   r   r    meta_scatter_reduce_two  s    r  c                 C   s   t | ||||dd | S r  r  r  r   r   r    meta_scatter_reduce__two  s    r  c                 C   s   d}| D ]}||9 }q|S rR   r   )vsr  vr   r   r    multiply_integers  s    
r  c                    s   t tkfdd d  t t k fdd t tdd dd  D oltdd D fdd d d \}}||gR S )Nc                      s   d  dt  S )Nz%It is expected output_size equals to , but got size r   r   )num_spatial_dimsr?  r   r    r}     r~   z'upsample_common_check.<locals>.<lambda>r5   c                      s   d  dt  S )Nz$It is expected input_size equals to r  r  r   )expected_input_dimsr=  r   r    r}     r~   c                 S   s   g | ]}|d kqS r  r   )r   r   r   r   r    r    r~   z)upsample_common_check.<locals>.<listcomp>c                      s   d  d S )NzDInput and output sizes should be greater than 0, but got input size z and output size r   r   )r=  r?  r   r    r}     s   )r   r   all)r=  r?  r  r   Zchannelsr   )r   r=  r  r?  r    upsample_common_check  s    

*r"  c                 C   sT   t |  dkp t|  dd  dd  t|  |dd}| |jt| dS )Nr   r6   c                   S   s   dS )NzLNon-empty 3D data tensor expected but got a tensor with sizes {input.size()}r   r   r   r   r    r}     r~   z$upsample_nearest1d.<locals>.<lambda>r  r   	r   r[   r  r/   r"  r.   r   r:   r   )r*  r?  Zscalesfull_output_sizer   r   r    upsample_nearest1d  s    

r&  c           	      C   s   t |  dkp t|  dd  dd  t|  |dd}| |}t| }| j\}}}}| j	j
dkrx|dk rxtj}|j|d	}|S )
Nr   r6   c                   S   s   dS )NzLNon-empty 4D data tensor expected but got a tensor with sizes {input.size()}r   r   r   r   r    r}     r~   z$upsample_nearest2d.<locals>.<lambda>r5   r#  r   r   r   )r   r[   r  r/   r"  r.   r:   r   ri   rE   r   r$   r   r   )	r*  r?  scales_hscales_wr%  ru  r   rq   Z
n_channelsr   r   r    upsample_nearest2d  s    


r)  c                 C   sT   t |  dkp t|  dd  dd  t|  |dd}| |jt| dS )Nr   r6   c                   S   s   dS )NzLNon-empty 5D data tensor expected but got a tensor with sizes {input.size()}r   r   r   r   r    r}   	  r~   z$upsample_nearest3d.<locals>.<lambda>r   r#  r   r$  )r*  r?  Zscales_dr'  r(  r%  r   r   r    upsample_nearest3d	  s    

r*  c                 C   s   t | t j| t jdfS rf   )r$   rp   r  )r0   stabler1   Z
descendingr   r   r    	meta_sort	  s    r,  c                    s  t jdkfdd t jjkfdd dd urt jdkfdd t  kfdd t jjkfdd t jdkfd	d d
   t   k fdd t tfddfD dd  d S )Nr5   c                      s    j  dS Nz != 2rN  r   input_gatesr   r    r}   #	  r~   z%rnn_cell_checkSizes.<locals>.<lambda>c                      s   j  d j  S N != r   r   )hidden_gatesr/  r   r    r}   &	  r~   r6   c                      s    j  dS )Nz != 1rN  r   )
input_biasr   r    r}   *	  r~   c                      s      d  S r0  rr  r   )
gates_sizer3  r   r    r}   -	  r~   c                      s   j  d j  S r0  r   r   )hidden_biasr3  r   r    r}   1	  r~   c                      s    j  dS r-  rN  r   )prev_hiddenr   r    r}   3	  r~   r   c                
      s,      dd d d d  d
S )Nr1  r   z * z // z (aka rP  )r[   r/   r   )expected_prev_hidden_numelfactorr4  r/  r6  r   r    r}   7	  r~   c                 3   s   | ]}|j  j kV  qd S r   rE   )r   re  r.  r   r    r   :	  s   z&rnn_cell_checkSizes.<locals>.<genexpr>c                   S   s   dS )Nz%expected all inputs to be same devicer   r   r   r   r    r}   >	  r~   )r   r?   ri   r/   r[   r!  )r/  r2  r3  r5  r8  r6  r   )r7  r8  r4  r5  r2  r3  r/  r6  r    rnn_cell_checkSizes 	  s8    





r:  c                 C   sL   t | |||d| tj| tjd}tj|tjd}tj|tjd}|||fS )Nr   r   )r:  r$   rp   r   )r/  r2  cxr3  r5  	workspacehycyr   r   r    _thnn_fused_lstm_cell_metaB	  s
    r?  c                 C   s*  t |dk}|r,t |}|d }| jd }n4|
r:| jd n| jd }|
rR| jd n| jd }d}|rhdnd}|dkrx|n|}|r||| g}n |
r|||| gn|||| g}| |}|	| ||g}|d u rtjd| jd}n
||}||	| ||g}|rdnd}| j|tjd}|||||fS )Nr   r6   r4   r5   r9  r7   )r   ri   r.   r$   rI   rE   uint8)r*  r   Zweight_stride0Z
weight_bufhxr;  r{  hidden_sizeZ	proj_size
num_layersbatch_firstZdropouttrainbidirectionalbatch_sizesZdropout_stateZis_input_packed
seq_length
mini_batchZbatch_sizes_sumZnum_directionsZout_sizerI  ru  Z
cell_shaper>  r=  Zreserve_shapeZreserver   r   r    
_cudnn_rnnM	  s2    

rJ  c                 C   s   |r| j d n| j d }|r&| j d n| j d }|
}|rB|||gn|||g}| |}|d u rptjd| jd}n||j }|d u rtjd| jd}n||j }tjd| jtjd}||||fS )Nr6   r   r9  )rE   r+   )ri   r.   r$   rI   rE   r@  )r*  Zw0Zw1Zw2Zw3hx_Zcx_reverserG  r{  rB  rC  
has_biasesrF  rD  rE  rH  rI  Zoutput_chanelsrI  ru  r=  r>  r<  r   r   r    mkldnn_rnn_layer	  s     
rN  c                    sR   | j dkr.t dkp dk fddt n t|  dk fddt d S )Nr   r4   c                      s    d  S )Nz4: Expected reduction dim -1 or 0 for scalar but got r   r   r1   fn_namer   r    r}   	  r~   z'zero_numel_check_dims.<locals>.<lambda>c                      s    d  dS )Nz: Expected reduction dim z to have non-zero size.r   r   rO  r   r    r}   	  r~   )r?   r   rT  r/   )r0   r1   rP  r   rO  r    zero_numel_check_dims	  s    
rQ  c                    sB   |d ur$t || }t||  nt| dk fdd d S )Nr   c                      s
     dS )Nz@: Expected reduction dim to be specified for input.numel() == 0.r   r   r  r   r    r}   	  r~   z%check_argmax_argmin.<locals>.<lambda>)rX   r1   rQ  r   r[   )r  r0   r1   r   r  r    check_argmax_argmin	  s    

rR  c                 C   sD   t d| | t| j|d ur"|fnd }t| ||}| j|tjdS )Nargmaxr7   )rR  r:   rh   ri   rj   r.   r$   r  )r0   r1   rl   r   ri   r   r   r    argmax_argmin_meta	  s    rT  c                 C   s   t jd||||dS )Nr   rC   rH   )r   r+   rD   rE   rF   r   r   r    scalar_tensor	  s    
rU  c                 C   s   t ||  dd}t|dko8||  dkr4| |ndkdd  |  dkrRdn| |}t|dkol||kdd  t| j}t|dkr|||< | || j|tj	dfS )	NT)r  r   r6   c                   S   s   dS )Nzselected index k out of ranger   r   r   r   r    r}   	  r~   ztopk_meta.<locals>.<lambda>c                   S   s   dS )Nzk not in range for dimensionr   r   r   r   r    r}   	  r~   r7   )
rX   r1   r   r/   r9   ri   r   r.   r$   r  )r0   r   r1   ZlargestsortedZ	sliceSizeZtopKSizer   r   r    	topk_meta	  s    $
rW  c                 C   s   | d ur| n|}t | dkdd  | }| d urLt |  |kdd  |d urjt | |kdd  t | |kdd  t | |kdd  t | dkdd  t | |d	 |d
  d kdd  d S )Nr5   c                   S   s   dS N r   r   r   r   r    r}   	  r~   z(checkLSTMBackwardSizes.<locals>.<lambda>c                   S   s   dS rX  r   r   r   r   r    r}   	  r~   c                   S   s   dS rX  r   r   r   r   r    r}   	  r~   c                   S   s   dS rX  r   r   r   r   r    r}   	  r~   c                   S   s   dS rX  r   r   r   r   r    r}   	  r~   c                   S   s   dS rX  r   r   r   r   r    r}   	  r~   r   r6   r   c                   S   s   dS rX  r   r   r   r   r    r}   	  r~   )r   r1   r/   r[   )grad_hygrad_cyr;  r>  r<  Zdefined_gradZexp_sizer   r   r    checkLSTMBackwardSizes	  s    r\  c           	      C   s`   | d u r|d u rdS t | |||| tj|td}tj|td}|rR|jdddnd }|||fS )N)NNNr   r   F)rl   )r\  r$   rp   legacy_contiguous_memory_formatr  )	rZ  r[  r;  r>  r<  Zhas_biasZ
grad_gatesZgrad_cxZ	grad_biasr   r   r    #_thnn_fused_lstm_cell_backward_impl	  s    r^  c                    s   t jdkr$jd ||  dks:J dj d| dd   fdd	}jd ||  }jd
 | }jd | }g jd d |||R }|}|j| d}|S )Nr5   r&  r   z'Invalid input shape for pixel_shuffle: z with upscale_factor = c                 S   s   t j| t jkS r   r   r   r   r   r    r   
  s    z,meta_pixel_shuffle.<locals>.is_channels_lastc                      sN    r"t dkrtjS tjS n(jtjdr6tjS jtjdrJtjS d S r   )r   r$   r   r   r   r   r   r   r0   r   r    r   
  s    z.meta_pixel_shuffle.<locals>.pick_memory_formatrw   r4   r   )r   ri   r.   r   )r0   Zupscale_factorr   CZHrZWrrI  rA   r   r_  r    meta_pixel_shuffle	
  s     
ra  c                 C   sZ   |  | j}| |j}| |j}| |j}| |j}| |j}|||||||fS r   r  )r*  Zweight0Zweight1Zweight2Zweight3rK  Zcx_tmpru  Zhy_Zcy_Zgrad_output_r_optZgrad_hy_r_optZgrad_cy_r_optrL  r{  rB  rC  rM  rE  rF  rG  rD  r<  Zdiff_xZdiff_hxZdiff_cxZdiff_w1Zdiff_w2Zdiff_br   r   r    mkldnn_rnn_layer_backward'
  s    rb  )	out_int32rightc                C   s   t j| |rt jnt jd S rf   )r$   rp   r   r  r   )r0   Z
boundariesrc  rd  r   r   r    meta_bucketizeJ
  s    re  c                  C   s   i } dD ]*}t | }|D ]}|| vr|| | |< qq|  D ]\}}t|tsRJ |tjjj| tj	|
 dr|t d v rt| dq<|jrq<|
 dv rq<d|
 v rt|| q<d|
 v rt|| q<t|| q<d S )N)r  Zpost_autogradZpre_autogradZCompositeImplicitAutogradr  z is a CompositeImplicitAutograd op, we shouldn't register meta function for it. Instead, we should let the decomposition run and write meta kernels for the base operators.>   zaten::copy_zaten::clonezaten::empty_stridedzaten::constant_pad_ndzaten::as_strided_scatterzaten::_to_copyzaten::rot90zmkldnn::zmkl::)r   itemsr   r	   Zpy_implr$   _CZDispatchKeyr   Z%_dispatch_has_kernel_for_dispatch_keyr  r   Zis_view2_meta_lib_dont_use_me_use_register_meta_for_mkldnnimpl/_meta_lib_dont_use_me_use_register_meta_for_mkl'_meta_lib_dont_use_me_use_register_meta)Zactivate_meta_tabler   registryZopoZop_overloadr   r   r   r    activate_metaY
  s4    	rm  )F)F)T)r   )r   )FF)F)FTN)r   N)r   N)FF)N)r   r  FTN)N)Fr   FNFr4   )T)NF)r4   F)r6   )F)F)N)r   r  r  F)NNNNN)r   NNr6   )T)F)F)N)N)NNF)N)N)r  FF)F)FF)T)T)N)NN)NNN)Nr4   F)NN)NF)NNNN)r4   TT(2  r  typingr   r   r   r$   Ztorch._prims_commonr   r:   r   Ztorch._decompr   r   r   Z
torch._opsr	   Ztorch._primsr
   r   r   r   r   r   r   r   r   Ztorch._prims_common.wrappersr   rV  r   Ztorch._subclasses.fake_tensorr   Ztorch.utils._pytreer   opsr   ZlibraryLibraryrk  r#   r,   Z_fft_c2cr  rA   r3   Z_fft_r2cr<   ZrandpermZgenerator_outrB   randintrk   rK   rL   rM   ZrandrN   Z_fft_c2rrO   r_   rQ   rW   Z
unsqueeze_rZ   r`   r]   ra   r  Z	unary_outre   r1   rn   r   ro   rs   rr   rt   r   rx   r   r   r   r   r   r   Zlinalg_inv_exr   Z_linalg_svdr   Z_linalg_detr   Zreflection_pad2d_backwardZreplication_pad2d_backwardr   Zreflection_pad2dr   Z	bernoullir   Z
bernoulli_r'   r   r   r   Z_fused_moving_avg_obs_fq_helperr   r   rK  r   mmr   rj   r   r   r   r   Zconvolutionr   rg  Z
has_mkldnnrh  r   r   Z_convolution_pointwiser  binaryr  Z_convolution_pointwise_r	  Z_linear_pointwiser  r  Zhas_mklrj  r  Z_mkl_linearr  r  Z
avg_pool2dr9  r<  Zavg_pool2d_backwardr>  Z_adaptive_avg_pool2dr@  Z_adaptive_avg_pool3drB  Z_adaptive_avg_pool2d_backwardrD  Zrepeat_interleaverF  complexrJ  rL  r\   rZ  Zconvolution_backwardr\  Zaddbmmrg  Z_cdist_forwardrn  Z_embedding_bagr  Z_embedding_bag_forward_onlyr  r  Znansumr  Z	nanmedianr  Z
dim_valuesr  Zlogical_not_r  repeatr  Zzero_r  Zmul_ZScalarZdiv_Zlogical_and_Zlogical_or_Zlogical_xor_r  Zadd_Zsub_r  roundZdecimalsr  Zzeror  Zfill_r  fillr  Zrelu_r  Z	index_putr  Zmasked_fill_r  Z
index_put_r  aliasr  r  Zbmmr  r  r  r(  r)  r  Z max_pool2d_with_indices_backwardr  Zmax_pool2d_with_indicesr  Zgrid_sampler_2d_backwardr  r  Zrandint_likeZ	low_dtypeZ
randn_likeZ	rand_likeZ	full_likeZ	ones_liker  r  rU  r  Zselect_scatterr  Zslice_scatterr  rX   r  r  Zgatherr  r  r  r  r  r  Zscatter_addr  Zscatter_add_r  r  rP   r  r  Zvalue_reducer  Zscatter_r  Z#_scaled_dot_product_flash_attentionr  Z,_scaled_dot_product_flash_attention_backwardr  Z'_scaled_dot_product_efficient_attentionr  Z0_scaled_dot_product_efficient_attention_backwardr  Zscatter_reduceZtwoZtwo_outr  Zscatter_reduce_r  r  r"  r&  r)  r*  sortr+  r,  r:  Z_thnn_fused_lstm_cellr?  rJ  rN  rQ  rR  rS  ZargminrT  rU  ZtopkrW  r   r]  r\  r^  Zpixel_shufflera  rb  Z	bucketizeZ
Tensor_outre  Ztorch._refs.nn.functionalZtorch._refs.specialrm  r   r   r   r    <module>   s8  $
	






	






	











#




 


	e
'









    J,

E










f


#
     g

	

	



















!


>G








$






/	
	  Q*
  +"
8'
"

8