a
    d                    @   s  d dl Z d dlZd dlZd dlmZ d dlmZmZmZ d dl	Z	d dl
Z
d dlZ
d dlm  mZ d dlmZmZmZmZmZmZmZmZ d dlmZmZ ddlmZ dd	lmZm Z m!Z!m"Z" dd
l#m$Z$ ddl%m&Z&m'Z' ddl m(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0 ddlm1Z1m2Z2m3Z3 ddl4m5Z5m6Z6 e7e8Z9i Z:i Z;e< Z=e
j5j>Z>e
j5j?Z?e< Z@dd ZAdd ZBeAe>jCe>jDe>jEe>jFe>jGe>jHe>jIe>jJe>jKe>jLe>jMe>jNg e
jOe
jPe
jQe
jRe
jSe
jTe
jUe
jVe
jWe
jXe
jWe
jYe
jZdZ[e\dddZ]dd Z^dd Z_dd Z`edddZad d! Zbd"ejcd"fd#d$Zdd%d& Zed{d'd(Zfd|d)d*Zgede?jhdde.e
jid+d,d-Zjede?jkdde.e
jld.d/d0Zmd1d2 Zndd3ejcd"ddd"d"fd4d5Zoede>jpd"dd6d7d8 Zpede>jqd"dd6d9d: Zqede>jre>jse>jte>jue?jvgd;d< Zwexe>d=rede>jyew ede>jzddd}d>d?Zzede>j{gd~d@dAZ{ede>j|dBdC Z|ede>j}dDdE Z}ede>j~dFdG Z~ede>jdHdI Zede>jdJdK Zede>jdLdM Zede>jdddNdO Zede?jdddPdQ Zede>jdddRdS Zede>jdTdU Zede>jddede>jddede>jdddVdW Zede>jdddXdY Zede>jdddd[d\Zede>jdde fd]d^Zede>jCdddd_d`ZCede>jddadbZede>jddcddZede>jdddedf Zede>jddddgdhZede>jddddidjZede>jddddkdlZede>jdddmdn Zede>jdddodp ZddqdrZede>jddtduZdvdw Ze  dxdy Zddzd{Zede>jddd|d} Zede>jddd~d Zede>jjdddd Zede>jdd Ze ddd Zdd Zdd Zee>jZee>jZedZedZede>je
jgdd Zede>je
jgdd Zede!jjdd Zede!jjdddd Zdd Zdd Zdd ZdhZee>je ee>jHe ee>je ee>je ee>jed"d ee>je ee>je ee>j ee>j ee>je ee>j ee>j ee>jj ee>j ee>je ee>j ee>je ee>je ee>jÃ ee>jă ee>jŃ ee>jƃ ee>jǃ ee>jȃ ee>jɃ ee>jʃ ee>j˃ ee>j̃ ee>j̓ ee>j΃ ee>jσ ee>jЃ ee>jу ee>j҃ ee>jӃ ee>jԃ ee>jd"d ee>jd"d ee>jd"d ee>j؃ ee>jك ee>jڃ ee>jd"d ee>jd"d ee>jd"d ee>jd"d ee>j؃ ee>jك ee>jڃ ee>j߃ ee>j ee>j ee>j ee>j ee>j ee>jd"d ee>jd"d ee>j ee>jd"d ee>jd"d ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>jd"d ee>jd"d ee>j ee>j  ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j	d"d ee>j
 ee>j ee>j ee>j ee>j ee>jd"d ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>j ee>jd"d ee>jd"d ee>jd"d ee>j ee>j  ee>j! ee>j"d"d ee>j# ee>j$ ee>j%d"d ee>j&d"d ee>j'd"d ee>j( ee>j) ee>j* ee>j+ ee>j, ee>j-d"d ee>j. ee>j/ ee>j0d"d ee>j1d"d ee>j2 ee>j3d"d ee>j4d"d ee>j5 ee>j6d"d ee>j7d"d ee>j8d"d ee>j9 ee>j: ee>j; ee>j< ee>j= ee>j> ee>j? ee>j@d"d ee>jA ee>jBd"d ee>jC ee>jD ee>jE ee>jF ee>jG ee>jH ee>jI ee>jJ ee>jK ee>jL ee>jM ee>jNd"d ee>jOd"d ee>jP ee>jQjR ee>jSjT ee>jUjT ee>jV ee>jW ee>jX ee>jY ee>jZj[ ee>j\jT ee>j]jT ee>j^ ee>j_ ee>j` ee>ja ee>jb ee>jcjTd"d ee>jd ee>jejf ee>jgd"d eBe>jGe ede>jGe.e.e.ee\ ee\ ee\ eYee\ e\d	ddZGede>jhdd Zhede>jid dddZiexe>drede>jjei ede?jkdd Zkede>jldde\e\dddZlede>jmdddddZmdd Znede
joe>jpgdddd"dddZoede
jqdddZqede
jrdd Zsede>jtdd Ztdd Zuede>jvdddd Zvdd Zwede
jxe>jxgdddddddddZxdd Zydd Zzede>j{eyexZ{eyewdZ|ej}sede>j~eyeZ~ede>jeyeZddÄ Zede>jdddddĜddƄZede>jdddddĜddȄZede>jdddddĜddʄZede?jjTdd̄ Zede
je>jgdd΄ Zede>jddddЄ Zede>jddddd҄ZddԄ Zede>jddddք Zede>jgddd؄Zddڄ Zdd܄ Zede>jdddddބZede>jdddddZede>jdde\dddZdd3de\eeYdddZede>jdddde\edddZede>jdde\dddZede>jdde\dddZede>jdde\dddZee>jZede>jddd3de\eYdddZdee e\dddZede>jjTdee dddZede>jMjTdee ee dddZMede>jjTdee ee ee dd dZede>jNjTdeYee ee dddZede>jdd Zede>jdd Zede?jjTd	d
 Zede>jdddddZe	jdddZe	je	jdddZe	je	jdddZdd Zdd Zee>jIZede>jIdddddZIee>jJZede>jJdddd ZJdd Zdd  Zee>jZede>jd!d" Zede>jjTdd#d$Zee>jDZede>jDdddd&d'ZDee>jEZede>jEdddd(d)ZEd*d+ Zded,d-d.Zede>jdddd/d0Zd1d2 Zede>je?jgddd"d3d4d5Zede>jddd"d3d6d7Zd8d9 Zegd:d; Zd<d= Zede>jd3d>d?d@ ZdAdB Zede>jdCdD Zede>jddddEdFZegdGdH ZegdIdJ Zede>jd3d>ddKdLZede>jgd3d>dMdN Zede?jgd3d>dOdP Zede>je>jjgd3ejd6eÃZede>je?jgd3d>dQdR Zede>jȃdSdT Zede>je?jgddddUdVZede>j˃edW ede>j̃edX ede>j̓edYZede>jσedZZede>jуed[e
jYd\ ede>j҃ed]e
jSd\Zede>jԃed^e
jSd\Zeoe>jd3d_d`Z֐dadb Zאdcdd Zؐee>jكZِee>jڃZڐee>jۃZeoe>j܃Zܐee>j݃Zݐee>jރZeoe>j߃Zeoe>jd3deZee>j ee>j eoe>j eoe>j eoe>jdfdg eoe>j eoe>j eoe>j ee>j ee>jZede>jejde ee>j ee>j ee>j ee>j9 eoe>jd3dh eoe>jZeoe>jZede>je ede>je eoe>j ee>j eoe>j eoe>jdidg eoe>j~ eoe>je
jYd\ eoe>jde
jYdj eoe>jde
jYdj eoe>jde
jYdj eoe>jde
jYdj eoe>jde
jYdj eoe>jde
jYdj eoe>jdd3e
jYdkZede>j dde ede>jddeoe>jdd3e
jYdk eoe>jdldd3e
jYdmZede>jdde ee>j ee>j ee>j ee>j ee>j	 ee>j
 ee>j ee>j ee>j ee>j ee>jۃ ee>j ee>j ee>j dndo Zee>jeփ ee>je ee>jjŐe ee>jje ee>je ee>je܃ ee>je݃ ede>jdpdq Zede>jdrds Zede>jdtdu Ze D ]"\ZZedeee qede>jdvdw Z ede
j5j!j"dxdy Z#ddzlm$Z$ ee$ dS (      N)Iterable)ListOptionalTuple)canonicalize_dimsdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator   )import_submodule   )configir	overridestest_operators)current_device)decompositionsget_decompositions)	
ExpandViewIndexingConstantPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)ceildivdeveloper_warningsympy_product)opsVc                 C   sX   t | tttfrdd | D S t|  t | tjjrT| 	 D ]}tt
| | q>d S )Nc                 S   s   g | ]}t |qS  )add_needs_realized_inputs.0xr(   r(   a/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_inductor/lowering.py
<listcomp>5       z-add_needs_realized_inputs.<locals>.<listcomp>)
isinstancelisttuplesetneeds_realized_inputsaddtorch_opsOpOverloadPacket	overloadsgetattr)fnoverloadr(   r(   r-   r)   3   s    
r)   c                 C   s8   t | tjjr,|  D ]}|tt| |< qn|t| < d S N)r0   r6   r7   r8   r9   layout_constraintsr:   )r;   
constraintr<   r(   r(   r-   add_layout_constraint<   s    r@   )r   r   r                     	   
         dtypec                 C   s2   t | ts| S | tv s&J d|  dt|  } | S )Nzid z missing from DTYPE_ID_LOOKUP)r0   intDTYPE_ID_LOOKUPrK   r(   r(   r-   decode_dtypen   s
    
rO   c                 C   s0   t | tr"t|  p t|  S t | tS d S r=   )r0   r    r   	get_dtyper
   rM   r,   r(   r(   r-   is_integer_typev   s    
rR   c                 C   s$   t | trt|  S t | tS d S r=   )r0   r    r
   rP   boolrQ   r(   r(   r-   is_boolean_type}   s    
rT   c                 C   sP   | d u rt djS t| tr(t | } | jdkrL| jd u rLt jdt dS | S )N        cudaindex)r6   tensordevicer0   strtyperX   r   )rZ   r(   r(   r-   decode_device   s    

r]   type_promotion_kindc                    s0   dd   fdd|D }t |d| i\}}|S )Nc                 S   sD   t | tr| S t| dsJ t|  }tjdg| |  dS d S )NrP   r   rK   )r0   r   hasattrlenget_sizer6   zerosrP   )inpdimr(   r(   r-   construct_input   s
    
z+get_promoted_dtype.<locals>.construct_inputc                    s   g | ]} |qS r(   r(   )r+   argrf   r(   r-   r.      r/   z&get_promoted_dtype.<locals>.<listcomp>r_   )r   )r_   argsZinps_rL   r(   rh   r-   get_promoted_dtype   s    	rk   c                    s   t  fddt ttfs4 g nt  t D ]<}t|tjjrD| D ] }t	||}|t
vr^ | q^qDt
fdd D  S )a  
    Add a lowering to lowerings dict

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c            
   
      s  t   d}t dkr6t d t tfr6d} d  dd t D }tdd | D rfJ d	td
d | D rtdd D sJ srJ|rJrt	j
}ndd  D }t|di}g }tt D ]j}||v r|t | | qt | tjr6|t | j| |d    q| |  q| |rV g r|rt|t fdd|D  D ]\}}| |< q~tt D ]<}t | tjrt | t  |d    |< q i |}	t|	 |	S )NFr   r   Tc                 S   s   g | ]\}}t |tr|qS r(   r0   r    r+   ir,   r(   r(   r-   r.      r/   z7_register_lowering.<locals>.wrapped.<locals>.<listcomp>c                 s   s   | ]}|d kV  qdS )outNr(   r*   r(   r(   r-   	<genexpr>   s   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>zout= ops aren't yet supportedc                 s   s   | ]}t |tV  qd S r=   rl   r*   r(   r(   r-   rp      r/   c                 s   s   | ]}|t v V  qd S r=   )	fallbacksr+   r;   r(   r(   r-   rp      s   c                 S   s$   g | ]}t |tst|d r|qS rP   )r0   r   r`   )r+   ar(   r(   r-   r.      s   r_   c                    s   g | ]} | qS r(   r(   r+   rn   ri   r(   r-   r.      r/   )r1   ra   r0   r2   	enumerateanykeysvaluesallr6   rS   rk   rangeappendto_dtyper   Constantvalue
get_devicezipbroadcast_tensorsr   createrb   r!   )
ri   kwargsunpackedindicesrL   Zpromoting_argsnew_argsrn   r,   ro   )aten_fn	broadcastconvert_input_to_bool	decomp_fnr_   rv   r-   wrapped   s`    
$z#_register_lowering.<locals>.wrappedc                    s   i | ]
}| qS r(   r(   rr   )r   r(   r-   
<dictcomp>   r/   z&_register_lowering.<locals>.<dictcomp>)	functoolswrapsr0   r1   r2   r6   r7   r8   r9   r:   	loweringsr}   update)r   r   r   r_   r   r;   r<   Zother_fnr(   )r   r   r   r   r_   r   r-   _register_lowering   s    :
r   Fc                 C   s   t jt| |||dS )z+
    Shim to support decorator syntax.
    r   r_   r   )r   partialr   )r   r   r_   r   r(   r(   r-   register_lowering   s    	r   c                 C   s   g }t jt| t|tddD ]v\} }|dkr>||  q"| dkrR|| q"tjj	| | t
t|jt
t| jk r|| q"||  q"tt|S )z
    Broadcasting logic based on symbolic shapes.

    We give the shapes 0 and 1 concrete values, while all other shapes
    are symbolic sympy formulas.
    r   )	fillvalue)	itertoolszip_longestreversedsympyIntegerr}   r'   graphsizevarsguard_equalsra   expandZfree_symbolsr2   )rt   boutputr(   r(   r-   broadcast_symbolic_shapes  s     r   c                    s   t dd | D s| S tdd | D rN|p:t| dtji  fdd| D S tdd | D }g }| D ]r}t|ttfr|	t
t|| | t|  qht|tjr|	t|| |  qh|	| qh|S )Nc                 s   s    | ]}t |tjttfV  qd S r=   )r0   r   ExprrM   floatr*   r(   r(   r-   rp   !  r/   z$promote_constants.<locals>.<genexpr>c                 s   s   | ]}t |ttfV  qd S r=   )r0   rM   r   r*   r(   r(   r-   rp   #  r/   r_   c                    s   g | ]}t | td qS r=   )r   r   r]   r*   rK   r(   r-   r.   '  r/   z%promote_constants.<locals>.<listcomp>c                 s   s    | ]}t |ttfr|V  qd S r=   )r0   r    r   r*   r(   r(   r-   rp   (  r/   )rx   r{   rk   r	   DEFAULTnextr0   rM   r   r}   r   r   r   r   rP   r   r1   rb   r   r   r   )inputsoverride_return_dtypeexro   r,   r(   rK   r-   promote_constants   s,    r   c                    s*   d dt t d fdd}|S )N)alpha)r   c              	      sL  t |	}r:| d urF| dkrFt|}t|d | |d< n| d u sFJ dd |D |d  	pn|d   t|d  jdk|dd  D ]B}t|t	j
stt| ksJ d d d|  q fd	d
}s0d }|D ] }| jdkr| } qq|s0|d  }p8|}tj| |dS )Nr   c                 S   s   g | ]}|  qS r(   make_loaderr*   r(   r(   r-   r.   I  r/   z1make_pointwise.<locals>.inner.<locals>.<listcomp>r   rV   zndim mismatch  c                    s   t  t ks$J d  d tjkrLd urL fddD  S rtrttjkrt fddD  S  fddD  S d S )Nzwrong ndim r   c                    s   g | ]}| qS r(   r(   r+   loadrW   r(   r-   r.   V  r/   zCmake_pointwise.<locals>.inner.<locals>.inner_fn.<locals>.<listcomp>c                    s   g | ]}| qS r(   r(   r   rW   r(   r-   r.   X  r/   c                    s   g | ]}| qS r(   r(   r   rW   r(   r-   r.   Z  r/   )ra   r6   rS   float64rW   )rL   r;   is_cudaloadersoverride_fn_when_cuda_float64override_fn_when_input_boolrangesrW   r-   inner_fnS  s    $z/make_pointwise.<locals>.inner.<locals>.inner_fnrZ   rL   r   r   )r   r1   mulrb   rP   r]   r   r\   r0   r   BaseConstantra   r   r   )r   r   otherr   rZ   rn   allow_alphar;   override_devicer   r   r   )rL   r   r   r   r-   innerA  s@    
	
zmake_pointwise.<locals>.inner)r   r    )r;   r   r   r   r   r   r   r(   r   r-   make_pointwise9  s    &-r   )r,   rL   c                    s,   |    kr| S  fdd}t| d| S )Nc                    s   t |  S r=   )r&   r~   rQ   rK   r(   r-   	_to_dtypev  s    zto_dtype.<locals>._to_dtyper   )rP   r   )r,   rL   r   r(   rK   r-   r~   q  s    r~   r,   rZ   c                 C   s,   t |}|  |kr| S ttj| |S r=   )r]   r   r    r   r   Z
DeviceCopyr   r(   r(   r-   	to_device|  s    r   c                    s   t  tsJ  fdd}|S )Nc                     s   t t | i |S r=   )r:   r&   ri   r   namer(   r-   r;     s    zops_wrapper.<locals>.fn)r0   r[   )r   r;   r(   r   r-   ops_wrapper  s    r   Tc	                 C   s   |p| j }t|}	|r"td| }
|dur2t|}t|	|||rB|
nd|d}	t| |||d|	}	tt|rttt|d|d|	 |	S )z3A pointwise function that maps ops.{name} to inputsZ
libdevice_N)r   r   r   r   r   )r_   r   )__name__r   r   r   r`   primsr:   )r   r   r   r_   r   r   r   r   use_libdevice_for_f64r;   Zfn_libdevicer(   r(   r-   register_pointwise  s<    


r   )r   r_   c                    s  dd }t |ttfr"t||}t |ttfr<t||}| ||g t d  d tjd}dd t D }t|t	 fdd|D  D ]\}}| |< qt
t D ]8}t  | tjrt | t |d	    |< qt||d
 d	 t d |t d |S )Nc                  W   s
   t j|  S r=   )r&   whererv   r(   r(   r-   r;     s    zwhere.<locals>.fnr   r   r^   c                 S   s   g | ]\}}t |tr|qS r(   rl   rm   r(   r(   r-   r.     r/   zwhere.<locals>.<listcomp>c                    s   g | ]} | qS r(   r(   ru   rv   r(   r-   r.     r/   r   r   )r0   r   rM   constant_likerk   r	   r   rw   r   r   r|   ra   r   r   r   r   r1   rb   r   r~   )condrt   r   r;   rL   r   rn   r,   r(   rv   r-   r     s$    
$
&
r   c                  G   s   t | dkr*t| d ttfr*t| d  S ttdd | D d}g }| D ]H}| }t |t |ks~t	dd t
||D rt||}|| qJ|S )Nr   r   c                 S   s   g | ]}|  qS r(   rb   r*   r(   r(   r-   r.     r/   z%broadcast_tensors.<locals>.<listcomp>r(   c                 s   s2   | ]*\}}|d kr|d kp(|d ko(|d kV  qdS r   Nr(   r+   rt   r   r(   r(   r-   rp     s   z$broadcast_tensors.<locals>.<genexpr>)ra   r0   r1   r2   r   r   reducer   rb   rx   r   r   r}   )r   targetoutputsr,   sizesr(   r(   r-   r     s    
r   c                 C   s   | S r=   r(   rQ   r(   r(   r-   nop  s    r   
lift_freshc                    s   t | tsJ |d u r&tt| jS tt|  |}tt |t	sJ|fn|  fddt
|  D }||  krt| |S | S )Nc                    s,   g | ]$\}}| v r$t jj|d s|qS )r   )r'   r   r   Zmaybe_guard_equals)r+   dsdimsr(   r-   r.     s   zsqueeze.<locals>.<listcomp>)r0   r    r   r   datar   ra   rb   r3   r2   rw   viewr,   re   Z	new_shaper(   r   r-   squeeze  s    

r   c                 C   s2   t | |}t| tsJ t|ts&J |j| _| S r=   )r   r0   r    r   r,   re   valr(   r(   r-   squeeze_  s
    
r   c                 C   s2   t | rt| dtjdS td}t|tjd| S )NFrK   isinfr   rR   	full_liker6   rS   r   r   r,   r;   r(   r(   r-   r     s    r   c                 C   s2   t | rt| dtjdS td}t|tjd| S )NFrK   isnanr   r   r   r(   r(   r-   r     s    r   c                 C   s    t | r| S td}t|| S )NceilrR   r   r   r   r(   r(   r-   r     s    r   c                 C   s    t | r| S td}t|| S )Nfloorr   r   r(   r(   r-   r     s    r   c                 C   s    t | r| S td}t|| S )Nroundr   r   r(   r(   r-   r   &  s    r   c                 C   s    t | r| S td}t|| S )Ntruncr   r   r(   r(   r-   r   .  s    r   c                 C   s   t | g\} t| tjr(t| t|S t| ts6J t|ttfsHJ t| 	 t|kr`| S t
jjt| 	 }|dkr| t
jjt||  tt| jt|S Nr   )r   r0   r   r   r   r   r2   r    r1   rb   r'   r   r   	size_hintr%   
mark_reuser   )r,   r   Zx_size_productr(   r(   r-   r   6  s    r   c                 C   sL   t |}|D ]}d||< q| }t|D ]\}}|dkr&t||}q&t||S )Nr   )r1   rw   	unsqueezer   )rt   shapeZbroadcast_dimensionsr   Zbroadcast_dimensionvidxr,   r(   r(   r-   broadcast_in_dimG  s    
r   c                 C   s   t | | S r=   )r   rb   r,   yr(   r(   r-   	expand_asU  s    r   c                    s@  t |   tt krJtdgtt      t| t  } tt|  ksbJ t |  }ttD ]4}| dksJ | dkrz|| |  ||< qztdd t D rt	| |S  fdd}t
jjt }|dkr| t
jjt||  |  tj|  |  |t |dS )Nr   r   c                 s   s"   | ]\}}|d kp|d kV  qdS r   r(   r   r(   r(   r-   rp   i  r/   zrepeat.<locals>.<genexpr>c                    sx   t | t ksJ t| } tt D ]F}| dkr( | dkrTtd| |< q(t| | d | | |< q(| S Nr   r   )ra   r1   r|   r   r   r   ModularIndexing)rX   rn   Zold_sizerepeatsx_loaderr(   r-   r   l  s    zrepeat.<locals>.inner_fnr   )r1   rb   ra   r   r   r   r|   r{   r   r   r'   r   r   r   r%   r   r   r   r   r   rP   )r,   r   new_sizern   r   Zold_size_productr(   r   r-   repeatZ  s2     

r  c                 C   s2   t | tsJ t |ttfs J tt| j|S r=   )r0   r    r1   r2   r"   r   r   )r,   r   r(   r(   r-   r     s    r   c                 C   s6   t | tsJ t |ttfs J tt| jt|S r=   )r0   r    r1   r2   r   r   r   )r,   r   r(   r(   r-   permute  s    r              c                 C   s4   t | tsJ t| |d}ttj| j||||S r   )r0   r    _validate_dimr   Z	SliceViewr   r   )r,   re   startendstepr(   r(   r-   slice_  s    r  c                    sv  t |ts|f}t |ts |f} fdd|D }t  dkrJt S t|}t|}|dksl|dkr|dkr|td|dkr|dkrt t  g}t||d}t|t	  S ||krtd| d| |dd }|dd }t |d |d }	t|	||S |\t
jj   |d     fd	d
}
tj    |
  dS )z
    This is based on torch._refs.roll(), but uses ir.ModularIndexing().

    We can't use the ref here because it is based on multiple calls to
    torch.cat() that this will result in terrible code.
    c                    s   g | ]}t  |qS r(   )r  r+   r   rt   r(   r-   r.     r/   zroll.<locals>.<listcomp>r   r   z`shifts` requiredz*shifts and dimensions must align. shifts: z, dims: Nc                    s6   t | } t|   tdt| <  | S Nr   )r1   r   r   r   r   r   rW   )a_loaderre   sizer  r(   r-   r;     s
    zroll.<locals>.fnr   )r0   r   r%   rb   clonera   RuntimeErrorr   rollr1   r'   r   r   guard_static_shaper   r   r   r   rP   )rt   Zshiftsr   Z
len_shiftsZlen_dimsZflatZrolledZtail_shiftsZ	tail_dimsZfirst_dim_rolledr;   r(   )rt   r  re   r  r  r-   r    sF    	

r  c              	   C   s   t | tr"t | jtjr"| j } |   t| sDtd|  dt	| \}}t
|j|jdd |D dd |D t|pd}tt||S )Nzunrealized as_strided(z, ...)c                 S   s   g | ]}t |qS r(   r   r   r+   r   r(   r(   r-   r.     r/   zas_strided.<locals>.<listcomp>c                 S   s   g | ]}t |qS r(   r  r  r(   r(   r-   r.     r/   r   )r0   r    r   r   ZBaseViewZunwrap_viewrealizeZis_storage_and_layoutNotImplementedErrorZas_storage_and_layoutFixedLayoutrZ   rL   r   r   ZReinterpretView)r,   r  stridestorage_offsetZstorageZ
old_layoutZ
new_layoutr(   r(   r-   
as_strided  s    

r  c                 C   s$   t | tsJ t| |||j| _| S r=   )r0   r    r  r   )r,   r  r  r  r(   r(   r-   as_strided_  s    r  c                    s\   t | dkrt| d S t| d |d}t| dtji  fdd| D } ttj	| |S )Nr   r   r_   c                    s   g | ]}t | qS r(   )r~   )r+   rd   rK   r(   r-   r.     r/   zcat.<locals>.<listcomp>)
ra   r  r  rk   r	   r   r    r   ZConcatKernelr   )r   re   r(   rK   r-   cat  s    r  c                 C   s,   t ||  | }tt| |||d |S r  )r"   Zhandle_negative_indexrb   r   r  )r,   re   r   r(   r(   r-   select  s    r  c                 C   s   t | |d}tjj|  | }t|tjr<tjj|}t|t	tj
frb|g|| d |  }g }d}|D ]$}|| }|t| ||| |}qn|S Nr   r   )r  r'   r   r   r  rb   r0   r   r   rM   r   r}   r  )r,   r   re   x_sizeresultr  r  r  r(   r(   r-   split  s    r   c                 C   s   t | ||S r=   )r   )r,   r   re   r(   r(   r-   split_with_sizes  s    r!  c                 C   sJ   t | |d}tjj|  | }g }t|D ]}|t| || q.|S r   )	r  r'   r   r   r  rb   r|   r}   r  )r,   re   r  r  rn   r(   r(   r-   unbind  s    r"  c                 C   s4   t | |d}t|  }||td t| |S r  )r  r1   rb   insertr   r   r   r   r(   r(   r-   r   %  s    r   c                 C   s2   t | |}t| tsJ t|ts&J |j| _| S r=   )r   r0   r    r   r   r(   r(   r-   
unsqueeze_-  s
    
r$  c                 C   sP   t |tsJ t|  }|dk r.||| 7 }d|  krF|| k sLn J |S r   )r0   rM   ra   rb   )r,   re   offsetndimr(   r(   r-   r  6  s    r  r   c                 C   sT   t | |d}tjj|  | d }t| |d|}t| |||d }t|t|S )Nr   r   )	r  r'   r   r   r  rb   r  r   sigmoid)r,   re   Znew_lenrt   r   r(   r(   r-   glu?  s
    r(  c                  C   s  t jjrtt jjjtttddd} tt jjjjttttddd}tt jjj	jttttddd}tt jjj
tttd	d
d}tt jjj
jttttddd}tt jjjtttddd}t jjrtt jjjttttddd}n d S )N)r,   weightbiasc
           
      S   s$   t tj| |||||||||	
S r=   )r    r   r   ZConvolutionUnary)
r,   r)  r*  paddingr  dilationgroupsattrscalars	algorithmr(   r(   r-   convolution_unaryK  s    z5register_onednn_fusion_ops.<locals>.convolution_unary)r,   r   r)  r*  c                 S   s*   t tj| |||||||||	|
||S r=   )r    r   r   ZConvolutionBinaryr,   r   r)  r*  r+  r  r,  r-  Zbinary_attrZbinary_alphaZ
unary_attrZunary_scalarsZunary_algorithmr(   r(   r-   convolution_binaryg  s"    z6register_onednn_fusion_ops.<locals>.convolution_binaryc                 S   s*   t tj| |||||||||	|
||S r=   )r    r   r   ZConvolutionBinaryInplacer2  r(   r(   r-   convolution_binary_inplace  s"    z>register_onednn_fusion_ops.<locals>.convolution_binary_inplace)r,   wr   c              
   S   s   t tj| |||||S r=   )r    r   r   ZLinearUnary)r,   r5  r   r.  r/  r0  r(   r(   r-   linear_unary  s    z0register_onednn_fusion_ops.<locals>.linear_unary)r,   r   r5  r   c              	   S   s   t tj| ||||S r=   )r    r   r   ZLinearBinary)r,   r   r5  r   r.  r(   r(   r-   linear_binary  s    z1register_onednn_fusion_ops.<locals>.linear_binaryc                 S   s&   t tj| |||||||||	|
S r=   )r    r   r   ZConvolutionTransposeUnary)r,   r)  r*  r+  output_paddingr  r,  r-  r.  r/  r0  r(   r(   r-   convolution_transpose_unary  s    z?register_onednn_fusion_ops.<locals>.convolution_transpose_unary)r,   packed_worig_wr   c                 S   s.   t tj| |||}|d ur*t||}|S r=   )r    r   r   ZMKLPackedLinearr5   )r,   r:  r;  r   Z
batch_sizer  r(   r(   r-   mkl_packed_linear  s    
z5register_onednn_fusion_ops.<locals>.mkl_packed_linear)r6   Z_CZ
has_mkldnnr   r&   ZmkldnnZ_convolution_pointwiser    binaryZ_convolution_pointwise_Z_linear_pointwiseZ _convolution_transpose_pointwiseZhas_mklZmklZ_mkl_linear)r1  r3  r4  r6  r7  r9  r<  r(   r(   r-   register_onednn_fusion_opsH  sH    
!!
r>  c                    s   t    fdd}|S )Nc                     s$   t tjtjj g| R i |S r=   )pytreetree_mapr    r   r   FallbackKernelr   kernelr(   r-   handler  s    z!fallback_handler.<locals>.handler)rq   r5   )rC  rD  r(   rB  r-   fallback_handler  s    
rE  c                 C   sb   | t vsJ d|  t| gr4|r4td|  d t|  |d urNt| | t| d dt| S )Nz.both a fallback and a decomp for same kernel: zmake_fallback(z1): a decomposition exists, we should switch to itr^   )r   r   r$   r)   r@   r   rE  )rC  Zlayout_constraintwarnr(   r(   r-   make_fallback  s    

rG  c              	   C   sB   t jsJ d|r0ttjtjtj	| ||S | t
| tjdfS )N?this should be handled in decomps unless config.fallback_randomrK   )r   fallback_randomr?  r@  r    r   r   rA  atennative_dropout	ones_liker6   rS   )r,   pZtrainr(   r(   r-   rK    s    rK  c                 G   s<   t jsJ d|   tj|   tj| g|R   | S NrH  )	r   rI  r  r'   r   realize_users_ofget_namer   ZInplaceBernoulliFallbackr,   ri   r(   r(   r-   
bernoulli_  s    rR  c                 G   s"   t jsJ dtt| g|R  S rN  )r   rI  rR  r  rQ  r(   r(   r-   bernoulli_p  s
    rS  c                 C   s
   t  d S r=   )AssertionError)rj   r(   r(   r-   _foobar(  s    rU  c                 C   s   t d d S )Nz1using triton random, expect difference from eager)r$   )Zsaltr(   r(   r-   _warn_triton_random-  s    rV  c                   C   s   t tjj d S r=   )rV  r'   r   Zcreation_timer(   r(   r(   r-   warn_triton_random2  s    rW  c                    s   d dd dd d fdd
}|S )Nr   F)rL   layoutrZ   
pin_memorymemory_formatc                    s   t   |rJ |dtjfv s J |d tjfv s2J t|} pDt  t|dkrtt|d tt	tj
frtt	|d }dd |D }tjt|}tj| |tj||d tj|  fdd}tj| |t|dS )	Nr   r   c                 S   s   g | ]}t |qS r(   r  r  r(   r(   r-   r.   H  r/   z4make_rand.<locals>.rand_or_randn.<locals>.<listcomp>r%  c                    s>   g }t |t dtj}tt |t | tj S )Ni  )r&   bitwise_xorconstantr6   int32r:   
index_expr)rX   seed)rL   fn_name
random_posseed_bufferr(   r-   r   U  s    z2make_rand.<locals>.rand_or_randn.<locals>.inner_fnr   )rW  r6   stridedcontiguous_formatr]   get_default_dtypera   r0   r1   r2   Sizer'   r   Zincrement_randomness_offsetr%   r   r  FlexibleLayoutcontiguous_stridesmake_indexerrandom_seed_bufferr   r   r   )rL   rX  rZ   rY  rZ  r  r%  r   ra  )rL   rb  rc  r-   rand_or_randn8  s2    "

z make_rand.<locals>.rand_or_randnr(   )ra  rm  r(   rl  r-   	make_rand7  s    /rn  randrandnc                  O   sB   t js|dd d ur$t| i |S |dd  t| i |S d S N	generator)r   rI  getfallback_randpop	fast_randr   r(   r(   r-   ro  p  s    c                  O   sB   t js|dd d ur$t| i |S |dd  t| i |S d S rq  )r   rI  rs  fallback_randnru  
fast_randnr   r(   r(   r-   rp  y  s    c                 C   s   t   tj|  S r=   )rW  r'   r   rk  r   rQ   r(   r(   r-   philox_seed_like  s    ry  c                    sl   |   }|   |  }tj| |tj|t|d	 |
  fdd}tj| |t|dS )Nr[  c                    s    t g t | tj S r=   )r&   ro  r_  r6   r^  rW   rL   rb  Zseed_loaderr(   r-   r     s
    z"philox_rand_like.<locals>.inner_fnr   )r   rP   rb   r   r  rh  ri  r   r   rj  r   r   r   r1   )r,   r`  r%  rZ   r  r   r(   rz  r-   philox_rand_like  s$    

r{  c                 O   s$   t tjdd ||f\}}||fS )Nc                 S   s   t j| S r=   )r   ExternKernelZrequire_stride1tr(   r(   r-   <lambda>  r/   zrequire_dense.<locals>.<lambda>r?  Ztree_map_onlyr   IRNoderj   ri   r   r(   r(   r-   require_dense  s    r  c                 O   s$   t tjdd ||f\}}||fS )Nc                 S   s   t j| S r=   )r   r|  require_contiguousr}  r(   r(   r-   r    r/   z$require_contiguous.<locals>.<lambda>r  r  r(   r(   r-   r    s    r  c                    sB   dd   fddt |jD } fdd| D }||fS )Nc                 S   s2   t | tjr.t|jd  }tj| |S | S )Nr   )r0   r   r  Zget_stride_ordermetar  r|  require_stride_order)rg   fx_argstride_orderr(   r(   r-   apply_constraint  s    z1constrain_to_fx_strides.<locals>.apply_constraintc                    s   g | ]\}} ||qS r(   r(   )r+   rg   r  )r  r(   r-   r.     r/   z+constrain_to_fx_strides.<locals>.<listcomp>c                    s"   i | ]\}}| |j | qS r(   )r   )r+   kr   r  fx_noder(   r-   r     r/   z+constrain_to_fx_strides.<locals>.<dictcomp>)r   ri   items)r  ri   r   r(   r  r-   constrain_to_fx_strides  s    r  ztorchvision::roi_align)rF  )	r,   r)  r*  r  r+  r,  
transposedr8  r-  c	                 C   s   t dd | ||fD }	ttj| ||	r.|nd ||||||	}
|	s|d urt| d }|
 d|  }t||g|dg  }t|
|}
|
S )Nc                 s   s$   | ]}|d ur|  jdkV  qd S )Ncpu)r   r\   )r+   inputr(   r(   r-   rp     s   zconvolution.<locals>.<genexpr>r   r   r   )	r{   r    r   r   ZConvolutionra   rb   r   r5   )r,   r)  r*  r  r+  r,  r  r8  r-  Zis_cpur  Zkernel_dimsZout_chanr(   r(   r-   convolution  s,    

r  c              
   C   s   t | ||||||||	S r=   )r  )r,   r)  r*  r  r+  r,  r  r8  r-  Z	benchmarkZdeterministicZcudnn_enabledZ
allow_tf32r(   r(   r-   _convolution  s    r  )rZ  c                C   s&   t j|  |  |  t|  dS Nr   )r   r   r   rP   r   r1   rb   )r,   rZ  r(   r(   r-   r    s    
r  lift_fresh_copyc                   s(    fdd}t jt| || gdS )Nc                    s   t j| d    dS )Nr   rK   )r&   r_  rW   rL   r  r  r(   r-   r;     s    ziota.<locals>.fnr   )r   r   r]   )lengthr  r  rL   rZ   Zrequires_gradr;   r(   r  r-   iota  s    
r  )re   rX   c                    s   |   |  ksJ |  t|  d dk r@|     tjjd tjj|     t	t
| |  }|  fdd}tj|  |   |t|  dS )Nr   c              	      s6   t t t |   tjt tj| | S r=   )r&   r   eqr_  r6   r^  r   re   rX   
src_loaderr   r(   r-   r     s    z select_scatter.<locals>.inner_fnr   )rP   r   r  rb   r'   r   r   Z	guard_leqZguard_ltr   r   r   r   r   r1   )r,   srcre   rX   r   r(   r  r-   select_scatter  s     

r  c                    s    |  ksJ  t d    d urLdk rL d urddk rd d u rpdd u stjj   rt }t	
t t| < t||}|  fdd}tj   |t dS )Nr   c                    s8  dkr kr dkr | S t |  tj}t|  t|    < g }dkr|t |t t	
tj kr|t |t t	
tj dkr|t t t|   dtjt dtj |sJ tt j|}t | fddtr dnd}t ||| S )Nr   r   c                      s    S r=   r(   r(   )src_idxr  r(   r-   r  ;  r/   z1slice_scatter.<locals>.inner_fn.<locals>.<lambda>rU   )r&   r_  r6   r^  r1   r   FloorDivr}   ger   r   ltr  r   r]  r   r   and_maskedrR   r   )r   Zidx_dimmaskZsrc_valre   Zdim_sizer  r  r  r  r,   r   )r  r-   r     sR    zslice_scatter.<locals>.inner_fnr   )rP   r   r  rb   r'   r   r   Zmaybe_guard_leqr1   r   r  r   r   r   r   r   )r,   r  re   r  r  r  Zsrc_sizer   r(   r  r-   slice_scatter  s.      
.
r  c                 C   s*   t | ttfr&t| dkr&t| d S | S r   )r0   r1   r2   ra   _unwraprQ   r(   r(   r-   r  L  s    r  rL   rZ   rX  rY  c                   s   |d t jfv sJ |du sJ tt tr8p4t jnpBt  t ttfrfg } fdd}nbt dkst d ttfrt dkrt	
t g} fdd}ntjt j |dS tjt|||dS )	NFc                    s   t  S r=   r&   r]  rW   r   rL   r(   r-   r   ^  s    ztensor.<locals>.inner_fnr   rF   c                    s8    fdd t dkr*tdS  dt S )Nc              	      sr   | |k sJ ||  dkr(t |  S ||  d |  }t t t d tjt |tj | | ||S )Nr   r   r   )r&   r]  r   r  r_  r6   int64)r  r  Zmid)binary_searchr   rL   rX   r(   r-   r  f  s    z/tensor.<locals>.inner_fn.<locals>.binary_searchr   )ra   r&   r]  rW   r  )r  rX   r-   r   e  s    rL   rZ   r   )r6   rd  r0   r  rM   r  rf  r   ra   r   r   r'   r   Zadd_tensor_constantrY   r   r   r]   )r   rL   rZ   rX  rY  r   r   r(   r  r-   rY   R  s(    *rY   c                 C   s@   t | tr2|d urt| |} |d ur.t| |} | S t| ||dS )Nr  )r0   r    r~   r   rY   )r   rL   rZ   r(   r(   r-   	as_tensor  s    


r  c                 C   s   t | tjdS )NrK   )rY   r6   r  r   r(   r(   r-   long_tensor  s    r  c                 C   s   t  S r=   )r   ZDynamicScalarr  r(   r(   r-   _local_scalar_dense  s    r  c                    s   | t | ttfs"tdr"jt tttjfrD fdd}n(t dksXJ 	 fdd}t
j| |t|dS )Nr   c                    s   t  S r=   r  rW   )rL   r   r(   r-   r     s    z_full.<locals>.inner_fnr   c                    s    g S r=   r(   rW   )value_loaderr(   r-   r     s    r   )r0   rM   r   r`   r   r   r   ra   rb   r   r   r   r1   )
fill_valuerZ   rL   r  r   r(   )rL   r   r  r-   _full  s    r  c                 K   s   t t|| fi |S r=   create_tensor_liketensor_constructor)r,   r  r   r(   r(   r-   r     s    r   c                    s    d d d ddd d fdd
}|S )Nr   F)namesrL   rZ   rX  rY  rZ  c                    s   | d u sJ |rJ |dt jfv s&J |d t jfv s8J t|}|pJt  }t|dkrzt|d ttt j	frzt|d }dd |D }t
 |||S )Nr   r   c                 S   s   g | ]}t |qS r(   r  r  r(   r(   r-   r.     r/   z5tensor_constructor.<locals>.inner.<locals>.<listcomp>)r6   rd  re  r]   rf  ra   r0   r1   r2   rg  r  )r  rL   rZ   rX  rY  rZ  r  r  r(   r-   r     s    	"z!tensor_constructor.<locals>.innerr(   )r  r   r(   r  r-   r    s    r  )r  rL   rX  rZ   rY  rZ  c                 G   sh   | d u sJ |d t jfv sJ t|}t|dkrTt|d ttt jfrTt|d }t|d ||||dS )Nr   r   rL   rX  rZ   rY  )	r6   re  r]   ra   r0   r1   r2   rg  empty_strided)r  rL   rX  rZ   rY  rZ  r  r(   r(   r-   empty  s    
"r  c                    s   dddddd fdd
}|S )zZ
    Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
    Nr   F)rL   rZ   rX  rY  rZ  c                   s^   |rJ |dt jfv sJ |d u r,|  }nt|}|p>|  }t|  } |||||dS )Nr   r  )r6   rd  rP   rO   r   r1   rb   )r,   rL   rZ   rX  rY  rZ  r  creation_fnr(   r-   _constant_like  s    

z*create_tensor_like.<locals>._constant_liker(   )r  r  r(   r  r-   r    s    
r  c                 C   s   t t| S r=   r  r  r(   r(   r-   r     s    r   c                    s   d d d d d fdd
}|S )Nr  c                   sd   t |ttfsJ |rJ |r,|tjks,J t|p:|  }|pF|  }dd |D }t |||S )Nc                 S   s   g | ]}t |qS r(   )r   r   r  r(   r(   r-   r.     r/   z7new_constant.<locals>._new_constant.<locals>.<listcomp>)	r0   r1   r\   r6   rd  rO   rP   r   r  r,   r  rL   rX  rZ   rY  r  r(   r-   _new_constant  s    z#new_constant.<locals>._new_constantr(   )r  r  r(   r  r-   new_constant  s    r  r  c                C   s4   |d u r|   }|d u r |  }t|d ||||dS Nr  rP   r   r  r  r(   r(   r-   	new_empty  s    r  c                C   s   t | ttfsJ t |tttd fs*J |r2J |rD|tjksDJ t|pRt }|pbtdj}t	d||| d}|
  |jj}dgt|  |j_t |tjsJ dd | D } |rdd |D n
tj| }tj||| |d|_|S )NrU   r   )r  rZ   rL   r  c                 S   s   g | ]}t |qS r(   r  r  r(   r(   r-   r.   1  r/   z!empty_strided.<locals>.<listcomp>c                 S   s   g | ]}t |qS r(   r  r  r(   r(   r-   r.   3  r/   )rZ   rL   r  r  )r0   r1   r\   r6   rd  rO   rf  rY   rZ   r  r  r   ra   r   r   ComputedBufferrh  ri  r  rX  )r  r  rL   rX  rZ   rY  Z	pointwisebufferr(   r(   r-   r  !  s.    
r  c                C   s4   |d u r|   }|d u r |  }t||||||dS r  r  )r,   r  r  rL   rX  rZ   rY  r(   r(   r-   new_empty_strided@  s    r  c                 C   s2   dd |D }t tt||jd}tj| |S )Nc                 S   s   g | ]}t jj|qS r(   )r'   r   r   r   r  r(   r(   r-   r.   O  r/   z copy_strided.<locals>.<listcomp>)key)sortedr|   ra   __getitem__r   r|  r  )r,   r  r  r(   r(   r-   copy_stridedM  s    r  c                 K   s   t || fi |S r=   )r  )r  r  r   r(   r(   r-   fullT  s    r  c                    sz   t | tsJ | tjks J t|  dk}t|  | |  |  fdd}t	j
|  |  || dS )Nr   c                    s.   t | } t| dkr&t| |  < | S r   )r1   ra   r&   indirect_indexingr  re   index_loaderr   r(   r-   r;   c  s    zgather.<locals>.fnr   )r0   r    rP   r6   r  ra   rb   r  r   r   r   r   )r,   re   rX   r%  r;   r(   r  r-   gatherY  s    r  c                    s   |rJ t | tsJ t |ts$J dt| v s8J |  |  t| g | |  dd   fdd}tj| 	 |  |dS )NrM   r   c                    sV   t | t ks"J |  d  | d  }t|gg | d   }|S )Nz != )ra   r&   r  )r   Z	var_indexZ
weight_idxindices_loaderZindices_ndimr   Zweight_loaderr(   r-   r;   }  s    "zembedding.<locals>.fnr   )
r0   r    r[   rP   r   ra   rb   r   r   r   )r)  r   Zpadding_idxZscale_grad_by_freqsparser;   r(   r  r-   	embeddingq  s    r  c           
         sB  t dd  D s*J ddd  D  tdd  D rDtddd t D }t|d	ksjJ d
d gt  }t|t fdd|D  D ]0\}}| |krtd|||< t| }qd	}t	|}|r|d d u r|
  q|r|d	 d u r|
d	 |d7 }qtdd |D r0td|| }	|||	fS )Nc                 s   s2   | ]*}|d ur|  tjtjtjtjfv V  qd S r=   )rP   r6   r  r^  rS   uint8ru   r(   r(   r-   rp     s   z.check_and_broadcast_indices.<locals>.<genexpr>z)indices must be int64, byte or bool. Got c                 S   s   g | ]}|d ur|  qS r=   rs   ru   r(   r(   r-   r.     r/   z/check_and_broadcast_indices.<locals>.<listcomp>c                 s   s*   | ]"}|d ur|  tjtjfv V  qd S r=   )rP   r6   rS   r  ru   r(   r(   r-   rp     s   zFallback for bool indicesc                 S   s   g | ]\}}t |tr|qS r(   rl   rm   r(   r(   r-   r.     r/   r   z"requires at least 1 non-None indexc                    s   g | ]} | qS r(   r(   ru   r   r(   r-   r.     r/   z.Fallback when indices is on a different devicer   r   c                 s   s   | ]}|d u V  qd S r=   r(   ru   r(   r(   r-   rp     r/   z.Fallback when None is in the middle of indices)r{   rx   r  rw   ra   r   r   r   rb   r1   ru  )
r   rZ   Z
valid_idxsZnew_indicesrn   r,   Z
output_dimstart_offsettmp
end_offsetr(   r  r-   check_and_broadcast_indices  s8    
$


r  c                    s   t |ttfsJ |  zt||  \} W n* ty\   |   tt	j
| | Y S 0 dd |D }dd |D t|d |  }g |d  |t d   fdd}tj|  |  |dS )Nc                 S   s   g | ]}|d ur|  qS r=   r   ru   r(   r(   r-   r.     r/   zindex.<locals>.<listcomp>c                 S   s   g | ]}|d ur|  qS r=   r   ru   r(   r(   r-   r.     r/   r   c                    sR   t  t ksJ  fddD }g  d  | d  }|S )Nc                    s"   g | ]}t |  qS r(   r&   r  r+   loader)r  r   r  r(   r-   r.     s   z%index.<locals>.fn.<locals>.<listcomp>ra   )r   	new_indexr  indices_loadersoutput_sizer  r   r  r-   r;     s     zindex.<locals>.fnr   )r0   r1   r2   r   r  r   r  r  rE  rJ  rX   rb   ra   r   r   rP   )r,   r   indices_sizesr  r;   r(   r  r-   rX     s6    
	rX   c                 C   s   t t| |||S r=   )
index_put_r  )r,   r   rz   
accumulater(   r(   r-   	index_put  s    r  c                 C   sB   |  |   krt||   }|r,t| |}t| t|d || S r   )r   r   r5   	mutate_tor   )selfr   r   r  r(   r(   r-   index_put_as_masked_fill  s
    
r  c                 C   s   t | ||| | S r=   )r   ZIndexPutFallback)r  r   rz   r  r(   r(   r-   index_put_fallback  s    r  c                    s,  |  dkr>t|dkr>|d  tjtjhv r>t| |||S |D ]2}|d urB| tjtjhv rBt| |||  S qB|  }t|}|  tj	tjhv r|dkrt
| dg} t| |||} |dkrt
| g } | S t||  }zt||  \} W n" ty   t| ||| Y S 0 dd |D }dd |D t| tsHJ |   tj|   |dkrvt
| dg} t|d }g |d  ||t| d  t|} fdd}	tj|  |  | |	|rdnd d	}
td t| |
}tj||_|dkr(t
| g } | S )
Nr   r   c                 S   s   g | ]}|d ur|  qS r=   r   ru   r(   r(   r-   r.     r/   zindex_put_.<locals>.<listcomp>c                 S   s   g | ]}|d ur|  qS r=   r   ru   r(   r(   r-   r.     r/   c                    sN   t  t ksJ  fddD }g  d  | d  }|S )Nc                    s"   g | ]}t |  qS r(   r  r  )r  rX   r  r(   r-   r.   .  s   z6index_put_.<locals>.output_indexer.<locals>.<listcomp>r  )rX   r  r  Zexpected_vals_sizer  r  rW   r-   output_indexer,  s     z"index_put_.<locals>.output_indexer
atomic_addrZ   rL   r   r   r  Zscatter_mode)	get_numelra   rP   r6   rS   r  r  r  rb   r  r   r~   r  r   r  r0   r    r  r'   r   rO  rP  r1   r   r   Scatterr   r  MutationLayoutregister_bufferr   )r  r   rz   r  rX   r  Zx_ndimr  r  r  scatterr  r(   r  r-   r    sx    





	

r  c                 C   s$   t | }t||||}t|| |S r=   )r  r  copy_)r  r  r  r  r  r   Zoutput_viewr(   r(   r-   as_strided_scatterI  s    
r  )re   c                 K   s   t t| |||fi |S r=   )scatter_r  )r,   re   rX   r  r   r(   r(   r-   r  Q  s    r  r   include_self)re   r   r  c                C   sH   |dvs$|dkrD|  tjtjhv rD|  t| ||||||dS d S )N>   sumNr  r  )rP   r6   rS   r  r  rE  )r;   r  re   rX   r  r   r  r(   r(   r-   scatter_fallbackV  s    r  r   )re   r   c                C   sV   |dkrd}n|dkrd}n|d u s(J t tj| ||||d}|rF|S t| ||||S )Nr5   r  multiplyprodr  )r  rJ  r  scatter_reduce_)r  re   rX   r  r   fallback_resultr(   r(   r-   r  e  s    r  c                 C   s   t t| |||S r=   )scatter_add_r  r,   re   rX   r  r(   r(   r-   scatter_addx  s    r  c                 C   s   t t| |||dS )Nr  r   r  r  r(   r(   r-   r  }  s    r  c                 K   s   t t| ||||fi |S r=   r  )r,   re   rX   r  reduction_typer   r(   r(   r-   scatter_reduce  s    r  )r  )re   r  c             	      s  |dv sJ t tj |||d}|r,|S tts:J dt| v sNJ t }|dkrnt	dgttrt dkrt	dgt|trt| dkrt	|dg}t     krt k sn J 
  tj  | ttr nd  fdd}fdd	}	d
d }
|stj  fdd| |d d}td t|}tj||_tj  |	| ||
|d}td t|}tj||_|dkrt	g S )N>   meanNamaxaminr  r  r  rM   r   r   c                    s   t | }t| | < |S r=   )r1   r&   r  )r   Zindirect_idx)re   r  r(   r-   r    s    z'scatter_reduce_.<locals>.output_indexerc                    s    r| S t   S d S r=   r&   r]  rP   r  )r  r  r  r(   r-   r;     s    zscatter_reduce_.<locals>.fnc                 S   s    | dkrdS | d u sJ d S d S )Nr  r  r(   r  r(   r(   r-   backend_reduce_str  s    z+scatter_reduce_.<locals>.backend_reduce_strc                    s   t d  S r   r  rW   )r  r(   r-   r    r/   z!scatter_reduce_.<locals>.<lambda>r  )r  rJ  r   r0   r    r[   rP   ra   rb   r   r  r'   r   rO  rP  r   r   r  r   r  r  r  r   )r  re   rX   r  r   r  r  r&  r  r;   r  Zzero_outr  r  r(   )re   r  r  r  r  r-   r     sz    
,


r   )scales_xnc           	         s   |    |  |    d  }|  d    }dd |D }t| ksRJ |}dd t||D tD ]\}rr|< qrdd  fdd}tj|  | 	 |g ||dS )	Nc                 S   s   g | ]}t jj|qS r(   )r'   r   r   r  ru   r(   r(   r-   r.     r/   z&upsample_nearestnd.<locals>.<listcomp>c                 S   s   g | ]\}}|| qS r(   r(   )r+   rn   or(   r(   r-   r.     r/   c                 S   s<   t | tj} t | t |tj} t | tj} t | S r=   )	r&   r_  r6   float32r   r]  r~   r^  r  )r,   scaler(   r(   r-   r    s    z!upsample_nearestnd.<locals>.scalec                    s@   |   d  }| d    }g |fddt |D S )Nc                    s   g | ]\}} ||qS r(   r(   )r+   rn   r   )r  r(   r-   r.   	  r/   z2upsample_nearestnd.<locals>.fn.<locals>.<listcomp>)r   )r   r,   r   r  r  scalesr   r(   r-   r;    	  s    zupsample_nearestnd.<locals>.fnr   )
realize_hintr   rb   ra   r   rw   r   r   r   rP   )	r,   r  r  r  Zi_sizesbatchZo_sizesrn   r;   r(   r  r-   upsample_nearestnd  s&    

r  )r  c                 C   s   t | ||fddS )Nr   r  r  )r,   r  r  r(   r(   r-   upsample_nearest1d	  s    r  )scales_hscales_wc                 C   s   t | |||fddS )Nr   r  r  )r,   r  r  r  r(   r(   r-   upsample_nearest2d	  s    r  )scales_dr  r  c                 C   s   t | ||||fddS )NrA   r  r  )r,   r  r  r  r  r(   r(   r-   upsample_nearest3d	  s    r  )align_cornersr  r  c              
      s   |    |  |  \}}	
|\}}tjj		tjj

dd ddd}	dd dd d	d
 fddfdd|		| ||	
| |dd  	
f
dd}
tj| 	 | 
 |
||t|t|gdS )Nc                 S   s   | t t jjkrt jS t jS r=   )r6   Ziinfor^  maxr  )maxvalr(   r(   r-   get_int_dtype5	  s    z1upsample_bicubic2d_default.<locals>.get_int_dtypec                 S   sD   |r |dkr| d |d  S dS |d ur8|dkr8d| S | | S d S r   r(   )Zin_sizeZout_sizer  r  r(   r(   r-   compute_scale:	  s    z1upsample_bicubic2d_default.<locals>.compute_scalec              	   S   s>   t |tj}|rt | |S t t | t |ddS d S )N      ?)r&   r_  r6   r  r   subr5   )r  Z	dst_indexr  Zdst_index_ier(   r(   r-   compute_source_index@	  s    z8upsample_bicubic2d_default.<locals>.compute_source_indexc                 S   s4   t t t t t |d | |d | | dS )Nr   rA         ?)r&   r5   r   r%  r,   Ar(   r(   r-   cubic_convolution1I	  s    z6upsample_bicubic2d_default.<locals>.cubic_convolution1c                 S   s@   t t t t t t || d| | d| | d| S )NrC   rF   rB   )r&   r%  r   r5   r(  r(   r(   r-   cubic_convolution2M	  s    8z6upsample_bicubic2d_default.<locals>.cubic_convolution2c                    sT   d}t | d|} | |}t d| } ||}t |d|}||||fS )Ng      r'  )r&   r5   r%  )r~  r)  Zc0Zc1Zx2c2c3)r*  r+  r(   r-   get_cubic_upsample_coefficientsS	  s    

zCupsample_bicubic2d_default.<locals>.get_cubic_upsample_coefficientsc                    sd    |}t t | d |d t t | d |d t t | d |d t | d |d S )Nr   r   r   rA   )r&   r5   r   )xsr~  cs)r.  r(   r-   cubic_interp1db	  s    *z2upsample_bicubic2d_default.<locals>.cubic_interp1dc                 S   s   t |t || S r=   )r&   maximumminimum)r   minr   r(   r(   r-   clampp	  s    z)upsample_bicubic2d_default.<locals>.clampc           
         s   | \ }}
|}t |}t ||
|}t |}t ||} 	fddt |d t |d tfdddD }tfdddD fdd	tfd
d|D }	|	|S )Nc                    s<   t | dd }t |dd } ||gS r  r  )ZfyZfxiyix)cr5  iHiWr  r   r(   r-   load_bounded~	  s    z<upsample_bicubic2d_default.<locals>.fn.<locals>.load_boundedr   c                 3   s   | ]}t  |V  qd S r=   r&   r5   r+   Zofs)r6  r(   r-   rp   	  r/   z9upsample_bicubic2d_default.<locals>.fn.<locals>.<genexpr>)r   r   r   r   c                 3   s   | ]}t  |V  qd S r=   r<  r=  )r7  r(   r-   rp   	  r/   c                    s"   t  fddD }|S )Nc                 3   s   | ]} |V  qd S r=   r(   r*   )r;  r   r(   r-   rp   	  r/   zOupsample_bicubic2d_default.<locals>.fn.<locals>.get_x_interp.<locals>.<genexpr>)r2   )r   Zcoeffs_x)r1  ixs_ofsr;  t_x)r   r-   get_x_interp	  s    z<upsample_bicubic2d_default.<locals>.fn.<locals>.get_x_interpc                 3   s   | ]} |V  qd S r=   r(   )r+   r   )r@  r(   r-   rp   	  r/   )r&   r   r%  r~   r2   )
r   ZoyZoxZreal_xZin_xZreal_yZin_yZt_yZiys_ofsZcoeffs_y)
r  r5  r&  r1  r"  height_scaler9  r:  width_scaler   )r8  r@  r7  r>  r6  r;  r  r?  r-   r;   s	  s    

z&upsample_bicubic2d_default.<locals>.fnr   )N)r  r   rb   r'   r   r   r  r   r   r   rP   r   r   )r,   r  r  r  r  NCZoHZoWr#  r;   r(   )r  r5  r&  r*  r+  r1  r.  r"  rA  r9  r:  rB  r   r-   upsample_bicubic2d_default$	  s.    
	rE  c              	      s   t |dksJ |\}}|  |  ^ } tjj  tjjdd  fdd}tj| 	 | 
 |g |t  | t | dS )NrB   c                 S   s`   t |d tj}t | tj} t | t |tj} t |t t |t | } t | S r  )r&   r]  r6   r^  r_  r%  absr  )r,   r  r%  r(   r(   r-   reflect	  s
     z!reflection_pad2d.<locals>.reflectc                    s8   | ^ }}}| }|}g |||S r=   r(   )r   r   r,   r   hleftrG  topr5  r   r(   r-   r;   	  s    zreflection_pad2d.<locals>.fnr   )ra   r   rb   r'   r   r   r  r   r   r   rP   r   r   )r,   r+  rightbotr  r;   r(   rH  r-   reflection_pad2d	  s    *rN  c                    s   t |dksJ |\ | ^ }tjjd tjjd |   fdd}tj| 	 | 
 |t| dS )NrB   r   c           
         sh  | ^  }} fdddd d	fdd	}|	 |  }}	| |  }}d 	 | d
  |  }}	||||||df |||	|
 
d f ||||d	f |||| d f ||||d	f|df |||	|d	f|
 
d f |||| d f|df |||	| d f|
 
d f S )
Nc                    s<   t t | tj} t t |tj}g  | |S r=   )r&   r  r_  r6   r^  r   )r   grad_loaderr(   r-   load_from_output	  s    z?reflection_pad2d_backward.<locals>.fn.<locals>.load_from_outputc                 S   s4   | \}}}t |tj}t t ||t ||S r=   )r&   r_  r6   r^  r  r  le)Zindex_rangern   ZlbZubr(   r(   r-   index_range_condition	  s    
zDreflection_pad2d_backward.<locals>.fn.<locals>.index_range_conditionc                    sr   |d |d k rd S |}|d urH|d |d k r8d S t ||}t | fddd}t |d S )Nr   r   c                      s
    S r=   r(   r(   )rP  out_xout_yr(   r-   r  	  r/   zKreflection_pad2d_backward.<locals>.fn.<locals>.accumulate.<locals>.<lambda>rU   )r&   r  r  r5   )rS  rT  Zindex_range1Zindex_range2r   g)gradrR  rP  )rS  rT  r-   r  	  s    z9reflection_pad2d_backward.<locals>.fn.<locals>.accumulater   r   )Nr(   )
r   r,   r   r  Zcenter_xZcenter_yZtop_reflect_xZleft_reflect_yZbot_reflect_xZright_reflect_yrM  rO  rI  rJ  rL  rK  r5  )r   rV  rR  rP  r-   r;   	  s&    "
""$z%reflection_pad2d_backward.<locals>.fnr   )ra   rb   r'   r   r   r  r   r   r   r   rP   r1   )grad_outputr,   r+  rj   r;   r(   rW  r-   reflection_pad2d_backward	  s    9
rY  c                    s:   |   |   fdd}tj|  |  |dS )Nc                    sF   t | } t| tksJ  D ]}| d | |  | |< q | S r  )r1   ra   )r   re   r   r   r   r(   r-   r  
  s
    zrev.<locals>.loaderr   )r   rb   r   r   r   rP   )r,   r   r  r(   rZ  r-   rev
  s    r[  c           	   	      s:  t |d dksJ tdd |D r*| S |  }tttt|d d d |dd d  t |t   t|d  }g t |d  D ]<\\}}}tjj	|}
| |
t|| |  qt |t |ksJ t|   fdd fdd	}|  tj|  |  ||d
S )Nr   r   c                 s   s   | ]}|d kV  qdS r   Nr(   )r+   rM  r(   r(   r-   rp   
  r/   z"constant_pad_nd.<locals>.<genexpr>r   c                    s|   g }t  d  D ]<\}\}}}|dkr<|t| |dkr|t|| qttj|}t| fddS )Nr   c                      s    S r=   r(   r(   )rX   r   r(   r-   r  3
  r/   z/constant_pad_nd.<locals>.mask.<locals>.<lambda>)	r   r}   range_mask_lowrange_mask_highr   r   r&   r  r  )rX   r  r   lowhighr  )boundsr  
mask_sizesr  r   rW   r-   r  +
  s    "zconstant_pad_nd.<locals>.maskc                    sZ   t | d  }t| d   D ]\}\}}|||  q"t|t| ksRJ |S r=   )r1   r   r}   ra   )rX   r  r   r_  r`  )ra  r  r  r(   r-   	offset_fn5
  s
    z"constant_pad_nd.<locals>.offset_fnr   )ra   r{   rb   r1   r   r   r'   r   r   r  r}   r   r   r   rP   r   r   r   r   )	r,   r+  r  r   r  r_  r`  r  rc  r(   )ra  r  r  rb  r  r   r-   constant_pad_nd
  s.    *

rd  rn   c                 C   s&   t t | tjt tdtjS r   )r&   r  r_  r6   r  r   r   re  r(   r(   r-   r]  E
  s    r]  rn   r  c                 C   s    t t | tjt |tjS r=   )r&   r  r_  r6   r  rf  r(   r(   r-   r^  L
  s    r^  c                 C   s   t t| t| |S r=   )r&   r  r]  r^  rf  r(   r(   r-   
range_maskS
  s    rg  c                    s.   |   ^ }|   fdd}|S )Nc                    s@   | ^  t t t}t | fddS )Nc                      s   g  S r=   r(   r(   )ihiwprefixr   r(   r-   r  e
  r/   z>constant_boundary_condition_2d.<locals>.load.<locals>.<lambda>)r&   r  rg  r  )rX   r  r  rI  r5  r   )rh  ri  rj  r-   r   ^
  s    z,constant_boundary_condition_2d.<locals>.loadrb   r   )r,   r  r+  rj   r   r(   rk  r-   constant_boundary_condition_2dZ
  s    	rm  c                 C   s   t | d||   || d  || d  || }|rt | d||   || d  d|| d   || }tjj|| dkrtjj|| d}n|}||fS )Nr   r   r   F)r   r  r'   r   r   r   r   )r,   rn   kernel_sizer  r+  	ceil_modeZx_outZx_altr(   r(   r-   pooling_sizej
  s    ,0rp  c                    s  dkrddgs |dks6t dd |D s6J t| tsDJ t dksTJ tdksdJ tdkstJ t|  dv sJ |   |  ^ }}t|d |\}}	td |\}
}d sd s|	s|rt| tdn| 	 t
|||
g } d  d  }|dkr:t|  ||S  fd	d
}tj|  |  tj|dd|d}tj|  tjtj|dd|d}||fS )Nr   r   c                 s   s   | ]}|d kV  qdS r   r(   r	  r(   r(   r-   rp   
  r/   z*max_pool2d_with_indices.<locals>.<genexpr>r   rA   rB   z-inf   c                    s   | ^ }}}d }d }t t d t d D ]\}}|d  | d  }|d  | d  }g |||}	|rt| | tj}
|d u r|
}ntt|	||
|}|d u r|	}q0t	|	|}q0|r|S |S d S r  )
r   productr|   r&   r_  r6   r  r   gtr2  )r   return_indexrj  bhbwr!  Zmaxindexrh  ri  r   rX   rn  r+  r  r5  r   r(   r-   r;   
  s$    $z#max_pool2d_with_indices.<locals>.fnF)ru  r   T)r{   r0   r    ra   rb   r  rp  rm  r   r   r1    fallback_max_pool2d_with_indicesr   r   r   rP   r   r   r6   r  )r,   rn  r  r+  r,  ro  r  rI  h_out
ceil_mode1w_out
ceil_mode2r   window_sizer;   Zr1Zr2r(   rx  r-   max_pool2d_with_indices
  sJ    
r  c              
      sj  dkrddgs|dks6t dd |D s6J t|tsDJ tdksTJ tdksdJ tdkstJ t| dv sJ |   z|  }W n ty   d }Y n0 t|trt|jjt	r|jj}	t
jd t
j|	 |	 |	 d|	d}
|
  |
 }n&z| }W n ty6   d }Y n0 |d urP|d dksh|d ur~|d dkr~t| ||||S |  | ^ }}	|  ^ }| |   t| }tfd	d
td d D tfdd
td d D  }|dkr2t| ||||S  	f
dd}t	j|  |  ||dS )Nr   r   c                 s   s   | ]}|d kV  qdS r   r(   r	  r(   r(   r-   rp   
  r/   z3max_pool2d_with_indices_backward.<locals>.<genexpr>r   rq  )rZ   rL   r  )r   rX  r   c              	      s8   g | ]0}t |d   t d | d   d    dqS r   r   r   r+   rI  rn  r  r(   r-   r.     s   z4max_pool2d_with_indices_backward.<locals>.<listcomp>c              	      s8   g | ]0}t |d   t d| d   d    d qS r   r   r  r+   r5  r  r(   r-   r.     s   rr  c                    sP  | ^ }}}t |	 | tj}|d  }|d  }t t|d  d  d tj}t t|d  d  d tj}t t|d d tj}t t|d d tj}t |t dtj}t |t dtj}t |t tj}t |t tj}d }	t	D ]}
t	D ]}t 
|t |
tj}t 
|t |tj}g |t t |t |t dtjt t |t |t dtj}|} |}t ||}|	d u rt ||t dtj}	n:t t t ||t |||}t |t 
|	||	}	q<q.|	d usLJ |	S Nr   r   rU   )r&   r_  r6   r^  r   r  r2  r]  r3  r|   r5   r  r%  r  r   r  r  r  )r   rj  rI  r5  Z
index_testphstartpwstartphendpwendgradientph_pw_phpwZ
grad_indexZindex_actualZ	grad_partcheckr  
rO  h_window_sizer  rn  r+  pooled_heightpooled_widthr  w_window_sizewidthr(   r-   r;   #  sb    ""



z,max_pool2d_with_indices_backward.<locals>.fnr   )r{   r0   r    ra   rb   r  
get_strideAttributeErrorr   r   r   r  rh  r   rP   Zdecide_layout)fallback_max_pool2d_with_indices_backwardr   r1   r   r|   r   )rX  r,   rn  r  r+  r,  ro  r   Z	gO_strider   Zx_bufferZx_strider  heightrj   r   r~  r;   r(   r  r-    max_pool2d_with_indices_backward
  s    
	




7r  c                    s(   |   ^ }}}|    fdd}|S )Nc              
      s   |\|\ |\}}t t t   tjt |tjt t  tjt |tj}t | fdddS )Nc                      s   g    S r=   r(   r(   )h_start_indexrh  ri  rj  w_start_indexr   r(   r-   r  x  r/   z3pad_adaptive_loader.<locals>.load.<locals>.<lambda>rU   )r&   r  r  r_  r6   r  r  )rj  Z
incrementsZstart_indicesZend_indicesh_end_indexw_end_indexr  r   )r  rh  ri  rj  r  r-   r   f  s$    z!pad_adaptive_loader.<locals>.loadrl  )r,   rj   rI  r5  r   r(   r  r-   pad_adaptive_loaderb  s    r  c                    s(   |\|\  fdd}|S )Nc                    s   | ^ }}}|} |}|}|}d }	t td td D ]<\}
}|||
|g||g||g}|	d u r||}	qLt||	}	qL|	S r  r   rs  r|   r&   r5   )r   r  rj  rv  rw  r  r  r  r  totalrh  ri  r   Zh_end_index_fnZh_start_index_fnkernel_maxesZw_end_index_fnZw_start_index_fnr(   r-   fn_sum  s"    $z)_adaptive_pooling_idx_sum.<locals>.fn_sumr(   )r  Zstart_index_fnsZend_index_fnsr  r(   r  r-   _adaptive_pooling_idx_sum  s    r  c                    s  t tsJ t|dksJ    ^ }}}tjj|}tjj|}|\}}||krr||krrt	S || dkr|| dkr|| || g}t
|S t|| d |}t|| d |}	t|||g }
 }dd }dd }tj|||d}tj|||d}tj|||d}tj|||d}||	 }|d	krLt|S t||	g||g||g tt fd
d}tj |||
d}|S )Nr   r   r   c                 S   s   t | | |S r=   r   r  rX   out_diminp_dimr(   r(   r-   start_index  s    z)_adaptive_avg_pool2d.<locals>.start_indexc                 S   s   t | d | | d |S r  r  r  r(   r(   r-   	end_index  s    z'_adaptive_avg_pool2d.<locals>.end_indexr  r  rr  c                    s   t  | t | S r=   )r&   divr  r  r  ones_loaderr,   r(   r-   r;     s    z _adaptive_avg_pool2d.<locals>.fnr   )r0   r    ra   r  rb   r'   r   r   r  r  
avg_pool2dr#   r1   rP   r   r   fallback_adaptive_avg_pool2dr  r  rL  r   r   r   )r,   r  r  Zh_inZw_inrz  r|  rn  h_kernel_maxw_kernel_maxr   rL   r  r  r  r  r  r  r~  r;   rvr(   r  r-   _adaptive_avg_pool2d  sN    


r  c                    s"      ^ }}}tjj|}tjj|}|^ }}}	|| dkrr||	 dkrrt|| ||	 gddS t||}
t||	}dd fdd}tj	||d}tj	|||d}tj	|	|d}tj	||	|d}t
|
|g||g||g  fd	d
}tj  |t|d}|S )Nr   r   )divisor_overridec                 S   s   t | | |S r=   )r   ZCeilDivr  r(   r(   r-   r    s    z0upsample_nearest2d_backward.<locals>.start_indexc                    s    | d ||S r  r(   r  )r  r(   r-   r    s    z.upsample_nearest2d_backward.<locals>.end_indexr  c                    s    | t S r=   )r  r  )r  r,   r(   r-   r;     s    z'upsample_nearest2d_backward.<locals>.fnr   )r  rb   r'   r   r   r  r  r#   r   r   r  r   r   r   rP   r1   )r,   r  Z
input_sizer  r  r  Zinp_hZinp_wZout_hZout_wr  r  r  r  r  r  r  r;   r  r(   )r  r  r,   r-   upsample_nearest2d_backward  s8    

r  r(   c                    s  ssddgt | ts"J tdks2J tdksBJ tdksRJ t|  dv sfJ |   |  ^ }}}	t|d|\}
}t|	d|\}}d sd s|s|rt| dd}n|  d}t||
|g }| 	  d d  }|dkr&t
| |||S fd	d
|sH|rH|r|rXd| ndd d    fdd}n tt| dfdd}tj|   ||d}|S )Nr   r   rq  r   rU   TFrr  c           	         s   | ^ }}}d }t t d t d D ]f\}}|d  | d  }|d  | d  }|g |||}|d u r|}q,t||}q,|S r  r  )	r   r  rj  rv  rw  r  rh  ri  r   )rn  r+  r  r(   r-   r  G  s    $zavg_pool2d.<locals>.fn_sumr'  c                    s   t | t  S r=   )r&   r   r]  r  )rL   r  r  r   r(   r-   r;   Z  s    zavg_pool2d.<locals>.fnc                    s   t  |  | S r=   r&   r  r  )r  r  r   r(   r-   r;   `  s    r   )r0   r    ra   rb   r  rp  rm  r   r1   rP   fallback_avg_pool2drL  r   r   r   )r,   rn  r  r+  ro  count_include_padr  r  rI  r5  rz  r{  r|  r}  had_paddingr   r~  r;   r  r(   )rL   r  rn  r  r+  r  r  r   r-   r    s\    



r  c                    s  |rJ 
s
sddgt | ts*J t |ts8J tdksHJ t
dksXJ tdkshJ t| dv s|J |   | ^ }td
|\}	}
td
|\}}|  d pވd p|
p||  ^ }	t| }| }t	
fddt
d d D t	
fddt
d d D  }|dkrtt| |
||S 
fd	d
  	
fdd}tj|  |||d}|S )Nr   r   rq  r   c              	      s8   g | ]0}t |d   t d | d   d    dqS r  r  r  r  r(   r-   r.     s   z'avg_pool2d_backward.<locals>.<listcomp>c              	      s8   g | ]0}t |d   t d| d   d    d qS r  r  r  r  r(   r-   r.     s   rr  c              	      sX  t d tj}t d tj}t d tj}t d tj}t d tj}t d tj}t t | ||}t t |||}	t t ||t t  tj|}
t t |	|t t tj|}t 	|t dtj}t 	|	t dtj}	t |
t  tj}
t |t tj}t t |
|t ||	}|S )z{
        This computes the scaling factor that we will divide an element
        by when `count_include_pad=False`
        r   r   )
r&   r]  r6   r^  r%  r   r3  r5   r_  r2  )r  r  Zstride_hZstride_wZpad_hZpad_wZkernel_hZkernel_wZhstartZwstartZhendZwendZdivide_factor)r  rn  r+  r  r  r(   r-   !compute_pool_size_without_padding  s,    

z>avg_pool2d_backward.<locals>.compute_pool_size_without_paddingc                    sL  | ^ }}}|d  }|d  }t t|d  	d  	d tj}t t|d  	d  	d tj}t t|	d d tj}t t|	d d tj}t |t dtj}t |t dtj}t |t tj}t |t tj}d }t	D ]}	t	
D ]}
t 
|t |	tj}t 
|t |
tj}sdsvd d  }n
 ||}t g |t t |t |t dtjt t |t |t dtj|}t t ||t ||}|d u rt ||t dtj}nt |t 
|||}q&q|d usHJ |S r  )r&   r_  r   r  r6   r^  r2  r]  r3  r|   r5   truedivr  r%  r  r  r   r  )r   rj  rI  r5  r  r  r  r  r  r  r  r  r  r  partr  )r  r  rO  r  r  rn  r+  r  r  r  r  r(   r-   r;     sh    ""



zavg_pool2d_backward.<locals>.fnr   )r0   r    ra   rb   r  rp  r   r1   rP   r   r|   fallback_avg_pool2d_backwardr   r   r   )rX  r,   rn  r  r+  ro  r  r  r  rz  r{  r|  r}  rj   r   rL   r~  r;   r  r(   )r  r  rO  r  r  r  rn  r+  r  r  r  r  r  r-   avg_pool2d_backwardq  sf    
 ;r  c                 C   s   |   }t|tr|g}n|s*tt|}t|}tt|D ]j}|| dk rn||  t|rft|nd7  < d||   krt|k s>n t|dkr|| dks>J q>tt|t|ksJ d|S )Nr   r   zreduction axis not unique)rb   r0   rM   r|   ra   r1   r3   )r,   axisr  rn   r(   r(   r-   _validate_reduction_axis  s    
 <r  )r  c                    s   dd d fdd}|S )NFrK   c          
         s  dkr,|d ur,t | ||dt| |fS dkrX|d urXt| ||dt| |fS |d urjt| |} dkr~t| tj} |  tt	| |}g }g g }g t
tD ]>}||v rֈ| ||  q| ||  q fdd}r.t}D ]}td||< qn|}|   tj|  pP|  |  |||dddd	}	t|	jjtr|	  |	S )
Nr4  rK   r   rx   c                    s   t |t ksJ rXt  t ks,J t fddD sFJ  fddD  t  t kslJ d gt  t |  }tt t|D ]\}}|||< q|S )Nc                 3   s   | ]} | d kV  qdS r\  r(   ru   rW   r(   r-   rp   F  r/   z@make_reduction.<locals>.inner.<locals>.loader.<locals>.<genexpr>c                    s   g | ]} | qS r(   r(   ru   rW   r(   r-   r.   G  r/   zAmake_reduction.<locals>.inner.<locals>.loader.<locals>.<listcomp>)ra   r{   r   chainr   )rX   Zreduction_indexr  r   varZinner_loaderkeepdimsZkept_idxZreduced_idxr  rW   r-   r  B  s    
z-make_reduction.<locals>.inner.<locals>.loaderr   )r	  r
  )rZ   Z	dst_dtypeZ	src_dtyper   r   Zreduction_rangesr  )reduce_aminreduce_argminreduce_amaxreduce_argmaxr~   r6   rS   rb   r3   r  r|   ra   r}   r1   r   r   r   r   r   r   rP   rs  r0   r   r  )
r,   r  r  rL   Z
kept_sizesZreduced_sizesrn   r  r   r  r   r  r  r-   r   $  s^    





zmake_reduction.<locals>.inner)NFr(   )r  r   r   r(   r  r-   make_reduction#  s    Er  c                   s   |d urt | |} |   t| |}|  }|tjtjfv rHt | tj} t| ||}t	 fdd|D }t
||  |  }t|t| }t t|||S )Nc                 3   s   | ]} | V  qd S r=   r(   ru   r  r(   r-   rp   w  r/   zmean.<locals>.<genexpr>)r~   rb   r  rP   r6   float16bfloat16r   sum_r%   r   r   r   r   r   r1   r  )r,   r  keepdimrL   Zoutput_dtype
sum_resultdenomr(   r  r-   r  l  s    

r  c           
         s   |d u rd}|    t| |}t| |dd}|r8|  tt| |}t|||}t fdd|D }|rt|| }t	|| 
 |  }t|t|  }t||}	|s|	S |r|nt||}|	|fS )Nr   T)r  c                 3   s   | ]} | V  qd S r=   r(   ru   r  r(   r-   rp     r/   zvar_mean_.<locals>.<genexpr>)rb   r  r  r  squarer%  r  r%   r   r   rP   r   r   r   r1   r  r   )
r,   r  
correctionr  return_meanZx_meanZdiffsr  r  Zx_varr(   r  r-   	var_mean_}  s&    

r  )r  r  c                C   s   t | |||ddS )NFr  r  r  r  r  r,   r  r  r  r(   r(   r-   var_  s    
r  c                C   s   t | |||ddS )NTr  r  r  r(   r(   r-   var_mean  s    
r  c                 C   st   |dk rt t| | |S |dkr0td|S |dkr<| S t | |d |}t||}|d dkrpt|| }|S )Nr   r   r   )pow_recursiver&   
reciprocalr]  r   )r,   r   rL   r  r(   r(   r-   r    s    r  c                 C   s   t | |S r=   )r&   powrt   r   r(   r(   r-   
pow_native  s    r  c                 C   s&   t | tjr"t|  jdkr"dS dS )NrV   TF)r0   r   r  r]   r   r\   rQ   r(   r(   r-   _is_ir_node_and_cuda  s    r  )r   c                    s&  t  r0t r0  tjtjtjfv s0J dttrTtkrTt	 tS ttrndkrnt
 S ttrdkr S ttrd  k rdk rn n6   fdd}tj    |  dS t tr dk rtdS  d	krt rtS t S )
Nz!Pow input must be floating point.r$  r   i    c                    s   t |   S r=   )r  rP   r  rt   r   r  r(   r-   r;     s    zpow.<locals>.fnr   r   )r  rP   r6   r  r  r   r0   r   rM   r  sqrtr   r   r   r   rb   r   r   r   exp2r  )rt   r   r;   r(   r  r-   r    s:    "

r  c                 C   s   t | tr| j}n| }t |tr&|j}t |tjsftj|  |  |	 | 
 dj}t |tjsfJ t |tjr| st |jtjs|  |j|_| S tj|| | S r  )r0   r    r   r   Z
StorageBoxr   r   r   rP   r   rb   Zis_input_bufferZ	NopKernelr  r  Zrealize_into)changedr   Zchanged_datar(   r(   r-   r    s.    

r  c                 C   s   t | t| |S r=   )r  r   )r,   r  r(   r(   r-   fill_  s    r  c                 C   s4   t ||  }t||  }t||  }t| |S r=   )r   r   r~   rP   r   rb   r  )dstr  Znon_blockingr(   r(   r-   r    s    r  c                 C   s   t | |S r=   )r&   floordivr  r(   r(   r-   r    s    r  c                 C   s   t | |S r=   )r&   truncdivr  r(   r(   r-   r    s    r  c                 C   s   t | ot |}t| ot|}|dkrP|r4J d|rBt| |S tt| |S |dkr|rdJ d|rrt| |S tt| |S t| |S )Nr   z5floordiv operands can not be boolean at the same timer   z5truncdiv operands can not be boolean at the same time)rR   rT   r  r   r  r  r   )rt   r   Zrounding_modeZboth_integerZboth_booleanr(   r(   r-   div_mode  s    r  c                 C   s<   t | ot |}|rt| |S ttjj}t|| |S d S r=   )rT   logical_andr   rJ  r   r   r   )rt   r   Z	both_boolr;   r(   r(   r-   r   *  s
    
r   c                 C   s4   t | pt| }|rt| |S dd }t|| |S )Nc                  W   s
   t j|  S r=   r  rv   r(   r(   r-   r;   =  s    zdiv_prim.<locals>.fn)rT   rR   r  r   rt   r   Zis_integralr;   r(   r(   r-   div_prim6  s
    
r  c                 C   s4   t | pt| }|rdd }ndd }t|| |S )Nc                 S   s   t | |S r=   )r&   modr  r(   r(   r-   r;   P  s    zfmod.<locals>.fnc                 S   s   t | |S r=   )r&   fmodr  r(   r(   r-   r;   U  s    )rT   rR   r   r  r(   r(   r-   r  J  s
    
r  c                 C   s:   |   }t|st|r&t| t } dd }t|| S )Nc                 S   s
   t | S r=   )r&   rsqrtrQ   r(   r(   r-   _rsqrta  s    zrsqrt.<locals>._rsqrt)rP   r   r
   r~   r6   rf  r   )r,   rL   r  r(   r(   r-   r  [  s
    r  c                C   sB   t |  st|  r&|d u r&tj}td|d}|| |||dS )Nr  r   rK   )r   rP   r
   r6   r  r  )r,   r  r  rL   r;   r(   r(   r-   r  g  s    

r  r   r4  r	  r
  rx   r   argmaxargmin
logical_or)r   r   c                 C   s   t | tjdS )Nr^   r   r	   INT_TO_FLOATopr(   r(   r-   register_pointwise_numeric  s    r  c                 C   s   t | tjddS )NT)r_   r   r  r  r(   r(   r-    register_pointwise_numeric_ldf64  s
    r  )r   logical_not)r   )r   identity)r_   r   )r_   r   r   r\  )r   r_   r   r   c                    s   t | d d fdd}|S )Nr^   c                     s.    | i |}t || d  }t| d |S r   )r~   rP   r  )ri   r   r  outplace_opr(   r-   r;     s    zregister_inplace.<locals>.fn)r   )Zaten_opr  r;   r(   r  r-   register_inplace  s    
r  c                 C   s   |   | S r=   r   rt   re   r(   r(   r-   sym_size  s    r  c                 C   s   |   | S r=   )r  r  r(   r(   r-   
sym_stride   s    r  c                 C   s   |   S r=   )r  r
  r(   r(   r-   	sym_numel  s    r  c                 O   s   t dd S )NzHelpful for debugging)r  )r  ri   r   r(   r(   r-   foobar  s    r  c                 C   s   |    t| S r=   )r  r  rQ   r(   r(   r-   _realize  s    r	  rB  )N)NNNNF)N)N)r   r   r  r   )N)N)r   )r   )r   )r   )r   )r   )NT)r   NNr   )NN)r   FF)F)F)N)Nr   )N)NN)NNN)NN)r   )Nr   r   F)NNNN)r(   r   FTN)N)N)NF)N)N)F)N)NF(%  r   r   loggingcollections.abcr   typingr   r   r   r   r6   Ztorch.fxZtorch.utils._pytreeutilsZ_pytreer?  Ztorch._prims_commonr   r   r   r	   r
   r   r   r   Z%torch.fx.experimental.symbolic_shapesr   r   Z_dynamo.utilsr    r   r   r   r   Zcuda_propertiesr   decompositionr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   Zvirtualizedr&   r'   	getLoggerr   logr   r>   r3   rq   rJ  r   r4   r)   r@   r  r  r  Zbmmr  Zconvolution_backwardr  r  mmZupsample_bilinear2dr  Zupsample_bicubic2dr  Zint8int16r^  r  r  r  r   Z	complex32Z	complex64rS   r  rN   rM   rO   rR   rT   r]   rk   r   r   r   r   r   r   Zconvert_element_typerL   r~   Z
device_putrZ   r   r   r   r   r   aliasdetachZdetach_ZliftZview_ofr   r`   r   r   r   r   r   r   r   r   r   r   r   r   r  Z_unsafe_viewr   Zreshaper  slicer  r  r2   r  r  r  r   r!  r"  r   r$  r  r(  r>  rE  rG  rK  rR  Z	bernoullirM  rS  rU  	lru_cacherV  rW  rn  ro  rt  rp  rw  rv  rx  ry  Z_overloadpacketr{  r  r  r  ZFALLBACK_ALLOW_LISTZ_adaptive_avg_pool2d_backwardZ
_cudnn_rnnZ_cudnn_rnn_backwardZcumsumZ_embedding_bagZ_embedding_bag_forward_onlyZ_fused_moving_avg_obs_fq_helperZ*_fused_moving_avg_obs_fq_helper_functionalZgrid_sampler_2d_backwardZrandpermsortZstableZ(_sparse_coo_tensor_with_dims_and_tensorsZ_thnn_fused_lstm_cellZtopkZupsample_bicubic2d_backwardZupsample_bilinear2d_backwardZupsample_linear1dZupsample_trilinear3dZupsample_linear1d_backwardZupsample_trilinear3d_backwardZ_adaptive_avg_pool3dZadaptive_max_pool2dZadaptive_max_pool3dZaddbmmZaddmvZaminmaxZ
avg_pool3dZ
block_diagZ_cdist_forwardZcount_nonzeroZcummaxZcumminZcumprodZdeg2radZdiagonal_copyZdiagonal_scatterZdigammadistZ_efficientzerotensorZ*_embedding_bag_per_sample_weights_backwarderfcZerfinvZfmaxZfminZfractional_max_pool2dZfractional_max_pool3dfrexpZgeqrfZhistcZi0ZigammaZigammacisinZisneginfZisposinfZkthvalueZlinalg_cholesky_exZlinalg_crossZ_linalg_detZlinalg_householder_productZlinalg_inv_exZlinalg_ldl_factor_exZlinalg_ldl_solveZ	linalg_luZlinalg_lu_factor_exZlinalg_lu_solveZlinalg_matrix_expZ	linalg_qrZ_linalg_slogdetZ_linalg_solve_exZlinalg_solve_triangularZ_linalg_svdZ
logaddexp2ZlogcumsumexpZlog_sigmoid_forwardZlogspaceZ	lu_unpackZmax_pool3d_with_indicesZmax_unpool2dZmax_unpool3dZmedianmodeZmultilabel_margin_loss_forwardZmulti_margin_lossZ	nanmedianZnansumZnarrow_copyZormqrZ_pdist_forwardZpixel_shuffleZpixel_unshuffleZ	polygammar  putZrad2degZreflection_pad1dZrenormZreplication_pad1dresizeZresize_Z	resize_asZ
resize_as_ZsearchsortedZsmooth_l1_lossZspecial_airy_aiZspecial_bessel_j0Zspecial_bessel_j1Zspecial_bessel_y0Zspecial_bessel_y1Zspecial_chebyshev_polynomial_tZspecial_chebyshev_polynomial_uZspecial_erfcxZspecial_hermite_polynomial_hZspecial_hermite_polynomial_heZspecial_i0eZ
special_i1Zspecial_i1eZspecial_laguerre_polynomial_lZspecial_modified_bessel_i0Zspecial_modified_bessel_i1Zspecial_modified_bessel_k0Zspecial_modified_bessel_k1Zspecial_ndtriZ!special_scaled_modified_bessel_k0Z!special_scaled_modified_bessel_k1Zspecial_spherical_bessel_j0Zspecial_zetaZtake	thresholdtraceZ
_trilinearZunfold_copyuniformZunsafe_splitZvdotZview_as_complexZ	view_copyZ_adaptive_avg_pool3d_backwardZadaptive_max_pool2d_backwardZadaptive_max_pool3d_backwardZavg_pool3d_backwardZbitwise_or_Z_cdist_backwardZdiagonal_backwardZ_embedding_bag_dense_backwardZfractional_max_pool2d_backwardZfractional_max_pool3d_backwardZ_linalg_check_errorsZ max_pool3d_with_indices_backwardZmultilabel_margin_loss_backwardZmulti_margin_loss_backwardZ_pdist_backwardZreflection_pad1d_backwardZreplication_pad1d_backwardZsmooth_l1_loss_backwardZsoft_margin_loss_backwardZsoftshrink_backwardZsqueeze_copyZlinalg_pinvZatol_rtol_tensorZsegment_reducedefaultZ_segment_reduce_backwardZangleZcholesky_inverseZcholesky_solveZ_fft_r2cZ	histogramZbin_ctZ_histogramdd_bin_edgesZ_histogramdd_from_bin_ctsZindex_reduceZmasked_scatterZ	to_sparseZtriangular_solveZexpand_copygcdZ_linalg_eighrc   r  Zbitwise_right_shiftr  r  r  r  r  r  r  rY   Zscalar_tensorr  Z
LongTensorr  r  r  r   r  r  r  r   Z
empty_likerL  rI  Z	rand_likeZ
randn_liker  r  r  r  r  r  r  r  r  rX   r  r  r  r  r  r  r[   r  r  r  r  r  r   Zfallback_scatter_reduce_r   r  r  r  rE  rN  rY  r[  rd  r   r]  r^  rg  rm  rp  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  Ztrue_divideZTensorr  r  r  r  r  r   r4  r	  r  r
  r  rx   r  r  r  r  r5   r  r  expr  expm1Zrelur'  r  r  r%  cossinrF  Zbitwise_andZbitwise_notZ
bitwise_orr\  Zbitwise_left_shiftlgammaerfZspecial_erflog1ptantanhr  r2  r3  Z	clamp_minZ	clamp_maxnegr  	remaindersignZsignbitrQ  r  r  rt  r  ner  __and____or__r  logical_xor__xor__coshsinhacosacoshasinasinhatan2atanatanhcopysignhypotlog10	nextafterr  Zadd_Zmul_Zdiv_ZTensor_modeZsub_Zrelu_Zsigmoid_r  r  r  r  methodfuncr  Z_inductor_testr  r	  rC  r(   r(   r(   r-   <module>   sl  (
,
	
[
    
8

*













,
9
	

	 #





3































































%



I
2














%1

T`"  
 r

J-  J 
@0    X	 #I

$


 









