a
    dp                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl m	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlmZ d dlZd d	lmZmZ d dlZd dl m!  m"Z# d d
l$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* ddl+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1 ddl-m2Z2m3Z3 ddl!m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z< ddl=m>Z>m?Z? e@eAZBejejCddZCej>jDZDdd ZEdd ZFdd ZGdd ZHdd ZIdd  ZJd!d" ZKdd$d%ZLd&d' ZMG d(d) d)ejNZOG d*d+ d+e*ZPG d,d- d-ejNZQd.d/ ZRd0d1 ZSd2d3 ZTejUG d4d5 d5ZVejUG d6d7 d7eVZWG d8d9 d9eWZXejUG d:d; d;eXZYG d<d= d=eZZG d>d? d?eZ[ejUG d@dA dAeWZ\dBdC Z]dDdE Z^ddGdHZ_eje_d#dIZ`dJdK ZaejUG dLdM dMeVZbejUG dNdO dOebZcejUG dPdQ dQebZdG dRdS dSebZeejUG dTdU dUebZfejUG dVdW dWebZgG dXdY dYefZhG dZd[ d[eVZiejUG d\d] d]eiZjejUG d^d_ d_eiZkejUG d`da daeVZlG dbdc dcelZmG ddde deelZnG dfdg dgelZoG dhdi dielZpejUG djdk dkeVZqG dldm dmeqZrG dndo doerZsG dpdq dqesZtG drds dseVZuG dtdu dueVZvejUG dvdw dweqZwG dxdy dyeqZxejUG dzd{ d{eqZyG d|d} d}eyZzG d~d dezZ{ejUG dd deyZ|ejUG dd de|Z}G dd de|Z~G dd de|ZG dd de|ZG dd de}ZG dd deVZejUG dd de~ZejUG dd deVZG dd de|ZG dd de~Zddddee ee ee eeee d	ddZG dd de~ZG dd de~ZG dd de~ZG dd de~ZG dd de~ZG dd de~ZG dd de~ZejUG dd deVZG dd deZG dd deZG dd dejjZG dd dZG dd dZdS )    N)OrderedDict)nullcontext)Enum)partial)	signature)	AnyCallableClassVarDictListOptionalSetTupleUnion)patch)ExprInteger)is_boolean_dtypeis_float_dtypemake_channels_last_strides_formake_contiguous_strides_for)FloorDiv   )configdependencies)index_prevent_reordering)get_device_properties)extract_read_writesvar_builder)	argsortcache_on_selfconvert_shape_to_inductorconvert_shape_to_symintdeveloper_warning	sympy_dotsympy_product
sympy_subssympy_symbol)opsVz  prefixc                 C   s6   dd }t | ttfr*| D ]}|| qn||  d S )Nc                 S   s2   t | tttjjjjtfs.J dt	|  dd S )NzFound zE, which is not a supported top level IR node. See [Note: Inductor IR])

isinstance	TensorBoxRandSeedBuffertorchfxZexperimentalZsymbolic_shapesZSymbolr   type)node r3   [/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_inductor/ir.py_check_tensorboxW   s    
z%validate_ir.<locals>._check_tensorbox)r,   r   r   )Znode_or_nodesr5   r2   r3   r3   r4   validate_irV   s
    r6   c                    s&   t t| tt|   fdd}|S )Nc                    s0   t  t ksJ  fddtt  D S )Nc                    s   g | ]} |  qS r3   r3   .0i)index	inv_orderr3   r4   
<listcomp>q       z4inverse_reorder.<locals>.reindex.<locals>.<listcomp>lenranger:   r;   rA   r4   reindexo   s    z inverse_reorder.<locals>.reindex)dictzipr@   r?   orderrC   r3   rB   r4   inverse_reorderl   s    rH   c                    s    fdd}|S )Nc                    s0   t  t ksJ  fddtt  D S )Nc                    s   g | ]} |  qS r3   r3   r7   )r:   rG   r3   r4   r<   y   r=   z1same_reorder.<locals>.reindex.<locals>.<listcomp>r>   rA   rG   rA   r4   rC   w   s    zsame_reorder.<locals>.reindexr3   rF   r3   rI   r4   same_reorderv   s    rJ   c                    s    fdd}|S )Nc                    s    | S Nr3   rA   reindex1reindex2r3   r4   rC      s    z fuse_reindexing.<locals>.reindexr3   )rM   rN   rC   r3   rL   r4   fuse_reindexing~   s    rO   c                    s0   dd t | D   fddtt| D }|S )z
    Convert stride order to fill order
    For channel last format,
    stride order = [3, 0, 2, 1] and fill order = [1, 3, 2, 0]
    c                 S   s   i | ]\}}||qS r3   r3   r8   idxposr3   r3   r4   
<dictcomp>   r=   z+stride_order2fill_order.<locals>.<dictcomp>c                    s   g | ]} | qS r3   r3   r7   lookupr3   r4   r<      r=   z+stride_order2fill_order.<locals>.<listcomp>)	enumerater@   r?   )rG   
fill_orderr3   rT   r4   stride_order2fill_order   s    rX   c                 C   s<   t | }dd tt| D }t|D ]\}}|||< q&|S )z)
    Convert strides to stride order
    c                 S   s   g | ]}d qS rK   r3   r8   _r3   r3   r4   r<      r=   z$get_stride_order.<locals>.<listcomp>)r   r@   r?   rV   )seqZ
sorted_idxoutr9   elemr3   r3   r4   get_stride_order   s
    
r^   c           
      C   s   | du rdS t | trD| j  }t|dd d}||}d|fS t | jtr|  j}dd |D }|D ]"}t	||\}}	|rld|	f  S qldS )	z[
    return:
    if reads_from_conv: boolean
    the new memory_addr: Sympy Expression
    N)FNc                 S   s   | j S rK   namevarr3   r3   r4   <lambda>   r=   z!reads_from_conv.<locals>.<lambda>keyTc                 S   s0   g | ](}|j tjj v r(tjj|j  nd qS rK   r`   r)   graphname_to_bufferkeysr8   rr3   r3   r4   r<      s   z#reads_from_conv.<locals>.<listcomp>)
r,   Convolutionlayoutas_fixedmake_indexersortedAliasedLayoutget_read_writesreadsreads_from_conv)
buf
var_rangesindexer
index_varsr:   rs   
reads_bufs	reads_bufread_from_convaddrr3   r3   r4   rt      s"    

rt   Tc                    s   |st jjj ndd }|  fdd|  D }t| rT fdd|  jD }nt|}| 	 }| 
 }t|}t|}tj||||d }|S )Nc                 S   s   | S rK   r3   xr3   r3   r4   nop   s    zir_node_to_tensor.<locals>.nopc                    s   g | ]} |qS r3   r3   r8   sZshape_fnr3   r4   r<      r=   z%ir_node_to_tensor.<locals>.<listcomp>c                    s   g | ]} |qS r3   r3   r   r   r3   r4   r<      r=   )sizestridedtypedevice)r)   rg   sizevars	size_hintget_sizeis_storage_and_layout
get_layoutr   r   	get_dtype
get_devicer"   r/   Zempty_stridedZzero_)r~   guard_shaper   r   r   r   r   tr3   r   r4   ir_node_to_tensor   s     
r   c                 C   sB   g }t | D ],\}}t||\}}|r|| |||< q||fS )z
    if reads from conv that needs to use specific layout
    return:
    priority_idx regarding memory_addrs idx
    memory_addrs - update memory_addrs with the true addr if needed
    )rV   rt   append)ry   memory_addrsrv   priority_idxr9   rz   r{   Zmem_addrr3   r3   r4   layout_priority_idx   s    

r   c                   @   s$   e Zd ZdZdZdZedd ZdS )ModularIndexingz2
    ModularIndexing(a, b, c) => (a // b) % c
    )   Tc                 C   sf  |dks|dkrt dS t|t jrJt|t jrJt|t jrJ|| | S |dkrzt ||}|dkrzt|| || |S t|t jr:g }d}|jD ]x}t ||| || krt|t jr|dk st|t jrt|jd t jr|jd dk rd} qq|| qt	|t	|jkr:|r:tt
|||S t|trbt|jd |jd | |S d S )Nr   r   TF)sympyr   r,   gcdr   AddargsZMulr   r?   sumr   )clsbasedivisormodulusr   Z	new_termsZall_positiveZtermr3   r3   r4   eval   s@    





	zModularIndexing.evalN)__name__
__module____qualname____doc__nargs
is_integerclassmethodr   r3   r3   r3   r4   r      s
   r   c                   @   s   e Zd ZdZdS )CleanDivzZ
    Div where we can assume no rounding.
    This is to enable future optimizations.
    N)r   r   r   r   r3   r3   r3   r4   r     s   r   c                   @   s   e Zd ZdZdZdd ZdS )CeilDivz.
    Div used in indexing that rounds up.
    Tc                 C   s0   t |||krt||S t||d  |S d S Nr   )r   r   r   r   )r   r   r   r3   r3   r4   __new__*  s    
zCeilDiv.__new__N)r   r   r   r   r   r   r3   r3   r3   r4   r   #  s   r   c                 C   s.   t | dd rt|  S t| tjr*| jS d S )Nr   )getattrget_device_typer   r,   r/   r   r1   r}   r3   r3   r4   r   1  s
    r   c                 C   s   t | dkS )NZcudar   r}   r3   r3   r4   	is_triton9  s    r   c                 C   s   t | dkS )Ncpur   r}   r3   r3   r4   is_cpu=  s    r   c                   @   sl   e Zd ZU e Zeee  ed< e	e
jeejj dddZdd Zdd Zd	d
 Zdd Zdd ZdS )IRNode_current_origins)originsc                 c   s    t j}|| B t _d V  |t _d S rK   )r   r   )r   oldr3   r3   r4   current_originsE  s    
zIRNode.current_originsc                 C   s   t | j| _d S rK   )setr   r   selfr3   r3   r4   __post_init__M  s    zIRNode.__post_init__c                 C   s   t | drd| j gS dgS )Nr   zorigins=zno origins?)hasattrr   r   r3   r3   r4   common_reprP  s    zIRNode.common_reprc                 C   s6   ||    }tdtt|}t| j d| dS )Nz,
z(
z
))r   indentjoinmapstrr1   r   r   linesr3   r3   r4   
str_helperU  s    zIRNode.str_helperc                    s   t  fdd|  D S )Nc                 3   s   | ]} |j kV  qd S rK   r_   )r8   depr_   r3   r4   	<genexpr>[  r=   z$IRNode.is_user_of.<locals>.<genexpr>)any	get_readsr   r`   r3   r_   r4   
is_user_ofZ  s    zIRNode.is_user_ofc                 C   s   t |  S rK   )r%   r   r   r3   r3   r4   	get_numel]  s    zIRNode.get_numelN)r   r   r   r   r   r	   r   r   __annotations__staticmethod
contextlibcontextmanagerr/   r0   Noder   r   r   r   r   r   r3   r3   r3   r4   r   A  s   
r   c                   @   s   e Zd ZU ejed< ejed< eed< ee	 ed< dddZ
e
Zdd	 Zd
d Zdd Zdd Zedd ZedddZedd Zdd Zedd ZdS )Loopsr   r   inner_fnrangesr   c                    s8     d jj dt j  g fdd|D  S )N'c                    s    g | ]}| d t  | qS =)r   )r8   r`   r   r3   r4   r<   o  r=   z!Loops.__str__.<locals>.<listcomp>)r   r   r1   r   r   inner_fn_str)r   namesr3   r   r4   __str__h  s    zLoops.__str__c                 C   s   | j S rK   r   r   r3   r3   r4   r   t  s    zLoops.get_dtypec                 C   s   | j S rK   r   r   r3   r3   r4   r   w  s    zLoops.get_devicec                 C   s   | j S rK   r   r   r3   r3   r4   r   z  s    zLoops.get_sizec                 C   s   dS NFr3   r   r3   r3   r4   	is_extern}  s    zLoops.is_externc                 O   s   t | |i |S rK   )r-   create)r   r   kwargsr3   r3   r4   r     s    zLoops.creater9   c                    s    fddt | D S )Nc                    s2   g | ]*\}}|d krt dnt  | qS )r   r   )r   r   r'   )r8   nr   r*   r3   r4   r<     s   z Loops._index.<locals>.<listcomp>)rV   )r   r+   r3   r*   r4   _index  s    
zLoops._indexc              	   C   s   t t  }t |h ttdd: | | | j	}|
|W  d    W  d    S 1 sh0    Y  W d    n1 s0    Y  d S Nallow_indexingT)r)   KernelFormatterHandlerMockHandlerset_ops_handlerr   objectFlexibleLayoutr   r   r   getvaluer   	formatterresultr3   r3   r4   r     s    zLoops.inner_fn_strc                 C   s   t dd | jD S )Nc                 s   s   | ]}|d kV  qdS )r   Nr3   rj   r3   r3   r4   r     r=   z)Loops.is_zero_elements.<locals>.<genexpr>)r   r   r   r3   r3   r4   is_zero_elements  s    zLoops.is_zero_elementsc                 C   s   t tddb |  r@t|  |  |  jW  d    S t|  |  jW  d    S W d    n1 sv0    Y  d S r   )	r   r   r   get_reduction_typer   make_loaderr   get_reduction_sizers   r   r3   r3   r4   r     s    zLoops.get_readsN)r   )r9   )r   r   r   r/   r   r   r   r   r   r   r   __repr__r   r   r   r   r   r   r   r   r    r   r   r   r3   r3   r3   r4   r   a  s&   






r   c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )	Pointwisec                 C   s   | j S rK   )r   r   r3   r3   r4   r     s    zPointwise.make_loaderc                 C   s   g S rK   r3   r   r3   r3   r4   r     s    zPointwise.get_reduction_sizec                 C   s   d S rK   r3   r   r3   r3   r4   r     s    zPointwise.get_reduction_typec                 C   s   t |||| |S rK   )r(   storer   r   output_namerw   varsr3   r3   r4   store_output  s    zPointwise.store_outputc                 C   s,   |   }ttd||}t|| j|| jS FMove this to a given device. Requires that all reads are to constants.override_device)r   r   r   ConstantBufferr   r   r   r   r   loaderr3   r3   r4   constant_to_device  s    zPointwise.constant_to_deviceN)r   r   r   r   r   r   r   r   r3   r3   r3   r4   r     s
   r   c                   @   sD   e Zd ZU eee gef ed< dZee	 ed< dd Z
dd ZdS )Scatteroutput_indexerNscatter_modec                 C   s4   |   }ttd||}t|| j|| j| j| jS r   )	r   r   r   r   r   r   r   r   r  r   r3   r3   r4   r     s    zScatter.constant_to_devicec                 C   s$   t j||| || || jdS )N)mode)r(   r   r   r   r  r   r3   r3   r4   r     s    zScatter.store_output)r   r   r   r   r   r   r   r  r   r   r   r   r3   r3   r3   r4   r     s   
r   c                   @   s   e Zd ZdZdZdZdZdS )ReductionHintr   r      r   N)r   r   r   INNEROUTER
OUTER_TINYDEFAULTr3   r3   r3   r4   r    s   r  c                   @   s   e Zd ZdZdZdS )TileHintr   r   N)r   r   r   ZSQUAREr  r3   r3   r3   r4   r	    s   r	  c                   @   s   e Zd ZU ee ed< eed< ejed< e	ed< dd Z
e
Zdd Zd	d
 Zdd Zdd Zedd Zdd Zedd Zedd Zee	jfejejejeee ee ee	dddZedd Zeejejejeee ee eee	d	ddZdS ) 	Reductionreduction_rangesreduction_type	src_dtypereduction_hintc                 C   s   t j| ddS )N)r   r  r  )r   )r   r   r   r3   r3   r4   r     s    zReduction.__str__c                 C   s   | j S rK   )r  r   r3   r3   r4   r     s    zReduction.get_reduction_sizec                 C   s   | j S rK   )r  r   r3   r3   r4   r     s    zReduction.get_reduction_typec                 C   s&   t || j| j| j||| ||S rK   )r(   	reductionr   r  r  r   )r   r   rw   r   reduction_varsr3   r3   r4   store_reduction  s    
zReduction.store_reductionc                 C   s   t | jt | j S rK   )r?   r   r  r   r3   r3   r4   index_length  s    zReduction.index_lengthc              	   C   s   t t  }t |t ttddF | | | j	| | j
d}||W  d    W  d    S 1 st0    Y  W d    n1 s0    Y  d S )Nr   Trk   )r)   r   r   r   r   r   r   r   r   r   r  r   r   r3   r3   r4   r     s    
zReduction.inner_fn_strc              	   C   s<   |   }ttd||}t|| j|| j| j| j| j	t
jS r   )r   r   r   r   r
  r   r   r  r  r  r  r  r   r3   r3   r4   r     s    zReduction.constant_to_devicec              	      s  t | jddd      fdd} fdd}	tjj|}
tjjt|}|dkrtj||
|fS |
ks|d	 d krtj	dfS t
| ||||||tj	}d
d }||\}}|r||\}}t|dkrtj	dfS t| | \}\}}}d}d}|D ]@}tjj||}tdd |D }|rh|d7 }n|d7 }q2||krtj||
|fS tj|	|
|fS d S )N       i   c           	         s  d}d| }|d krdS | dkr(dS | | kr:}n| | k r d|  }|| d | }| ||  d ||   t | }t| fddd}t|  d	k rt|}q }n8t | }t|fd
dd}t| dk r|}n}| ||  d ||  S )N   r  r  r   i    c                    s   t |   S rK   absr}   Ztmp_split_sizer3   r4   rc   C  r=   zFReduction.num_splits.<locals>.inner_reduction_splits.<locals>.<lambda>rd      c                    s   t |   S rK   r  r}   max_elements_per_threadr3   r4   rc   K  r=   2   r   divisorsminr  max)	reduction_numel_hint
numel_hint	num_warpsnum_threads
split_sizetarget_blocksZblocks_per_outputr  closestZmax_elements_per_devicer  Zmin_elements_per_deviceZmin_elements_per_threadZnum_smZthreads_per_smr  r4   inner_reduction_splits1  s6    

z4Reduction.num_splits.<locals>.inner_reduction_splitsc                    s  d}|d }d}d}|| d | }| | k r6}n| | k r | }|| d | }| ||  d ||   t | }	t|	 fddd}
t |
 d	k rt|
}q }n8t | }	t|	fd
dd}
t|
 dk r|
}n}| ||  d ||  S )Nr  r        r   c                    s   t |   S rK   r  r}   r  r3   r4   rc   f  r=   zFReduction.num_splits.<locals>.outer_reduction_splits.<locals>.<lambda>rd      c                    s   t |   S rK   r  r}   r  r3   r4   rc   m  r=   r  r  )r!  r"  r#  r$  Zrvals_per_threadZxvals_per_blockZxblocksr%  r&  r  r'  r(  r  r4   outer_reduction_splitsU  s4    

z4Reduction.num_splits.<locals>.outer_reduction_splitsr   r  c                    s   t d t|  |  |  d| d}| }dd |jD }g }d}t|jdd dD ]b t	 fd	d|D rV|
 j  jtjjv rVtjj j }|jj}|  |jj|krVd
}qV||fS )Nr   r   r   r`   rm   datac                 S   s(   g | ] }t |tjrt |tjs|qS r3   )r,   r   r   Numberrj   r3   r3   r4   r<     s   zBReduction.num_splits.<locals>.get_read_indices.<locals>.<listcomp>Fc                 S   s   | j S rK   r_   r}   r3   r3   r4   rc     r=   z@Reduction.num_splits.<locals>.get_read_indices.<locals>.<lambda>rd   c                    s   g | ]}| j jv qS r3   )r:   Zfree_symbolsrj   Zmdr3   r4   r<     r=   T)ComputedBufferr   r   r   r   rr   
range_varsrp   rs   allr   r:   r`   r)   rg   rh   rm   r   decide_layout)rk   cbread_writesr4  indiceschangedru   Zoriginal_strider3   r2  r4   get_read_indices  s2    	z.Reduction.num_splits.<locals>.get_read_indicesr   c                 S   s   g | ]}|d kqS r   r3   r   r3   r3   r4   r<     r=   z(Reduction.num_splits.<locals>.<listcomp>)r   Zmulti_processor_countr)   rg   r   r   r%   r  r  r  r
  r?   r   index_vars_squeezer   r   stride_hintsr5  r  )r   	dst_dtyper  r   r   r  r  reduction_numelr)  r-  r!  r"  rk   r;  r9  r:  rZ   r  Z	num_outerZ	num_innerr9   stridesouterr3   r(  r4   
num_splits  sj    
$#
 


zReduction.num_splitsc                    s   dd D |dkr dd  nh|dkr2dd  nV|dkrDd	d  nD|d
krVdd  n2|dkrhdd  n |dkrzdd  nt d|  fdd|dv rtddt fddfddS S dS )z1Convert inner_fn from a reduction to an pointwisec                 S   s   g | ]}t jj|qS r3   )r)   rg   r   guard_static_shaper8   r~   r3   r3   r4   r<     s   z2Reduction._unroll_reduction_fn.<locals>.<listcomp>r   c                 S   s   t | |S rK   )r(   addabr3   r3   r4   
combine_fn  s    z2Reduction._unroll_reduction_fn.<locals>.combine_fnr  c                 S   s   t | |S rK   )r(   minimumrG  r3   r3   r4   rJ    s    r   c                 S   s   t | |S rK   )r(   maximumrG  r3   r3   r4   rJ    s    r   c                 S   s   t | |S rK   )r(   
logical_orrG  r3   r3   r4   rJ    s    argminc                 S   s:   t | d |d t t |d | d |d | d fS Nr   r   )r(   rK  whereltrG  r3   r3   r4   rJ    s    argmaxc                 S   s:   t | d |d t t |d | d |d | d fS rO  )r(   rL  rP  gtrG  r3   r3   r4   rJ    s    zunknown reduction_type=c                    s,   t  fddtjdd D  D S )Nc                 3   s   | ]} |V  qd S rK   r3   )r8   rindex)r:   value_fnr3   r4   r     s   z=Reduction._unroll_reduction_fn.<locals>.fn.<locals>.<genexpr>c                 S   s   g | ]}t |qS r3   )r@   rE  r3   r3   r4   r<     r=   z>Reduction._unroll_reduction_fn.<locals>.fn.<locals>.<listcomp>)	functoolsreduce	itertoolsproductrA   )rJ  r  rU  rA   r4   fn  s    z*Reduction._unroll_reduction_fn.<locals>.fnrN  rR  Nc                    s*   dd |D }| |t  |tjfS )Nc                 S   s   g | ]}t |qS r3   )r   expandr7   r3   r3   r4   r<     r=   zDReduction._unroll_reduction_fn.<locals>.value_fn.<locals>.<listcomp>)r(   
index_exprr/   int64r:   rT  )flatten_indexr   r3   r4   rU    s    z0Reduction._unroll_reduction_fn.<locals>.value_fnc                    s    | d S r   r3   rA   )rZ  r3   r4   rc     r=   z0Reduction._unroll_reduction_fn.<locals>.<lambda>)NotImplementedErrorFixedLayoutr   contiguous_stridesro   )r   r  r  r3   )rJ  r`  rZ  r   r  rU  r4   _unroll_reduction_fn  s8    





zReduction._unroll_reduction_fn)r   r?  r  r   r   r  r  r  c	                    s  t jjt}	|	dkr~ fdd}
|
d|
d|
dd v sXJ  d fdd}tj|||t|d	S |	dkrd
v r fdd}nfdd}t| ||S t	|	t
jrt jj|	tjk rt|dkrt| | |S t|rjdvrj| | |||	\}}|tjkrF|}|dkrj| | ||||	S tt| |||S )Nr   c                    s(    t jkrt| S  jr t| S t| S rK   )r/   boolZis_floating_pointfloatint)valr?  r3   r4   py_cnst+  s    

z!Reduction.create.<locals>.py_cnstr   )r   prodr   z* not supported for zero-dimension tensors!c                    s   t   S rK   r(   constantrA   )r?  r  rtypes_to_initsr3   r4   const_fn?  s    z"Reduction.create.<locals>.const_fnr   r   r   r   r[  c                    s   t d S Nr   rl  rA   ri  r3   r4   rZ  M  s    zReduction.create.<locals>.fnc                    s   dd D } | |S )Nc                 S   s   g | ]}t d qS r   r   r   rY   r3   r3   r4   r<   S  r=   z0Reduction.create.<locals>.fn.<locals>.<listcomp>r3   r:   reduction_index)r   r  r3   r4   rZ  R  s    >   rN  rR  )r)   rg   r   simplifyr%   ri   r   r   listr,   r   r   r   r   Zunroll_reductions_thresholdrd  r   rC  r  r  create_multilayerr-   r
  )r   r   r?  r  r   r   r  r  r  r@  rj  ro  rZ  hintsplitr3   )r?  r   r  r  rn  r4   r     s    




zReduction.createc                 C   sn   | dv r0t |rtdS t|r$dS t|jS | dv r`t |rHtdS t|rTdS t|jS ddd|  S )N>   r   rR  z-infr   >   r  rN  infr   )r   r   )r   rf  r   r/   Ziinfor  r   )r  r   r3   r3   r4   default_value  s"    zReduction.default_value)	r   r?  r  r   r   r  r  rz  r  c
                    sb  t |dd |D }ttjjtjj| dkrFdndt|}t|d  | t	
|g fdd}
tjtjfvrntj}t||||
g || g|	}|  | fd	d
}tjjt |}|dkr|dkr|	tjkrtj}	|dkrD|dkrD|	tjkrDtj}	tt||||g||	S )za
        Break a large reduction up into multiple smaller reductions
        recursively
        c                 S   s    g | ]}t tjj|qS r3   r   r   r)   rg   r   rD  r   r3   r3   r4   r<     s   z/Reduction.create_multilayer.<locals>.<listcomp>r   FTr   c                    sr   |\}| ^ }| |   	fdd}rht t  tjt tj}t ||S | S d S )Nc                      s    gS rK   r3   r3   )r9  r   	new_indexrC   r3   r4   body  s    z=Reduction.create_multilayer.<locals>.wrapper_fn.<locals>.body)r(   rQ  r]  r/   int32maskedr|  )r:   ru  Zreduction_blockr  mask)
block_sizer   r?  r   	need_maskr@  r  rC   )r9  r~  r4   
wrapper_fn  s    
z/Reduction.create_multilayer.<locals>.wrapper_fnc                    s    g | |S rK   r3   rt  )intermediate_loaderr3   r4   intermediate_fn  s    z4Reduction.create_multilayer.<locals>.intermediate_fnr  i      )r%   r   r   r)   rg   r   rD  r   r   Viewdynamic_reshape_indexerr/   float16bfloat16rf  r
  r   realizer   r  r  r  r-   )r   r   r?  r  r   r   r  r  rz  r  r  Zintermediate_dtypeZintermediater  r"  r3   )	r  r   r?  r   r  r  r@  r  rC   r4   rx    sj    


 zReduction.create_multilayerN)r   r   r   r   r   r   r   r/   r   r  r   r   r   r   r  r  r    r   r   r   rC  rd  r   r  r   r   r   r|  rg  rx  r3   r3   r3   r4   r
    sV   




 .
J
x
r
  c                 C   s,   zt | dd W dS  ty&   Y dS 0 d S )NFfreezeT)as_storage_and_layoutra  r}   r3   r3   r4   r     s
    r   c                 C   s4   zt | dd\}}| W S  ty.   Y dS 0 d S NFr  )r  is_contiguousra  )r~   bufferrm   r3   r3   r4    is_contiguous_storage_and_layout  s
    
r  Fc                 C   s   t | trt| j|||dS t | trrt | jtrr|rf|rF| j  n |dur\| j| n
| j  | | jj	fS t | t
rt| j|d\}}|| j	fS tdS )z0Try to simplify x into a StorageBox and a Layoutr  want_contiguousstride_orderNr  )r,   r-   r  r0  
StorageBoxBufferfreeze_layoutfreeze_layout_with_stride_orderr6  rm   ReinterpretViewra  )r~   r  r  r  r  rZ   r3   r3   r4   r     s,    




r  )r  c                 C   s6   zt | dd\}}||W S  ty0   Y dS 0 d S r  )r  is_stride_orderedra  )r~   r  r  rm   r3   r3   r4   "is_stride_order_storage_and_layoutB  s
    r  c                   @   sz   e Zd ZU eed< dd Zdd Zdd Zdd	 Zd
d Z	dd Z
dd Zdd Zdd Zedd Zdd Zdd ZdS )BaseViewr0  c                 C   s
   | j  S rK   )r0  r   r   r3   r3   r4   r   N  s    zBaseView.get_dtypec                 C   s
   | j  S rK   )r0  r   r   r3   r3   r4   r   Q  s    zBaseView.get_devicec                 C   s
   | j  S rK   r0  get_namer   r3   r3   r4   r  T  s    zBaseView.get_namec                 C   s   | j |S rK   )r0  
mark_reuser   usersr3   r3   r4   r  W  s    zBaseView.mark_reusec                 C   s
   | j  S rK   )r0  has_exceeded_max_readsr   r3   r3   r4   r  Z  s    zBaseView.has_exceeded_max_readsc                 C   s
   | j  S rK   )r0  r  r   r3   r3   r4   r  ]  s    zBaseView.realizec                 C   s
   | j  S rK   )r0  realize_hintr   r3   r3   r4   r  `  s    zBaseView.realize_hintc                 C   s
   | j  S rK   )r0  get_storage_numelr   r3   r3   r4   r  c  s    zBaseView.get_storage_numelc                 C   s
   | j  S rK   )r0  r   r   r3   r3   r4   r   f  s    zBaseView.is_externc                 C   sF   t tdd$ t|  |  jW  d    S 1 s80    Y  d S r   )r   r   r   r   r   r   rs   r   r3   r3   r4   r   i  s
    zBaseView.get_readsc                 C   s   | }t |tr|j}q|S rK   )r,   r  r0  )r   r~   r3   r3   r4   unwrap_viewq  s    
zBaseView.unwrap_viewc                 C   s0   |   }ttd||}t||  ||  S r   )r   r   r   r   r   r   r   r   r3   r3   r4   r   w  s    zBaseView.constant_to_deviceN)r   r   r   r   r   r   r   r  r  r  r  r  r  r   r    r   r  r   r3   r3   r3   r4   r  J  s   

r  c                   @   sB   e Zd ZU ee ed< edd Zedd Z	dd Z
dd	 Zd
S )
ExpandViewr   c                 C   s   t ttj|}|  }dgt|t|  t | }t|t|ksJJ tt|D ],}|| dkrV|| dusvJ || ||< qV|S )zReplace `-1` with correct sizesN)rw  r   r   r\  r   r?   r@   )r~   new_sizeold_sizer9   r3   r3   r4   _normalize_size  s    zExpandView._normalize_sizec           
      C   s   |  ||}t|rt|\}}t|t|j }|dks>J tdg| }t|j|jD ]$\}}|	|dkrt|ntd q\t
|j|jt|||j}	t||	S t||S rO  )r  r   r  r?   r   r   r   rE   r   r   rb  r   r   rw  offsetr  r  )
r   r~   r  storage
old_layoutskip
new_strider   r   
new_layoutr3   r3   r4   r     s"    
zExpandView.createc                 C   s   | j S rK   r   r   r3   r3   r4   r     s    zExpandView.get_sizec                    s@   |   }| j   t|t  | j  fdd}|S )Nc                    sX   t | d  } t| t ks$J tt D ]} | dkr0td| |< q0| S )Nr   r   )rw  r?   r@   r   r   )r:   r9   actualinnerr  r3   r4   load  s    z$ExpandView.make_loader.<locals>.load)r   r0  r?   r   )r   targetr  r3   r  r4   r     s    

	zExpandView.make_loaderN)r   r   r   r   r   r   r   r  r   r   r   r   r3   r3   r3   r4   r  ~  s   


r  c                   @   sB   e Zd ZU ee ed< edd Zedd Zdd Z	dd	 Z
d
S )PermuteViewdimsc                    s   |  |}t|ttt|ks&J t|rvt|\} t j j fdd|D  fdd|D  j	}t
||S t||S )Nc                    s   g | ]} j | qS r3   r  r7   r  r3   r4   r<     r=   z&PermuteView.create.<locals>.<listcomp>c                    s   g | ]} j | qS r3   )r   r7   r  r3   r4   r<     r=   )_map_neg_dimsr   r@   r?   r   r  rb  r   r   r  r  r  )r   r~   r  r  r  r3   r  r4   r     s    

zPermuteView.createc                    s    fdd D S )Nc                    s$   g | ]}|d kr|n
t  | qS rr  )r?   )r8   dimr  r3   r4   r<     r=   z-PermuteView._map_neg_dims.<locals>.<listcomp>r3   )r   r  r3   r  r4   r    s    zPermuteView._map_neg_dimsc                    sD   t | | jt tt| jks&J | j   fdd| jD S )Nc                    s   g | ]} | qS r3   r3   r7   r  r3   r4   r<     r=   z(PermuteView.get_size.<locals>.<listcomp>)r   r  r  r@   r?   r0  r   r   r3   r  r4   r     s    &
zPermuteView.get_sizec                    sj   | j   dd t| jD fddtt| jD tttt| jksXJ  fdd}|S )Nc                 S   s   i | ]\}}||qS r3   r3   )r8   r9   jr3   r3   r4   rS     r=   z+PermuteView.make_loader.<locals>.<dictcomp>c                    s   g | ]} | qS r3   r3   r7   )invr3   r4   r<     r=   z+PermuteView.make_loader.<locals>.<listcomp>c                    s    fddD   S )Nc                    s   g | ]} | qS r3   r3   r7   rA   r3   r4   r<     r=   z9PermuteView.make_loader.<locals>.load.<locals>.<listcomp>r3   rA   r  r  rA   r4   r    s    z%PermuteView.make_loader.<locals>.load)r0  r   rV   r  r@   r?   r   r   r  r3   r  r4   r     s    
zPermuteView.make_loaderN)r   r   r   r   r   r   r   r   r  r   r   r3   r3   r3   r4   r    s   


r  c                   @   sB   e Zd ZeddddZeeejdf dddZ	d	d
 Z
dS )SqueezeViewNr  c                   sD  t |rt|\}}g }g } d urPt ts6J dd krL t|jk sPJ tt|j|jD ]`\}\}}	 d u r|dkr|	| |	|	 qb| kr|	| |	|	 qb|dksbJ dqbt
|j|j|||j}
t||
S  d u rt|dd | D S |   dksJ t| fddt| D S d S )Nzexpected integer dim argumentr   r   zexpected squeezed size to be 1c                 S   s   g | ]}|d kr|qS r<  r3   r   r3   r3   r4   r<   	  r=   z&SqueezeView.create.<locals>.<listcomp>c                    s   g | ]\}}| kr|qS r3   r3   r8   r9   r   r  r3   r4   r<     r=   )r   r  r,   rg  r?   r   rV   rE   r   r   rb  r   r   r  r  r  r   r   )r   r~   r  r  r  r  r  r9   r   r   r  r3   r  r4   r     s8    



zSqueezeView.create.r  c                    sR   dd | D }dd t | D t|  ttj ttj d fdd}||fS )Nc                 S   s   g | ]}|d kr|qS r<  r3   r   r3   r3   r4   r<     r=   z(SqueezeView.squeezer.<locals>.<listcomp>c                 S   s   g | ]\}}|d kr|qS r<  r3   r  r3   r3   r4   r<     r=   )r:   returnc                    sV   t | t ks"J |  d tdg  }t| D ]\}}|||< q<t|S )N r   )r?   r   r   rE   tuple)r:   r~  rQ   r   lengthZnot_oner3   r4   rC     s
    "
z%SqueezeView.squeezer.<locals>.reindex)rV   r?   r   r   r   )r   r  rC   r3   r  r4   squeezer  s
    "zSqueezeView.squeezerc                 C   s   t dd S )Nzuse SqueezeView.create())AssertionError)r   r0  r3   r3   r4   __init__  s    zSqueezeView.__init__)r   r   r   r   r   r   r   r   r   r  r  r3   r3   r3   r4   r    s
   &r  c                   @   s   e Zd ZU ee ed< eed< dd Zedd Z	dd Z
d	d
 ZeZedd Zedd Zedd Zedd Zdd Zdd ZdS )r  r   rC   c                    s   j    fdd}|S )Nc                    s     | S rK   rC   )rQ   Zbase_indexerr   r3   r4   rw   )  s    z"View.make_indexer.<locals>.indexer)r0  ro   r   rw   r3   r  r4   ro   &  s    
zView.make_indexerc                 C   sB   t | } t |}tjj}|| dk r>|| d | | } | S rq  )r   r\  r)   rg   r   r   Zguard_lt)rQ   r   r   r3   r3   r4   handle_negative_index.  s    

zView.handle_negative_indexc                 C   sB   dd t t| jD }t| |}ddtt| d| S )Nc                 S   s   g | ]}t d | qS )r9   r'   )r8   r   r3   r3   r4   r<   9  r=   z$View.reindex_str.<locals>.<listcomp>zlambda , z: )r@   r?   r   rw  rC   r   r   r   )r   Z	index_oldZ	index_newr3   r3   r4   reindex_str8  s    zView.reindex_strc                 C   s$   |  | jd| j d|   gS )Nzsize=zreindex=)r   r0  r   r  r   r3   r3   r4   r   =  s    zView.__str__c                 C   s   t |ttfsJ | | |\}}tjj||r:|S t	|rt |j
tst|\}}t|j|j|t||j}t||S | ||}| |t||S rK   )r,   r  rw  resolve_negative_sizer   r)   rg   r   Zmaybe_guard_list_equalsr  r0  ExternKernelAlloc as_contiguous_storage_and_layoutrb  r   r   r   rc  r  r  r  )r   r~   r  r  r  r  r  rC   r3   r3   r4   r   D  s$    

zView.createc                 C   s   dd |D }dd | D } t |}tt|D ]8}|| dkr0td||< tt| t|||<  qjq0tjj	
t| t| | |fS )Nc                 S   s   g | ]}t jj|qS r3   r)   rg   r   rv  rE  r3   r3   r4   r<   _  r=   z.View.resolve_negative_size.<locals>.<listcomp>c                 S   s   g | ]}t jj|qS r3   r  rE  r3   r3   r4   r<   `  r=   r  r   )rw  r@   r?   r   r   r   r%   r)   rg   r   guard_equals)r  r  r9   r3   r3   r4   r  ]  s    zView.resolve_negative_sizec              	   C   sX   z|  ||}W nB ttfyR   t|g}|  ||}|  ||}t||}Y n0 |S rK   )_dynamic_reshape_indexerr  
IndexErrorr%   rO   )r   r  r  rC   ZflatrM   rN   r3   r3   r4   r  l  s    
zView.dynamic_reshape_indexerc                    s@  t jjj}dd tt|D  tt |}t| }g |r|r| }| \}}|dkr	t
d |	||f q:|dkr|	| q:||||kr	| t jj|| q:||||k r$||||k r| \}}	|| | }||	 }qԈ	| t jj|| q:||||krt
d}
|}	t||
| |
| }
||||kr| }	t||
| |
| }
|| }q^t jj|| q:t q:|r| }|dksJ 	t
d q|r| \}}|dksJ qtttt| ks.J  fdd}|S )zG
        Perform a reshape entirely by modifying indexing math
        c                 S   s   g | ]}t d | qS viewr  r7   r3   r3   r4   r<   ~  r=   z1View._dynamic_reshape_indexer.<locals>.<listcomp>r   r   c                    sH   t | t ks$J t | t ftt|  t fddD S )Nc                 3   s   | ]}t | V  qd S rK   r&   rE  replacementsr3   r4   r     r=   zAView._dynamic_reshape_indexer.<locals>.reindex.<locals>.<genexpr>)r?   rD   rE   r  rA   r   Z	view_exprr  r4   rC     s    $z.View._dynamic_reshape_indexer.<locals>.reindex)r)   rg   r   r   r@   r?   rw  rE   popr   r   r   r  r   r  reversed)r  r  r   Z	stack_newZ	stack_oldZsize_oldrb   Zsize_newZvar2Z	size_new2r   r   rC   r3   r  r4   r  x  s\    




zView._dynamic_reshape_indexerc                 C   s   | j S rK   r  r   r3   r3   r4   r     s    zView.get_sizec                    s    fdd}j   |S )Nc                    s     | S rK   r  rA   r  r   r3   r4   r    s    zView.make_loader.<locals>.load)r0  r   r  r3   r  r4   r     s    
zView.make_loaderN)r   r   r   r   r   r   r   ro   r   r  r  r   r   r   r   r  r  r  r   r   r3   r3   r3   r4   r  !  s$   

	



>r  c                   @   s~   e Zd ZU dZded< dd Zdd ZeZdd	 Zd
d Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd ZdS )r  z*Pretend our storage has a different layoutLayoutrm   c                 C   s   t | jtr| j | _d S rK   )r,   r0  r  r  r   r3   r3   r4   r     s    zReinterpretView.__post_init__c                 C   s   |  | j| jgS rK   )r   r0  rm   r   r3   r3   r4   r     s
    zReinterpretView.__str__c                 C   s
   | j  S rK   r  r   r3   r3   r4   r    s    zReinterpretView.get_namec                 C   s   | j jS rK   rm   r   r   r3   r3   r4   r     s    zReinterpretView.get_devicec                 C   s   | j jS rK   )rm   r   r   r3   r3   r4   r     s    zReinterpretView.get_dtypec                 C   s   t | jjS rK   rw  rm   r   r   r3   r3   r4   r     s    zReinterpretView.get_sizec                 C   s   t | jjS rK   rw  rm   r   r   r3   r3   r4   
get_stride  s    zReinterpretView.get_stridec                    s    fdd}|S )Nc                    s    j  }t  || S rK   )rm   ro   r(   r  r  r:   rw   r   r3   r4   r     s    
z+ReinterpretView.make_loader.<locals>.loaderr3   r   r   r3   r   r4   r     s    zReinterpretView.make_loaderc                 C   s
   | j  S rK   rm   ro   r   r3   r3   r4   ro     s    zReinterpretView.make_indexerc                 C   s   | j S rK   rm   r   r3   r3   r4   r     s    zReinterpretView.get_layoutc                 C   s   d S rK   r3   r   r3   r3   r4   r    s    zReinterpretView.freeze_layoutc              
   C   s   t jj| jj}t jj| jj}t jj| jj}t jjj	}|dkrn| d| 
  d| d| d| d
S | d| 
  d| d| dS )N0(r  ))r)   rg   r   codegen_shape_tuplerm   r   r   Zcodegen_sizevarr  
as_stridedr  )r   r   r   r  r  r3   r3   r4   codegen_reference  s    
&z!ReinterpretView.codegen_referenceN)r   r   r   r   r   r   r   r   r  r   r   r   r  r   ro   r   r  r  r3   r3   r3   r4   r    s   
r  c                   @   s   e Zd ZedddZdS )	SliceViewr   c                    sz  t dksJ z"dkr6|dkr6dkr6|W S W n tyJ   Y n0 tjj}t| |   | |  }|	|  }|	|	  |dkr|
|   dkrdkr||   |S t| d   < t|rZt|\}}t|j}	|	   |	 < t|j|j|	|j|j    }
t||
S  fdd}t||dS )Nr   l            r   c                    sD   t | t ks$J d|  d t| } |     |  < | S )Nzwrong ndim r  )r?   rw  rA   r  r  startstepr3   r4   rC   &  s    $z!SliceView.create.<locals>.reindex)r   rC   )r   r\  	TypeErrorr)   rg   r   rw  r   r  Z	guard_minr   r  r   r   r  r   rb  r   r   r  r  r  )r   r~   r  r  endr  r   r  r  r  r  rC   r3   r  r4   r      s>    

&


zSliceView.createN)r   )r   r   r   r   r   r3   r3   r3   r4   r    s   r  c                   @   sD   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dS )BaseConstantc                 C   s   dS Nr3   r3   r   r3   r3   r4   r   1  s    zBaseConstant.get_sizec                 C   s   | j S rK   r   r   r3   r3   r4   r   4  s    zBaseConstant.get_dtypec                 C   s   | j S rK   r   r   r3   r3   r4   r   7  s    zBaseConstant.get_devicec                 C   s   d S rK   r3   r  r3   r3   r4   r  :  s    zBaseConstant.mark_reusec                 C   s   dS r   r3   r   r3   r3   r4   r  =  s    z#BaseConstant.has_exceeded_max_readsc                 C   s   dS r  r3   r   r3   r3   r4   r   @  s    zBaseConstant.get_readsc                 C   s   dS r   r3   r   r3   r3   r4   r   C  s    zBaseConstant.is_externN)
r   r   r   r   r   r   r  r  r   r   r3   r3   r3   r4   r  0  s   r  c                   @   s:   e Zd ZU eed< ejed< ejed< dd Zdd Z	dS )	Constantvaluer   r   c                    s    fdd}|S )Nc                    s   t  j jS rK   )r(   rm  r  r   rA   r   r3   r4   r   N  s    z$Constant.make_loader.<locals>.loaderr3   r  r3   r   r4   r   M  s    zConstant.make_loaderc                 C   s   d S rK   r3   r   r3   r3   r4   r  S  s    zConstant.realizeN)
r   r   r   r   r   r/   r   r   r   r  r3   r3   r3   r4   r  G  s
   


r  c                   @   s2   e Zd ZU eed< ejed< ejed< dd ZdS )IndexingConstantr:   r   r   c                    s    fdd}|S )Nc                    s   t  j jS rK   )r(   r]  r:   r   rA   r   r3   r4   r   ^  s    z,IndexingConstant.make_loader.<locals>.loaderr3   r  r3   r   r4   r   ]  s    zIndexingConstant.make_loaderN)	r   r   r   r   r   r/   r   r   r   r3   r3   r3   r4   r  W  s   


r  c                   @   s   e Zd Zedfejejee ee edddZ	e
dd Zdd ZeZd	d
 Zdd Zdd Zdd Zdd Zdd Zdd ZedddZdS )r  r   r   r   r   r   r  c                 C   s8   || _ || _tdd |D s"J || _|| _|| _d S )Nc                 s   s   | ]}t |ttfV  qd S rK   )r,   r   rg  r   r3   r3   r4   r   p  r=   z"Layout.__init__.<locals>.<genexpr>)r   r   r5  r   _strider  r   r   r   r   r   r  r3   r3   r4   r  f  s    zLayout.__init__c                 C   s   | j S rK   )r  r   r3   r3   r4   r   u  s    zLayout.stridec                 C   sP   d}| j dkrd| j  }t| j d| jj d| j d| j d| j | dS )	N r   z	, offset=z('z', z, size=z	, stride=r  )r  r1   r   r   r   r   r   )r   r  r3   r3   r4   r   y  s    
zLayout.__str__c                 C   s>   t | jt| j| jD ] \}}}|dkr||kr dS qdS Nr   FT)rE   r   r   rc  r   r   leftrightr   r3   r3   r4   r    s    zLayout.is_contiguousc                 C   sR   t | j}|dvrdS t| jt| j| jD ] \}}}|dkr,||kr, dS q,dS )N)r*     Fr   T)r?   r   rE   r   r   )r   ndimr  r  r   r3   r3   r4   is_channels_last_contiguous  s    
z"Layout.is_channels_last_contiguousc                 C   sB   t | jtt| j| jD ] \}}}|dkr||kr dS qdS r  )rE   r   r  r   rc  r   r  r3   r3   r4   is_transposed  s    zLayout.is_transposedc                 C   s   t | jt |ksJ d gt | }tt |D ] }tjj| j| ||| < q0tt |d D ]}|| ||d  krb dS qbdS r  )r?   r   r@   r)   rg   r   r   )r   rG   stride_orderedr9   r3   r3   r4   r    s    zLayout.is_stride_orderedc                 C   s:   dgt ttdt| jd  }t|g| }| |S rO  )rw  r  r@   r?   r   r  r   rG   r3   r3   r4   is_channels_last_stride_ordered  s    "z&Layout.is_channels_last_stride_orderedc                 C   s   t | j| j| j| j| jS rK   )rb  r   r   r   r   r  r   r3   r3   r4   rn     s    zLayout.as_fixedc                 C   s(   t jsJ dt| j d|   S )Nzconvert z to FixedLayout first)r   r   r1   r   rn   ro   r   r3   r3   r4   ro     s
    zLayout.make_indexer)r  c                 C   s<   | j |j ko:| j|jko:| j|jko:| j|jko:| j|jkS rK   r   )r   otherr3   r3   r4   __eq__  s    



zLayout.__eq__N)r   r   r   r   r/   r   r   r   r   r  propertyr   r   r   r  r
  r  r  r  rn   ro   re  r  r3   r3   r3   r4   r  d  s(   
	
	r  c                       sL   e Zd ZdZdedfejejee	 ee	 e	d fddZ
dd Z  ZS )	rb  z A Tensor layout we cannot changeNr   r   c                    s*   |d u rt |}t ||||| d S rK   )r   rc  superr  r  	__class__r3   r4   r    s    
zFixedLayout.__init__c                    s    fdd}|S )z1A closure containing math to read a given elementc                    sd   t | t  j  kr$t  jks*n J  j}t|  j jD ]\}}}|dkr@|||  }q@|S r   )r?   r   r   r  rE   )r:   r   rQ   r   szr   r3   r4   rw     s    *z)FixedLayout.make_indexer.<locals>.indexerr3   r  r3   r   r4   ro     s    zFixedLayout.make_indexer)r   r   r   r   r   r/   r   r   r   r   r  ro   __classcell__r3   r3   r  r4   rb    s   rb  c                       sn   e Zd ZdZdZedd Zedd Zedd Zed	d
 Z	dd Z
dd Zdd Zd fdd	Z  ZS )r   z(A Tensor layout we are allowed to changeFc                 C   sP   t | dkrg S tdg}t| dd  D ]}|||d   q,tt|S )Nr   r   r  )r?   r   r   r  r   rw  )sizesZreversed_stridesr   r3   r3   r4   rc    s    z!FlexibleLayout.contiguous_stridesc                 C   sV   t tt| t |ksJ td}dgt| }|D ]}|||< || |  }q8|S )z
        Create a stride based on the order the dimensions should be filled in.

        In this format, channels last would be:
            [1, 3, 2, 0]
        r   N)r   r@   r?   r   r   )r  rG   Znext_striderA  r9   r3   r3   r4   fill_ordered  s    
zFlexibleLayout.fill_orderedc                 C   s0   t tt| t |ksJ t|}t| |S )z
        Create a stride based on the sorted order of a permuted range.

        In this format, channels last would be:
            [3, 0, 2, 1]
        )r   r@   r?   rX   r   r  )r  rG   rW   r3   r3   r4   r    s    zFlexibleLayout.stride_orderedc                 C   sD   t | t |ksJ dd |D }ttt ||jd}t| |S )z
        Create a stride that has the same stride order as given stride

        For example, if given stride is [1000, 1, 100, 10],
        the fill order should be [1, 3, 2, 0]
        c                 S   s   g | ]}t jj|qS r3   )r)   rg   r   r   rE  r3   r3   r4   r<   #  r=   z/FlexibleLayout.same_ordered.<locals>.<listcomp>rd   )r?   rp   r@   __getitem__r   r  )r  r   rW   r3   r3   r4   same_ordered  s    zFlexibleLayout.same_orderedc                 C   s"   t | j| j| j| | j|| jS rK   )rb  r   r   r   r  r  r  r3   r3   r4   as_stride_order'  s    zFlexibleLayout.as_stride_orderc                 C   s"   t | j| j| j| | j|| jS rK   )rb  r   r   r   r  r  r  r3   r3   r4   as_fill_order0  s    zFlexibleLayout.as_fill_orderc                 C   s"   t | j| j| j| | j|| jS rK   )rb  r   r   r   r  r  r   r   r3   r3   r4   as_same_order9  s    zFlexibleLayout.as_same_orderNc                    s2   |rt ||}n
t |}t |||| d S rK   )r   r  rc  r  r  )r   r   r   r   r  rA  r  r3   r4   r  B  s    
zFlexibleLayout.__init__)N)r   r   r   r   r   r   rc  r  r  r  r  r  r  r  r  r3   r3   r  r4   r     s   



			r   c                       s6   e Zd ZdZdd fddZdd Zdd	 Z  ZS )
rq   z)Shares the same storage as another tensorr  r  c                    s,   |  }t |j|j|j|j || _d S rK   )r   r  r  r   r   r   r   r  )r   r  rm   r  r3   r4   r  M  s    zAliasedLayout.__init__c                 C   s   |    S rK   )rn   ro   r   r3   r3   r4   ro   W  s    zAliasedLayout.make_indexerc                 C   s4   | j  j}|dkrdS ddlm} tjj||S )Nr   Tr   )	ALIGNMENT)	r  r   r  Z
compile_fxr  r)   rg   r   Zmaybe_guard_multiple_of)r   r  r  r3   r3   r4   maybe_guard_alignedZ  s
    z!AliasedLayout.maybe_guard_aligned)r   r   r   r   r  ro   r   r  r3   r3   r  r4   rq   J  s   
rq   c                       sV   e Zd Zed fddZejjdd Zdd Ze	dd	 Z
d
d Zdd Z  ZS )MutationLayout)r  c                    s(   t  | | | d  || _d S rK   )r  r  r   r   r   r  )r   r  r  r3   r4   r  d  s    zMutationLayout.__init__c                 C   s
   |   jS rK   )real_layoutr   r   r3   r3   r4   r   m  s    zMutationLayout.stridec                 C   s    t | jtr| j S | jjjS rK   )r,   r  r!  r"  r0  rm   r   r3   r3   r4   r"  q  s    
zMutationLayout.real_layoutc              	   C   s   |   tj|  t|tr(|j}t|tr@|	| rFd}n|   t|jj
t }|rtj| | | dd t| | D dj}|   t|jj
tsJ t||j_
|jS )NTc                 S   s    g | ]\}}t jj||qS r3   r)   rg   r   r  r8   rH  rI  r3   r3   r4   r<     s   z/MutationLayout.realize_into.<locals>.<listcomp>rp  )r  r)   rg   realize_users_ofr  r,   r-   r0  r  r   rm   r   r   r   r   r   r   rE   r   r!  )r   srcdstZ	need_copyr3   r3   r4   realize_intov  s*    
	zMutationLayout.realize_intoc                 C   s   | S rK   r3   r   r3   r3   r4   rn     s    zMutationLayout.as_fixedc                 C   s
   | j  S rK   )r  ro   r   r3   r3   r4   ro     s    zMutationLayout.make_indexer)r   r   r   r   r  r  r   getterr"  r   r(  rn   ro   r  r3   r3   r  r4   r!  c  s   	

r!  c                   @   s   e Zd ZU eed< eed< dd Zdd Zdd Zd	d
 Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zed)d* Zd+d, Zd-d. Zd/S )0r  r`   rm   c                 C   s
   | j  S rK   r  r   r3   r3   r4   ro     s    zBuffer.make_indexerc                 C   s   | j s
J | j S rK   r_   r   r3   r3   r4   r    s    
zBuffer.get_namec                 C   s   | j jS rK   r  r   r3   r3   r4   r     s    zBuffer.get_devicec                 C   s   t | jdd S )Nr   )r   rm   r   r3   r3   r4   r     s    zBuffer.get_dtypec                 C   s   t | jjS rK   r  r   r3   r3   r4   r     s    zBuffer.get_sizec                 C   s   t | jjS rK   r  r   r3   r3   r4   r    s    zBuffer.get_stridec                 C   s   | j S rK   r  r   r3   r3   r4   r     s    zBuffer.get_layoutc                 C   s   |   S rK   )r   r   r3   r3   r4   r    s    zBuffer.get_storage_numelc                 C   s   dS r   r3   r   r3   r3   r4   r     s    zBuffer.is_externc                 C   s   t | jts| j | _d S rK   )r,   rm   MultiOutputLayoutrn   r   r3   r3   r4   r    s    zBuffer.freeze_layoutc                 C   s"   t | jtsJ | j|| _d S rK   )r,   rm   r   r  r  r3   r3   r4   r    s    z&Buffer.freeze_layout_with_stride_orderc                 C   s"   t | jtsJ | j|| _d S rK   )r,   rm   r   r  r  r3   r3   r4   freeze_layout_with_fill_order  s    z$Buffer.freeze_layout_with_fill_orderc                 C   s"   t | jtsJ | j|| _d S rK   )r,   rm   r   r  r  r3   r3   r4   freeze_layout_with_same_order  s    z$Buffer.freeze_layout_with_same_orderc                    s    fdd}|S )Nc                    s    j  }t j|| S rK   )rm   ro   r(   r  r`   r  r   r3   r4   r     s    
z"Buffer.make_loader.<locals>.loaderr3   r  r3   r   r4   r     s    zBuffer.make_loaderc                 C   s   dS r   r3   r   r3   r3   r4   is_no_op  s    zBuffer.is_no_opc                 C   s   |   S rK   r  r   r3   r3   r4   r    s    zBuffer.codegen_referencec                 C   s   d S rK   r3   r   r3   r3   r4   r6    s    zBuffer.decide_layoutc                 C   s   t | jtr| jj gS dS r  )r,   rm   rq   r  r  r   r3   r3   r4   get_alias_names  s    zBuffer.get_alias_namesc                 C   s   t | jtr| jj gS dS r  r,   rm   r!  r  r  r   r3   r3   r4   get_mutation_names  s    zBuffer.get_mutation_namesc                 C   sD   t tdd" t|  |  W  d    S 1 s60    Y  d S r   )r   r   r   r   r   r   r   r3   r3   r4   rr     s
    zBuffer.get_read_writesc                 C   s
   |   jS rK   )rr   rs   r   r3   r3   r4   r     s    zBuffer.get_readsc                 C   s   d S rK   r3   r   r3   r3   r4   r    s    zBuffer.realizeN)r   r   r   r   r   r  ro   r  r   r   r   r  r   r  r   r  r  r+  r,  r   r-  r  r6  r/  r1  r    rr   r   r  r3   r3   r3   r4   r    s2   

r  c                   @   s   e Zd ZdS )InputBufferN)r   r   r   r3   r3   r3   r4   r2    s   r2  c                   @   s    e Zd ZdZdd Zdd ZdS )r   Nc                    s    fdd}|S )Nc                    s(    j  }ttj j j|| S rK   )	rm   ro   r(   r  r)   rg   constant_namer`   r   r  r   r3   r4   r     s    
z*ConstantBuffer.make_loader.<locals>.loaderr3   r  r3   r   r4   r     s    zConstantBuffer.make_loaderc                 C   s   t tj| j|| jS rK   )r   r)   rg   r3  r`   rm   r   r   r3   r3   r4   r     s    z!ConstantBuffer.constant_to_device)r   r   r   r   r   r   r3   r3   r3   r4   r     s   	r   c                   @   s   e Zd Zdd ZdS )r.   c                 C   s   |   d S )Nz.clone()r.  r   r3   r3   r4   r  
  s    z RandSeedBuffer.codegen_referenceN)r   r   r   r  r3   r3   r3   r4   r.   	  s   r.   c                   @   s   e Zd Zdd Zdd ZdS )NoneAsConstantBufferc                 C   s   dS )NNoner3   r   r3   r3   r4   r    s    z&NoneAsConstantBuffer.codegen_referencec                 C   s   dS )Nzat::Tensor()r3   r   r3   r3   r4   cpp_wrapper_codegen_reference  s    z2NoneAsConstantBuffer.cpp_wrapper_codegen_referenceN)r   r   r   r  r7  r3   r3   r3   r4   r5    s   r5  c                       s$   e Zd Z fddZdd Z  ZS )ShapeAsConstantBufferc                    s   t    || _d S rK   )r  r  shape)r   r9  r  r3   r4   r    s    
zShapeAsConstantBuffer.__init__c                 C   s   t tjj| jS rK   )r   r)   rg   r   rv  r9  r   r3   r3   r4   r    s    z'ShapeAsConstantBuffer.codegen_reference)r   r   r   r  r  r  r3   r3   r  r4   r8    s   r8  c                   @   sx   e Zd ZU eed< edd Zdd Zdd Zdd	 Z	d
d Z
edddZdd Zdd Zdd Zdd Zdd ZdS )r3  r0  c                 C   s   t tddf | j rDt|  | j | j W  d    S t|  | j W  d    S W d    n1 sz0    Y  d S r   )	r   r   r   r0  r   r   get_store_functionr   r   r   r3   r3   r4   rr   %  s    
zComputedBuffer.get_read_writesc                 C   s@   | j   }| j r*t| jj| j|S t| jj| j|S d S rK   )	rm   rn   ro   r0  r   r   r  r`   r   r  r3   r3   r4   r:  4  s    
z!ComputedBuffer.get_store_functionc           	         s   t | jtrt| j | j \}\ }|  j	}dd |D }g }t
|D ]&\}}t |trR|jdkrR|| qRfdd|D }|r fdd|D }ddlm} |||  |S dS )	al  
        If our layout is still flexible, try to determine the stride order based on stride orders of reads.

        TODO(jansel): A better algorithm here would look at downstream consumers of this
                      value and try to do global graph-level layout optimization.
                      This is also something just begging to be autotuned.
        c                 S   s0   g | ](}|j tjj v r(tjj|j  nd qS rK   rf   rj   r3   r3   r4   r<   H  s   z1ComputedBuffer.get_fill_order.<locals>.<listcomp>aten.convolutionc                    s"   g | ]}t |jd d  D qS )c                 S   s    i | ]}|d kr|t d qS rr  rs  r8   vr3   r3   r4   rS   Y  r=   z<ComputedBuffer.get_fill_order.<locals>.<listcomp>.<dictcomp>)r&   r:   rj   )r  r3   r4   r<   W  s   c                    s   g | ]}t jj| qS r3   r)   rg   r   r>  r8   exprrx   r3   r4   r<   _  s   r   pick_loop_orderN)r,   rm   r   r   r=  r0  r   r   rr   rs   rV   rl   kernelr   	schedulerrC  )	r   rZ   rs   ry   r   r9   rz   Zstride_lengthsrC  r3   )rx   r  r4   get_fill_order;  s2    


zComputedBuffer.get_fill_orderc                 C   s0   t | jtr,|  }|r$| | n|   d S rK   )r,   rm   r   rF  r+  r  r  r3   r3   r4   r6  h  s
    zComputedBuffer.decide_layoutc                    s  t jj j dd\}}}ttd 4 t	
  rJ|n
|dd |}W d   n1 sn0    Y  g |j  dd |j D }g g |j |j g }g }g }g }	| D ]Z\}
}|
|d v r|rJ ||
 || q|
|d v sJ ||
 |	| qttt|gt }t|D ],\}}t|trLt|d	rL|j||< qLd fd
d	}||||\}}}|||	\}}}|_t j||dd\\}}}t	|||||g|}||f|fS )a  
        This is a main place where we do loop transformations in a
        backend-agnostic way.

        Here we:
            1) Remove any 1 dimensions
            2) Fuse contiguous dimensions together
            3) Reorder dimensions based on stride orders
        qr*   r   Nr   c                 S   s,   g | ]$}|t jj v r$t jj| nd qS rK   )r)   rg   rh   ri   )r8   Z
reads_namer3   r3   r4   r<     s   z7ComputedBuffer.simplify_and_reorder.<locals>.<listcomp>r   iter_reordering_reindexc                    s\    | ||\}}}|| } tjj| |t | |\}}}|| } t||}|||fS rK   )_apply_loop_reorderingr)   rg   r   _simplify_loopsr   rO   )Zx_varsr  reordering_reindexZreindex0rM   rN   prunerC   Zindex_formulasr   r   r   r3   r4   simplify_and_reorder  s    




zAComputedBuffer.simplify_and_reorder.<locals>.simplify_and_reorderz)N)r   r=  r0  r   r   r   r   r   r   LoopBodyr:  r   indexing_exprsvaluesreads_name2exprri   writes_name2expritemsr   rJ   r@   r?   rV   r,   r3  r   rH  Zindex_vars_no_squeeze)r   rZ   r   rv   r  ry   rx   Zreduce_varsZ
index_sizeZreduce_sizer=  r   rK  r9   rz   rN  Ziter_rangesZiter_reindexrH  Zreduce_rangesZreduce_reindexZ	iter_varsr3   rM  r4   rN  p  sh    
"



z#ComputedBuffer.simplify_and_reorderNc           	   
      s  ddl m} |du rg }z fdd|D }t|t|krPt|d t ksTJ |durtt|D ]0}z|| || ||< W qh ty   Y qh0 qhtt|||}W nH ty   tj	rt
dtt  d|  ttt}Y n0 fd	d|D t|t|fS )
zU
        Shuffle the order of loops around to hopefully improve performance.
        r   rB  Nc                    s   g | ]}t jj| qS r3   r>  r?  rA  r3   r4   r<     s   z9ComputedBuffer._apply_loop_reordering.<locals>.<listcomp>r   z Did not simplify complex index:

c                    s   g | ]} | qS r3   r3   r7   )r  r3   r4   r<     r=   )rE  rC  r?   r@   r  rw  r  	Exceptionr   debuglogwarningrD   rE   rJ   rH   )	rx   r  r   rK  r   rC  rA  r9   rG   r3   )rx   r  r4   rI    s2    

z%ComputedBuffer._apply_loop_reorderingc                 C   s
   | j  S rK   )r0  r   r   r3   r3   r4   r     s    z!ComputedBuffer.get_reduction_sizec                 C   s
   | j  S rK   )r0  r   r   r3   r3   r4   r     s    z!ComputedBuffer.get_reduction_typec                 C   s
   | j  S rK   )r0  r   r   r3   r3   r4   r-    s    zComputedBuffer.is_no_opc                 C   s   dS NTr3   r   r3   r3   r4   should_allocate  s    zComputedBuffer.should_allocatec                 C   s   | j |S )r   )r0  r   r4  r3   r3   r4   r     s    z!ComputedBuffer.constant_to_device)NN)r   r   r   r   r   r    rr   r:  rF  r6  rN  r   rI  r   r   r-  r\  r   r3   r3   r3   r4   r3  !  s   

-Y $r3  c                       s\   e Zd ZdZ fddZdd Zedd Zdd	 Zd
d Z	dd Z
dd Zdd Z  ZS )TemplateBufferzu
    Represents a Triton (in the futurue other type) of template operator
    that we can fuse an epilogue onto.
    c                    s4   t  jd |d t|| _|| _tj| | _	d S )N)r`   rm   )
r  r  InputsKernelunwrap_storageinputsmake_kernel_renderr)   rg   register_bufferr`   )r   rm   r`  ra  r  r3   r4   r  	  s    zTemplateBuffer.__init__c                 C   s   |   S rK   )normalized_read_writesr   r3   r3   r4   rr   	  s    zTemplateBuffer.get_read_writesc                    sL   |   | j   fdd}tj||  ddd}dd | jD |_|S )Nc                    s"   t |dksJ t | dS )Nr   Zfake)r?   r(   r   r_  rw   r`   r3   r4   dummy	  s    z4TemplateBuffer.normalized_read_writes.<locals>.dummyr3   T	normalizec                 S   s   h | ]}t | qS r3   r   StarDepr  rE  r3   r3   r4   	<setcomp>	  r=   z8TemplateBuffer.normalized_read_writes.<locals>.<setcomp>)r  rm   ro   r   r   r   r`  rs   )r   re  depsr3   rd  r4   rc  	  s    
z%TemplateBuffer.normalized_read_writesc                 C   s   dS r   r3   r   r3   r3   r4   r   	  s    z!TemplateBuffer.get_reduction_sizec                 C   s   d S rK   r3   r   r3   r3   r4   r    	  s    z!TemplateBuffer.get_reduction_typec                 C   s   dS r   r3   r   r3   r3   r4   r-  #	  s    zTemplateBuffer.is_no_opc                 C   s   dS r[  r3   r   r3   r3   r4   r\  &	  s    zTemplateBuffer.should_allocatec                 C   s   |   dfd fS r  )r   r   r3   r3   r4   rN  )	  s
    z#TemplateBuffer.simplify_and_reorder)r   r   r   r   r  rr   r    rc  r   r   r-  r\  rN  r  r3   r3   r  r4   r]    s   
r]  c                   @   s6   e Zd ZU ee ed< dd Zedd Zdd Z	dS )	r^  r`  c                 C   s,   t dd | jD t |  ht g d S )Nc                 S   s   h | ]}t | qS r3   rh  rE  r3   r3   r4   rj  9	  r=   z/InputsKernel.get_read_writes.<locals>.<setcomp>)r   Z
ReadWritesr`  ri  r  r   r   r3   r3   r4   rr   7	  s    zInputsKernel.get_read_writesc                 C   sp   g }| D ]b}t |tr|j}t |tr,|j}t |trJt |tsJt|}t |ttfs`J ||	| q|S rK   )
r,   r-   r0  r  r  r  ExternKernelrealize_inputr  r   )r`  Z
inputs_newr~   r3   r3   r4   r_  @	  s    


zInputsKernel.unwrap_storagec                 C   s   dS r[  r3   r   r3   r3   r4   r   N	  s    zInputsKernel.is_externN)
r   r   r   r   r  r   rr   r   r_  r   r3   r3   r3   r4   r^  3	  s
   
	
r^  c                   @   s   e Zd Zdd ZdS )	NopKernelc                 C   s   dS r[  r3   r   r3   r3   r4   r-  S	  s    zNopKernel.is_no_opN)r   r   r   r-  r3   r3   r3   r4   rn  R	  s   rn  c                   @   s0   e Zd ZdZedd Zedd Zdd ZdS )	ConcatKernelzn
    There isn't actually a real kernel for concat, we just change the
    storage for the upstream data.
    c                 C   s  |d   }|d  }t|d  }dg}|| g}d|  krPt|k sVn J tdt|D ]}||  }	|||  t|	t|ksJ ||  |ksJ ||   |ksJ tt|D ]>}
|
|kr||
 |	|
  ||
< qtjj	
||
 |	|
 ||
< q|||  qdt|}tt|D ]D}|| }t|r0| }t|tr0| r0t|} qvq0td t||||dg d}t|}tt|D ]4}|jj| || t|||| ||  qtj|j|j_| |jj|j_|S )Nr   r   )r   r   r   r   )r`   rm   r`  )r   r   rw  r   r?   r@   r   r)   rg   r   r  r   rc  r   r   r,   rb  r
  r   ro  r  r0  r`  r(  r  r   rb  r`   r_  )r   r`  r  r   r   r  Zoffsets_startZoffsets_endr9   
input_sizer  output_strider~   rm   rD  r3   r3   r4   r   ]	  sf    





zConcatKernel.createc              	   C   s   t |ts(t|r(t|\}}t||}t |ts:J |t |trR| |j|S t |tr|  t |jj	t
rt |jtst||j_	|jS tj| | | dd t| | D d}| ||S )Nc                 S   s    g | ]\}}t jj||qS r3   r#  r$  r3   r3   r4   r<   	  s   z-ConcatKernel.realize_into.<locals>.<listcomp>rp  )r,   r  r   r  r-   r(  r0  r  r  rm   r   r  rq   r   r   r   r   r   rE   r   )r   r&  r'  r  rm   pwr3   r3   r4   r(  	  s.    



	zConcatKernel.realize_intoc                 C   s   dS r[  r3   r   r3   r3   r4   r\  	  s    zConcatKernel.should_allocateN)r   r   r   r   r   r   r(  r\  r3   r3   r3   r4   ro  W	  s   
;
 ro  c                   @   s   e Zd ZU dZeedf ed< eje	dZ
eeef ed< dZee ed< dd	 Zd
d Zedd Zedd Zedd Zedd Zedd Zedd Zedd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Z d&d' Z!d(d) Z"dS )*rl  r3   .constant_args)default_factoryr   Noutput_viewc                 C   s    t | jtr|   |   d S rK   )r,   rm   r   apply_constraintr  r   r3   r3   r4   r6  	  s    zExternKernel.decide_layoutc                 C   s   t d S rK   ra  r   wrapperr3   r3   r4   codegen	  s    zExternKernel.codegenc                 C   s.   t j|  |  |  |  d}|  |S )Nrp  )r   r   r   r   r   r   r  )r~   rr  r3   r3   r4   
copy_input	  s    zExternKernel.copy_inputc                    s  t |j|i |j}t|\} g g }g }|D ]R}t|t d r\|| q4t|tj	r|t
jjjj|d d}|| q4 fdd}	fdd|D }|D ]}
t|
rt|
dd qg }|D ]}
|t|
dd	 q|	||\}}||i |}||||	fS )
Nr  )ry  c                    sd   g }t | }t |}D ]&}|r0|t| q|t| qt| }|dg |di fS )Nr   r   )iterr   nextpytreeZtree_unflattenget)Znew_tensor_argsZnew_non_tensor_argsr   Z
it_tensorsZit_non_tensorsZ	is_tensor)	args_specis_arg_tensorr3   r4   unflatten_args	  s    z3ExternKernel.process_kernel.<locals>.unflatten_argsc                    s   g | ]}  |qS r3   rm  rE  )r   r3   r4   r<   	  r=   z/ExternKernel.process_kernel.<locals>.<listcomp>Tr  r   )r   bind	argumentsr~  Ztree_flattenr   r,   r   r   r   r)   rg   r   Z	shape_envZcreate_symintnoder   r  r   )r   rD  r   r   Zbinded_argsZ	args_flattensor_argsnon_tensor_argsargr  r~   Zexample_argsnew_argsZ
new_kwargsexample_outputr3   )r  r   r  r4   process_kernel	  s.    zExternKernel.process_kernelc              	   C   s   t |tsJ t |tr|S |   t| | dd}t|j	dksPJ t
jjt|j	d j|j}t
jj||j}t
jj||j}t|j|| }||krtd||| t t|jt| | | ||ddS )z
        In order to pass this to an extern kernel we need a
        ReinterpretView not a View.  This allows us to avoid some
        uneeded copies.
        Frf  r   r   z@convert_to_reinterpret_view failed: stride=%s offset=%s index=%sr   )r0  rm   )r,   r  r  r  r  r   r   r   r?   rs   r)   rg   r   Zsimplify_with_rangesrw  r:   rv   Zstride_varsr4  Z
offset_varr$   rY  rX  ra  r0  rb  r   r   )r   r~   rwr:   rA  r  expectedr3   r3   r4   convert_to_reinterpret_view	
  s<    
z(ExternKernel.convert_to_reinterpret_viewc                 C   s   |d u rt  S t|tjtfr&t|S t|trRtj	t
j|j| | dS t|tr`|S t|trv| |jS t|tr|S t|tr|  t| rt| jtsz| |W S  ty   Y n0 t|tr|  |S | |S )N)r   r   )r5  r,   r   r   rg  r8  r  r)   rg   Zadd_tensor_constantr/   Ztensorr  r   r   r   r-   rm  r0  r  r  r  r   r  r  r  ra  r  r{  r   r~   r3   r3   r4   rm  3
  s6    






zExternKernel.realize_inputc                 C   sD   t |r:t| dkr|S | D ]}|dkr$|  S q$| |S rO  )r   r?   r  r{  )r   r~   r   r3   r3   r4   require_stride1R
  s    
zExternKernel.require_stride1c                 C   s   |  dkr|S t|rt| tr:t|dd|d |S t| trZ| |rZ|S t| trt| 	 trt
dn(t| 	 tr| 	 |r|S t|tr| |r|S | |}t|dd|d t||sJ |S )Nr   TFr  z<the MutationLayout's real layout shouldn't be FlexibleLayout)r   r   r,   r   r   r  rb  r  r!  r"  r  r2  r{  r  )r   r~   rG   r3   r3   r4   require_stride_order\
  s>    
z!ExternKernel.require_stride_orderc              	   C   s    |  |tttt| S rK   )r  rw  r  r@   r?   r   r  r3   r3   r4   require_contiguous
  s    zExternKernel.require_contiguousc                 C   s   d S rK   r3   r   r3   r3   r4   rv  
  s    zExternKernel.apply_constraintc                 C   s&   dd | j D }|tt| j |S )Nc                 S   s   g | ]}|  qS r3   r  rE  r3   r3   r4   r<   
  r=   z-ExternKernel.codegen_args.<locals>.<listcomp>)r`  extendr   reprrs  )r   r   r3   r3   r4   codegen_args
  s    zExternKernel.codegen_argsc                 C   s"   g }| j rdd | j  D }|S )Nc                 S   s"   g | ]\}}| d t | qS r   r  r8   kr=  r3   r3   r4   r<   
  r=   z/ExternKernel.codegen_kwargs.<locals>.<listcomp>)r   rU  )r   r   r3   r3   r4   codegen_kwargs
  s    zExternKernel.codegen_kwargsc                 C   sJ   g }| j rF| jD ]4}|| j v s*J d| | j |}|t| q|S )Nzarg %s not found in self.kwargs)r   Zordered_kwargs_for_cpp_kernelr  r   r  )r   r   Zarg_namer=  r3   r3   r4   cpp_wrapper_codegen_kwargs
  s    
z'ExternKernel.cpp_wrapper_codegen_kwargsc              	   C   sP   t jrLtjj|  }tjj|  }|d| 	  d| d| d d S )Nzassert_size_stride(r  r  )
r   Zsize_assertsr)   rg   r   r  r   r  	writeliner  )r   ry  r   r   r3   r3   r4   codegen_size_asserts
  s    z!ExternKernel.codegen_size_assertsc                 C   s   |   }|  }|g g|fS )zD
        get output sizes and strides, for template_codegen
        )r   r  )r   _sizer  r3   r3   r4   get_group_stride
  s    zExternKernel.get_group_stridec                    s  t jj|  }|  }fdd|D }dd tt|D ttt||jdd}dd t	|D fddttD }fd	d|D | 
 }|}t jj||g\}}}	td
\}
 tt| fdd|D }tt||}|t|fS )zC
        Manually get cononicalization of the output index
        c                    s   g | ]}  |qS r3   )r   rE  )r   r3   r4   r<   
  r=   z-ExternKernel.canonicalize.<locals>.<listcomp>c                 S   s   g | ]}t d | qS )dr  r7   r3   r3   r4   r<   
  r=   T)re   reversec                 S   s   i | ]\}}||qS r3   r3   rP   r3   r3   r4   rS   
  r=   z-ExternKernel.canonicalize.<locals>.<dictcomp>c                    s   g | ]} | qS r3   r3   r7   rT   r3   r4   r<   
  r=   c                    s   g | ]} | qS r3   r3   r7   rA  r3   r4   r<   
  r=   cc                    s   g | ]} |qS r3   r3   rE  )add_varr3   r4   r<   
  r=   )r)   rg   r   r   r  r@   r?   rp   r  rV   ro   rJ  r   rD   rE   r&   r   r\  r  )r   r  rA  Zindex_orderrG   rw   r:   Z	new_sizesrC   rL  rZ   replacementr3   )r  rx   rU   r   r4   canonicalize
  s$    
 zExternKernel.canonicalizec                    s"    fddt  D } |S )Nc                    s$   g | ]}|j  d t |j  qS r   )r`   r   )r8   fieldr   r3   r4   r<   
  s   z(ExternKernel.__str__.<locals>.<listcomp>)dataclassesfieldsr   r   r3   r   r4   r   
  s    
zExternKernel.__str__)#r   r   r   rs  r   r   r   r  r  rD   r   r
   r   ru  r   r  r6  rz  r   r{  r   r  r  rm  r  r  r  rv  r  r  r  r  r  r  r   r3   r3   r3   r4   rl  	  s6   



1
)

	
"
	rl  c                       s@   e Zd ZU dZee ed< dd Zd
 fdd	Zdd	 Z	  Z
S )ExternKernelOutNru  c                 C   s^   |   }ddlm} t||r(|  }n|  }|r>|| || j| 	 || j
| j d S )Nr   )CppWrapperCodeGen)r  Ztorch._inductor.codegen.wrapperr  r,   r  r  r  Zgenerate_extern_kernel_outru  r  rD  
cpp_kernel)r   ry  r   r  r   r3   r3   r4   rz  
  s    


zExternKernelOut.codegenr3   c                    sJ   t  d || |||pi  || _tj| | _|d ur@|| _|| _	d S rK   )
r  r  r_  ru  r)   rg   rb  r`   rD  r  )r   rm   r`  rs  r   ru  rD  r  r  r3   r4   r  
  s    
zExternKernelOut.__init__c                 C   s   dS r[  r3   r   r3   r3   r4   r\  
  s    zExternKernelOut.should_allocate)r3   NNNN)r   r   r   ru  r   r  r   rz  r  r\  r  r3   r3   r  r4   r  
  s   
     r  c                       s6   e Zd Zdd Zd
 fdd	Zdd Zdd	 Z  ZS )r  c              
   C   sF   | |   d| j dd|   d t| jtrB| | d S N = r  r  r  	r  r  rD  r   r  r,   rm   r  r  rx  r3   r3   r4   rz    s
    $zExternKernelAlloc.codegenr3   c                    s*   t  d || || tj| | _d S rK   )r  r  r_  r)   rg   rb  r`   )r   rm   r`  rs  r  r3   r4   r  
  s    zExternKernelAlloc.__init__c                 C   s   dS r   r3   r   r3   r3   r4   r\    s    z!ExternKernelAlloc.should_allocatec                 C   s   t d S rK   rw  r   r3   r3   r4   rv    s    z"ExternKernelAlloc.apply_constraint)r3   )r   r   r   rz  r  r\  rv  r  r3   r3   r  r4   r    s   r  c                       s<   e Zd ZdZdZdd Zdd Zdd Z fd	d
Z  Z	S )InplaceBernoulliFallbackzE
    This needs to be a custom class to handle mutation properly
    zaten.bernoulli_c                 C   sB   dd | j D \}|| j d| ddtt| j d d S )Nc                 S   s   g | ]}|  qS r3   r  r8   r   r3   r3   r4   r<     r=   z4InplaceBernoulliFallback.codegen.<locals>.<listcomp>r  r  r  )r`  r  rD  r   r   r  rs  )r   ry  r~   r3   r3   r4   rz    s    $z InplaceBernoulliFallback.codegenc                 C   s   dS r   r3   r   r3   r3   r4   r\  "  s    z(InplaceBernoulliFallback.should_allocatec                 C   s   t | jtsJ | jj fS rK   r0  r   r3   r3   r4   r1  %  s    z+InplaceBernoulliFallback.get_mutation_namesc                    s0   t  d t|| |g| tj| | _d S rK   )r  r  r!  r_  r)   rg   rb  r`   )r   r~   rs  r  r3   r4   r  )  s    
z!InplaceBernoulliFallback.__init__)
r   r   r   r   rD  rz  r\  r1  r  r  r3   r3   r  r4   r    s   r  c                       s4   e Zd ZdZdZdd Zdd Z fddZ  ZS )	IndexPutFallbackzQ
    This needs to be a custom class to handle mutation and indices properly
    zaten.index_put_c           	      C   s   dd | j D ^}}}g }t|}t| jD ]0\}}| j| d urR|t| q,|d q,|| j d| dd| d| dt	| j
d	  d

 d S )Nc                 S   s   g | ]}|  qS r3   r  r  r3   r3   r4   r<   ;  r=   z,IndexPutFallback.codegen.<locals>.<listcomp>r6  r  z, [,z], r  r   r  )r`  r|  rV   r9  r   r}  r  rD  r   r  rs  )	r   ry  r~   rR  valid_indicesr9  Ziter_valid_indicesr9   rZ   r3   r3   r4   rz  :  s    2zIndexPutFallback.codegenc                 C   s   dS r   r3   r   r3   r3   r4   r\  G  s    z IndexPutFallback.should_allocatec                    s^   | _ dd |D } fdd||g|D }t d t| ||g tj  _d S )Nc                 S   s   g | ]}|d ur|qS rK   r3   r7   r3   r3   r4   r<   L  r=   z-IndexPutFallback.__init__.<locals>.<listcomp>c                    s   g | ]}  |qS r3   r  rE  r   r3   r4   r<   M  r=   )	r9  r  r  r!  r_  r)   rg   rb  r`   )r   r~   r9  rR  
accumulater  Ztensorsr  r   r4   r  J  s    zIndexPutFallback.__init__)	r   r   r   r   rD  rz  r\  r  r  r3   r3   r  r4   r  3  s
   r  c                   @   s    e Zd Zedd Zdd ZdS )
DeviceCopyc                 C   sx   |  s(tdd | D r(||S tjj|j tjj|	 j t
d tt|| | d| |gS )Nc                 s   s&   | ]}|j tjjv ot|d V  qdS )r:   N)r`   r)   rg   	constantsr   rj   r3   r3   r4   r   Z  s   z$DeviceCopy.create.<locals>.<genexpr>zDeviceCopy in input programr.  )r   r5  r   r   r)   rg   Zdevice_typesrF  r1   r   r#   r  r   r   r   rm  )r   r~   r   r3   r3   r4   r   X  s    

zDeviceCopy.createc                 C   sb   |   }t|dksJ | jr@|| j  d|d  d n||   d|d  d d S )Nr   z.copy_(r   r  )r  r?   ru  r  r  )r   ry  r   r3   r3   r4   rz  l  s    zDeviceCopy.codegenN)r   r   r   r   r   rz  r3   r3   r3   r4   r  W  s   
r  c                   @   s   e Zd ZdZdd ZdS )DynamicScalarz
    The result of a call to aten._local_scalar_dense.

    This is not yet implemented.  The one model (so far) that calls this
    (fastNLP_Bert) does not actually use the result.  So we expect this
    node to get dead code eliminated.
    c                 C   s   dS r  r3   r   r3   r3   r4   r     s    zDynamicScalar.get_readsN)r   r   r   r   r   r3   r3   r3   r4   r  w  s   r  c                       s>   e Zd Zd
 fdd	Zdd Zedd Z fdd	Z  ZS )FallbackKernelNc                    s   t  |t|t| ttjj|jd |u r>d|j | _n|j	
dd d|j | _|| _|d u rli n|| _tj| j d S )Nzaten.z._ops.z.ops..)r  r  r  r   r/   r(   atenr   rD  r   replacer  r   r)   rg   Zwarn_fallback)r   rm   rD  r  Znontensor_argsr  r   r  r3   r4   r    s    	zFallbackKernel.__init__c                    sx   t jG dd d dd  fdd| jD } fdd| jD }| ||\}}ttt|fdd| D  S )	Nc                   @   s   e Zd ZU eed< dd ZdS )z)FallbackKernel.codegen_args.<locals>.Shimrefc                 S   s   | j S rK   )r  r   r3   r3   r4   r     s    z2FallbackKernel.codegen_args.<locals>.Shim.__repr__N)r   r   r   r   r   r   r3   r3   r3   r4   Shim  s   
r  c                 S   s   |  dt | S )Nr   r  )r  r=  r3   r3   r4   	gen_kwarg  s    z.FallbackKernel.codegen_args.<locals>.gen_kwargc                    s   g | ]} |  qS r3   r  rE  r  r3   r4   r<     r=   z/FallbackKernel.codegen_args.<locals>.<listcomp>c                    s   g | ]} t |qS r3   r  rE  r  r3   r4   r<     r=   c                    s   g | ]\}} ||qS r3   r3   r  )r  r3   r4   r<     r=   )	r  	dataclassr`  rs  r  rw  r   r  rU  )r   r  rs  r   r   r3   )r  r  r4   r    s    zFallbackKernel.codegen_argsc           
         s   t jjt jjt jjt jjt jjt jjt jjt jf}||vrBt	j
jnt }|0 | j|g|R i |\}}}}	W d    n1 s0    Y  |st|tjsJ dtt|r|d  n|j||||	|d fdd	  |S )Nz"Not sure where to find device infor   r  c                    s   t ttfr4t fddttD S t tjrjtt	j
jt t  S t trxS d u sJ dd S d S )Nc                 3   s(   | ] } |  d | dV  qdS )[]Nr3   r7   )generate_outputr:   outputr3   r4   r     s   zAFallbackKernel.create.<locals>.generate_output.<locals>.<genexpr>z+FallbackKernel output type is not supported)r,   rw  r  r1   r@   r?   r/   TensorMultiOutputrb  r   r   r!   r   r   rg  )r  r:   r  packed)r:   r  r4   r    s&    




z.FallbackKernel.create.<locals>.generate_output)r  )r  Z_fft_r2cdefaultr\   Z_fft_c2rZ_fft_c2cZ_linalg_svdUZ*_fused_moving_avg_obs_fq_helper_functionalr)   rg   	fake_moder   r  r,   r/   r  r  r*  r   r   )
r   rD  r   r   Zfake_incorrect_kernelscontextr  r  r  r  r3   r  r4   r     sH    zFallbackKernel.createc                    s
   t   S rK   )r  rv  r   r  r3   r4   rv    s    zFallbackKernel.apply_constraint)N)	r   r   r   r  r  r   r   rv  r  r3   r3   r  r4   r    s   	 
=r  c                   @   s   e Zd ZU ejed< dS )r*  r   N)r   r   r   r/   r   r   r3   r3   r3   r4   r*    s   
r*  c                       s2   e Zd Zdd Zed fddZdd Z  ZS )r  c                 C   s6   | |   d| jd   | j  | | d S )Nr  r   )r  r  r`  r:   r  rx  r3   r3   r4   rz    s     zMultiOutput.codegenrA   c                    s,   t  d ||gd tj| | _|| _d S r  )r  r  r)   rg   rb  r`   r:   )r   rm   inputr:   r  r3   r4   r    s    zMultiOutput.__init__c                 C   s   dS r   r3   r   r3   r3   r4   r\    s    zMultiOutput.should_allocate)r   r   r   rz  r   r  r\  r  r3   r3   r  r4   r    s   r  c                       sl   e Zd ZdZd fdd	Zdd Zedddee ee ee e	ee ed		d
dZ
dd Zdd Z  ZS )rl   r;  r3   Nc                    s    t  ||| || _|| _d S rK   )r  r  rD  preferred_stride_order)r   rm   r`  rs  r  rD  r  r3   r4   r    s    zConvolution.__init__c              
   C   s^   | j dr|jd ||   d| j  dd|   d t| jt	rZ| 
| d S )Nztriton_ops.z&from torch._inductor import triton_opsr  r  r  r  )rD  
startswithheaderr  r  r   r  r,   rm   r  r  rx  r3   r3   r4   rz    s    $zConvolution.codegenr-   )	r~   weightbiasstride_padding_	dilation_
transposedoutput_padding_groupsc
           !      C   s  t jjj t|dd}
t|dd}|d ur6t|ddn|}tjj|
||||||||		}t|	 }W d    n1 sx0    Y  | 
||}| 
||}t|}t|}t|}t|tsJ t|}t|	tsJ |j}dd | D }|^}}}d}|dks2t|dks2t| r2|s2|	dkr8d}n`|d	krHd
}nP|dksVJ ddlm} || | | | ||||||	| | }d}|d
krd}n| rdnd}tjr"t| dkr"ddlm} ||| | ||||||	| | }||kr"|}d}|dkrvdgtttdt|d  }t|t|k rlt|g| }t|}ntttt|}t |}|r| 
||}t!| | t"|t"|} |d urt#| |||f||||||	f||S t#| ||f|||||||	f||S d S )NTr  c                 S   s    g | ]}t tjj|qS r3   r}  r   r3   r3   r4   r<   I  s   z&Convolution.create.<locals>.<listcomp>r  r  r   r;  Ztritonztriton_ops.convZautotune)
tuned_convFztorch.channels_lastztorch.contiguous_formatr*  )tuned_conv_layoutr   )$r)   rg   r  r   r/   r(   r  convolutionr^   r   r  r  r,   re  rg  r9  r   r?   r   r   Zcodegen.autotunerr  r  r   r  r   Ztune_layoutr  rw  r  r@   r   r   rb  r!   rl   )!r   r~   r  r  r  r  r  r  r  r  x_fakeweight_fake	bias_faker  req_stride_orderr   paddingdilationoutput_paddingoutput_sizeZweight_shaperZ   Zkernel_sizeZconfig_convrD  r  Ztransform_x_layoutZoutput_layout_strr  Zfaster_output_layout_strr  rA  Zoutput_layoutr3   r3   r4   r     s    
*


	



 

zConvolution.createc           	   "   C   sx  dd | j D }| j}t|dk r.|dd  }td|d  fd|d  fd|   fg}td	| j d  d  fd
| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd|  d  fd|  d  fd|  d  fd|  d  fdt|dkrp| j d  d  ndfd| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd| j d  d  fd|  d  fd |  d  fd!|d d  fd"|d d  fd#|d d  fd$|d d  fd%|d d  fd&|d d  fd'|d( d  fd)|d( d  fd*|d+  fg}| j d  tj	tj
tjfv rd,nd-}| j d  d dkr4| j d  d dkr4| j d  d dkr4d.nd/}td0|fd1|fg}td2d3| j d   d3fg}||||fS )4Nc                 S   s   g | ]}|  qS r3   r  rE  r3   r3   r4   r<     r=   z(Convolution.map_args.<locals>.<listcomp>r   r   r~   r   wyZ	stride_xnZ	stride_xcZ	stride_xhr  Z	stride_xwZ	stride_wnZ	stride_wcZ	stride_whZ	stride_wwZ	stride_ynZ	stride_ycZ	stride_yhZ	stride_ywZstride_biasnr6  ZBATCHZIN_CZIN_HZIN_WZKERNEL_NZKERNEL_HZKERNEL_WZOUT_HZOUT_WZstride_hZstride_wZ	padding_hZ	padding_wZ
dilation_hZ
dilation_wZoutput_padding_hr*  Zoutput_padding_wr  r  z
tl.float32ztl.int32TrueFalseACC_TYPECONV1X1_NHWCr   ")r`  rs  r?   r   r  r  r   r   r/   r  r  float32r   )	r   Zin_argsZ
const_argsZ
inout_dictZ	args_dictr  r  Z
const_dict
other_dictr3   r3   r4   map_args  s    
.zConvolution.map_argsc                 C   s(   |   \}}}}|| | |tdfS r   )r   r   r   )r   r   r  hr  r3   r3   r4   get_template_tiling  s
    
zConvolution.get_template_tiling)r3   Nr;  )r   r   r   rD  r  rz  r   r   rg  re  r   r  r  r  r3   r3   r  r4   rl     s(      	 Zrl   r-   )	r~   r  r  r  r  r  r  r  r  c
                 C   s  dd }
dd }t |}t |}t |}t|ts6J |	rBt |	nd}tjj t|dd}t|dd}|r|||}| }|
|||||||}n>|durt|ddn|}tj	j
|||||||||	}| }d	gtttd
t|d
  }t|g| }t|}W d   n1 s0    Y  | ||}| jdkrT| jdksXJ ||g}t| | t|t|}||||g}|r|d
| |dur|| n|d	| ||||fS )au  
    This function is a helper function to prepare inputs, layout and constant args
    for convolution post-op fusion's create function, including deciding the output
    layout (channels first or channels last), realizing inputs and make them etc. The
    function only supports the CPU device since conv post-op fusion kernel is only
    supported on CPU right now.
    c                 S   s   t | t |ksJ dt | }|dks0J dd}d}	g }
|
| |  |
||	 |  td|D ]b}|| d ||d   d }| | d ||d   ||d  d  | ||d   }|
| qfttt|
S )NzExpect input dim == weight dimr  zExpect input dim > 2r   r   )r?   r   r@   rw  r   rg  )r  weight_sizer  r  r   r  r  r  Z	BATCH_DIMZWEIGHT_INPUT_CHANNELS_DIMrp  r  rD  Zinput_size_dr3   r3   r4   _conv_input_size,  s(    
z<_prepare_convolution_fusion_create.<locals>._conv_input_sizec                 S   s   |   }t|}|dks J d|dkrpg }||d |  ||d |  td|D ]}|||  qZn| dd  }|S )Nr  zExpect weight dim > 2r   r   )r   r?   r   r@   Z	transpose)Zprepacked_weightr  Zprepacked_weight_sizer  r  r  r3   r3   r4   _original_deconv_weight_sizeG  s    zH_prepare_convolution_fusion_create.<locals>._original_deconv_weight_size)r   r   Tr  Nr   r   r   )r  r,   rg  r)   rg   r  r   r   r/   r(   r  r  rw  r  r@   r?   r   r  r   r1   rb  r   r!   insertr   )r   r~   r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  rp  r  r  r  r  rq  r`  kernel_layoutrs  r3   r3   r4   "_prepare_convolution_fusion_create  sn    

 ($
r  c                	       sT   e Zd ZdZd fdd	Zdd Zedddee ee ee edd	d
Z	  Z
S )ConvolutionUnary'torch.ops.mkldnn._convolution_pointwiser3   c                    s   t  ||| || _d S rK   r  r  rD  r   rm   r`  rs  rD  r  r3   r4   r    s    zConvolutionUnary.__init__c              
   C   sF   | |   d| j dd|   d t| jtrB| | d S r  r  rx  r3   r3   r4   rz    s
    $zConvolutionUnary.codegenr-   )r~   r  r  r  r  r  r  c              	   C   s@   d}t | |||||||\}}}}|||	|
g }t||||dS )Nr  rm   r`  rs  rD  )r  r  )r   r~   r  r  r  r  r  r  attrscalars	algorithmrD  r`  rs  r  rZ   r3   r3   r4   r     s    zConvolutionUnary.create)r3   r  r   r   r   rD  r  rz  r   r   rg  r   r  r3   r3   r  r4   r    s     
r  c                       sp   e Zd ZdZd fdd	Zdd Zeddddee ee ee ee	e
e e
e	 e
e e
e	 dd	d
Z  ZS )ConvolutionBinary.torch.ops.mkldnn._convolution_pointwise.binaryr3   c                    s   t  ||| || _d S rK   r  r  r  r3   r4   r    s    zConvolutionBinary.__init__c              
   C   sF   | |   d| j dd|   d t| jtrB| | d S r  r  rx  r3   r3   r4   rz    s
    $zConvolutionBinary.codegenr-   r~   r  r  r  r  r  r  r  binary_attrbinary_alpha
unary_attrunary_scalarsunary_algorithmc              	   C   s\   d}t | |||||||\}}}}| ||}|d| ||	|
|||g }t||||dS )Nr  r   r  )r  r  r  r   )r   r~   r  r  r  r  r  r  r  r  r  r  r  r  rD  r`  rs  r  r  r3   r3   r4   r     s0    zConvolutionBinary.create)r3   r  )r   r   r   rD  r  rz  r   r   rg  r   r   rf  r   r  r3   r3   r  r4   r     s(     
r   c                       sx   e Zd ZdZd fdd	Zdd Zdd Zed	d	d	d	ee	 ee	 ee	 e	e
ee ee
 ee ee
 d
ddZ  ZS )ConvolutionBinaryInplace/torch.ops.mkldnn._convolution_pointwise_.binaryr3   c                    s   t  ||| || _d S rK   r  )r   r  r`  rs  rD  r  r3   r4   r  
  s    z!ConvolutionBinaryInplace.__init__c              
   C   s0   | |   d| j dd|   d d S r  r  r  rD  r   r  rx  r3   r3   r4   rz    s    $z ConvolutionBinaryInplace.codegenc                 C   s   t | jtsJ | jj fS rK   r0  r   r3   r3   r4   r1    s    z+ConvolutionBinaryInplace.get_mutation_namesr-   r  c              	   C   sr   d}t | |||||||\}}}}| |}tj|  |d| ||	|
|||g }tt|d |||dS )Nr	  r   )r  r`  rs  rD  )	r  rm  r)   rg   r%  r  r  r  r!  )r   r~   r  r  r  r  r  r  r  r  r  r  r  r  rD  r`  rs  rZ   r3   r3   r4   r     s(    

zConvolutionBinaryInplace.create)r3   r	  )r   r   r   rD  r  rz  r1  r   r   rg  r   r   rf  r   r  r3   r3   r  r4   r    s*     
r  c                       s6   e Zd ZdZd	 fdd	Zdd Zedd Z  ZS )
MKLPackedLineartorch.ops.mkl._mkl_linearr3   c                    s   t  ||| || _d S rK   r  r  r  r3   r4   r  G  s    zMKLPackedLinear.__init__c              
   C   s0   | |   d| j dd|   d d S r  r
  rx  r3   r3   r4   rz  Q  s    $zMKLPackedLinear.codegenc                 C   s   d}|  | |}|  | |}| ^ }}| \}}t||g }	t|	}
|||g}d }||g}tt| | |	|
|||dS )Nr  r  )	r  rm  r   rw  r   r  rb  r   r   )r   r~   Zpacked_wZorig_wZ
batch_sizerD  mrZ   ocr  rq  r`  r  rs  r3   r3   r4   r   V  s$    
zMKLPackedLinear.create)r3   r  )	r   r   r   rD  r  rz  r   r   r  r3   r3   r  r4   r  D  s     
r  c                       s>   e Zd ZdZd fdd	Zdd Zedd Zd	d
 Z  Z	S )LinearUnary"torch.ops.mkldnn._linear_pointwiser3   c                    s   t  ||| || _d S rK   r  r  r  r3   r4   r  q  s    zLinearUnary.__init__c              
   C   s0   | |   d| j dd|   d d S r  r
  rx  r3   r3   r4   rz  {  s    $zLinearUnary.codegenc                 C   s   d}|  | |}|  | |}| ^ }}	| \}
}	||g}|||g}|d urt|  | |}|| n|d| tt| | t	||
g d|||dS )Nr  r   r.  r  )
r  rm  r   r   r  r  r   r   r   rw  )r   r~   r  rI  r  r  r  rD  r  icr  r`  rs  r3   r3   r4   r     s*    
zLinearUnary.createc                 C   s   d S rK   r3   r   r3   r3   r4   rv    s    zLinearUnary.apply_constraint)r3   r  
r   r   r   rD  r  rz  r   r   rv  r  r3   r3   r  r4   r  n  s     

r  c                       s>   e Zd ZdZd fdd	Zdd Zedd Zd	d
 Z  Z	S )LinearBinary)torch.ops.mkldnn._linear_pointwise.binaryr3   c                    s   t  ||| || _d S rK   r  r  r  r3   r4   r    s    zLinearBinary.__init__c              
   C   s0   | |   d| j dd|   d d S r  r
  rx  r3   r3   r4   rz    s    $zLinearBinary.codegenc                 C   s   d}|  | |}|  | |}|  | |}| ^ }}| \}	}|||g}
|g}|d ur|  | |}|
| n|d| tt| | t	||	g d|
||dS )Nr  r   r.  r  )
r  rm  r   r   r  r  r   r   r   rw  )r   r~   r  r  rI  r  rD  r  r  r  r`  rs  r3   r3   r4   r     s,    
zLinearBinary.createc                 C   s   d S rK   r3   r   r3   r3   r4   rv    s    zLinearBinary.apply_constraint)r3   r  r  r3   r3   r  r4   r    s     

r  c                
       sZ   e Zd ZdZd fdd	Zdd Zedddee ee ee ee edd	d
Z	  Z
S )ConvolutionTransposeUnary1torch.ops.mkldnn._convolution_transpose_pointwiser3   c                    s   t  ||| || _d S rK   r  r  r  r3   r4   r    s    z"ConvolutionTransposeUnary.__init__c              
   C   s0   | |   d| j dd|   d d S r  r
  rx  r3   r3   r4   rz    s    $z!ConvolutionTransposeUnary.codegenr-   )r~   r  r  r  r  r  r  groups_c                 C   sH   d}d}t | |||||||||
\}}}}||	|
|g }t||||dS )Nr  Tr  )r  r  )r   r~   r  r  r  r  r  r  r  r  r  r  rD  r  r`  rs  r  rZ   r3   r3   r4   r     s*    z ConvolutionTransposeUnary.create)r3   r  r  r3   r3   r  r4   r    s     
r  c                   @   s.   e Zd ZU dZeed< dd Zdd ZeZdS )
MutableBoxzC
    TensorBox / StorageBox allow in-place mutation of Tensors
    r0  c                 C   s8   t | j|}t|r|S tt| jj d| dd S )Nr  z not callable)r   r0  callableAttributeErrorr1   r   )r   r`   rZ  r3   r3   r4   __getattr__  s    zMutableBox.__getattr__c                 C   sn   t | jtr8t| j dt| jj d}d}| jj}nt| j d}| j}d}|tt||g}d|S )Nr  z))r  rV  )r,   r0  r  r1   r   r   r   r   )r   Zline0Zendlr  r   r3   r3   r4   r     s    

zMutableBox.__str__N)	r   r   r   r   r   r   r  r   r   r3   r3   r3   r4   r    s
   
r  c                   @   s   e Zd Zedd ZdS )r-   c                 C   s   t t| S rK   )r-   r  )r0  r3   r3   r4   r   .  s    zTensorBox.createN)r   r   r   r   r   r3   r3   r3   r4   r-   -  s   c                   @   s@   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zedd Z	dS )r  c                 C   s&   t | jttfr"| j tjjv S dS r   )r,   r0  r2  r  r  r)   rg   Zgraph_inputsr   r3   r3   r4   is_input_buffer4  s    zStorageBox.is_input_bufferc                 C   s   t | jtttttfr | j S t | jtt	fs>J t
| jtd t| j | j | j d| jd| _tj| j| j_| j| j_| jjS )Nr.  r/  )r,   r0  r3  r^  r2  r  r]  r  r   r
  r1   r   r   r   r   r)   rg   rb  r`   r   r   r3   r3   r4   r  9  s.    

	
zStorageBox.realizec                 C   s(   t | jttfr$|  dkr$|   dS )zL
        Called on buffers we expect to be forced to realize later.
        r   N)r,   r0  r   r
  	num_readsr  r   r3   r3   r4   r  S  s    zStorageBox.realize_hintc                 C   s,   t | jto*|  tjkp*t|  tjkS rK   )	r,   r0  r   r  r   Zrealize_acc_reads_thresholdr?   r   realize_bytes_thresholdr   r3   r3   r4   r  Z  s    z!StorageBox.has_exceeded_max_readsc                 C   sn   t ttf ddd}|dkrjt| jttfrj|  tjksbt| 	 tj
ksbt| jrj|| jrj|   dS )zj
        A heuristic to decide if we should realize a tensor
        that is used multiple times.
        )loopsc                    s$   dg}|    t fdd|D S )zW
            The heuristic for realizing reused result of heavy ops on cpu
            expc                    s   g | ]}|d   v qS )r  r3   )r8   opZfn_strr3   r4   r<   l  r=   zHStorageBox.mark_reuse.<locals>.should_realize_on_cpu.<locals>.<listcomp>)r   r   )r  Z	heavy_opsr3   r"  r4   should_realize_on_cpuf  s    z4StorageBox.mark_reuse.<locals>.should_realize_on_cpur   N)r   r   r
  r,   r0  r  r   Zrealize_reads_thresholdr?   r   r  r   r  )r   r  r#  r3   r3   r4   r  `  s    		zStorageBox.mark_reusec                 C   sz   | j }t|tttfrdS t|tr.| }nBt|ttfsHJ t	|td t
| | | d|d }t|jS )Nr   r.  r/  )r0  r,   r^  r2  r  r3  rr   r   r
  r1   r   r   r   r   r?   rs   )r   r0  r8  r3   r3   r4   r  y  s     


	zStorageBox.num_readsN)
r   r   r   r  r  r  r  r  r    r  r3   r3   r3   r4   r  3  s   r  c                       s<   e Zd Zdd Zejjed fddZ fddZ	  Z
S )InterpreterShimc                 C   s6   | | _ || _|| _d| _i | _|j| _d| _d| _dS )z
        We don't call super() here to avoid constructing a
        GraphModule which is very expensive (it does codegen).
        Fr$  N)	modulerg   
submodulesZgarbage_collect_valuesenvr  Z
fetch_attrr`   current_noder   rg   r&  r3   r3   r4   r    s    zInterpreterShim.__init__)r   r  c                    s   || _ t |S rK   )r(  r  run_node)r   r   r  r3   r4   r*    s    zInterpreterShim.run_nodec                    s@   t | " t j|i |W  d    S 1 s20    Y  d S rK   )r)   Zset_interpreter_handlerr  run)r   r   r   r  r3   r4   r+    s    zInterpreterShim.run)r   r   r   r  r/   r0   r   r   r*  r+  r  r3   r3   r  r4   r$    s   r$  c                       s`   e Zd ZdZ fddZdd ZejdddZd	d
 Z	dd Z
dd Zdd Zdd Z  ZS )rP  z
    Captures the body of a Loops subclass into an FX graph.  Persists any
    indexing simplifications and makes it easier to analyze loop bodies.
    c                    sj   t    || _i | _i | _g | _g | _i | _i | _g | _	d| j
i| _i | _g | _t| ||| _d | _d S )N	get_index)r  r  rv   rQ  indexing_exprs_namers   writesrS  rT  r  r,  r&  	subblocksindirect_varsLoopBodyBlock
root_blockindexing)r   rZ  r   rv   r  r3   r4   r    s    
zLoopBody.__init__c                 C   s`   dt | j g}|dd | j D  |dd td| jfg| j D  d	|S )Nzvar_ranges = c                 S   s   g | ]\}}| d | qS )r  r3   )r8   r`   rh  r3   r3   r4   r<     r=   z&LoopBody.debug_str.<locals>.<listcomp>c                 S   s   g | ]\}}| |qS r3   )	debug_str)r8   r`   blockr3   r3   r4   r<     s   r  rV  )
rD   rv   r  rQ  rU  rX  chainr2  r/  r   r   r3   r3   r4   r4    s    zLoopBody.debug_str)r@  c                 C   sd   t | || |d ur,|t | | d|< || jvrZdt| j }|| j|< || j|< | j| S )NZ
_name2exprr:   )r   r   r-  r?   rQ  )r   r@  categorybuf_namer`   r3   r3   r4   add_index_expr  s    


zLoopBody.add_index_exprc                 C   s<   |d   r|| jvr|}n| t| j }|| j|< |S )zaNot actually for nn.Modules, but subblocks in generated code are mapped to FX call_module opcodesr  )	isnumericr&  r?   )r   r5  r+   r`   r3   r3   r4   add_submodule  s
    
zLoopBody.add_submodulec                 C   s(   dt | j }t|}| j| |S )NZindirect)r?   r0  r'   r   )r   r`   rb   r3   r3   r4   add_indirect  s    zLoopBody.add_indirectc                    s4   t t  krdS  fdd| j D | _dS )z,Swap in a variable used in indirect indexingNc                    s    i | ]\}}|t | iqS r3   r  r  newr   r3   r4   rS     r=   z-LoopBody.replace_indirect.<locals>.<dictcomp>)r   r3  rU  )r   r   r>  r3   r=  r4   replace_indirect  s    zLoopBody.replace_indirectc                 C   s
   | j | S rK   )r3  r   r3   r3   r4   r,    s    zLoopBody.get_indexc                    s   t tj| }t|tjks.J |jftfdd|D sHJ ttj |  fddj	
 D _ }d _|S )Nc                 3   s   | ]}| j vV  qd S rK   )rv   r<  r   r3   r4   r     r=   z$LoopBody.__call__.<locals>.<genexpr>c                    s   i | ]\}}|t | qS r3   r  )r8   r`   r@  r  r3   r4   rS     s   z%LoopBody.__call__.<locals>.<dictcomp>)rw  rX  r6  r?   rv   r5  rD   rE   ri   rQ  rU  r3  r2  )r   r9  r:   r   r3   )r  r   r4   __call__  s     
zLoopBody.__call__)r   r   r   r   r  r4  r   r   r9  r;  r<  r?  r,  r@  r  r3   r3   r  r4   rP    s   
	rP  c                   @   s8   e Zd ZdZeeee dddZdd Z	ddd	Z
d
S )r1  a  
    Captures the body of a Loops subclass into an FX graph.
    In normal cases there will be a 1:1 mapping between LoopBody and
    LoopBodyBlock, hower in the case of ops.masked() the masked out
    operations will manifest as an extra LoopBodyBlock.
    )r  rZ  r   c                    s   |_ dfdd	 G  fdddtj}tj tjjjd_	dddi }d	d
l
m} t|||j j& 	dd|| fi  W d    n1 s0    Y  j_d S )Nc              	      s    dd j| ||fi S )Ncall_moduler,  )create_proxyr  r9  )r@  r7  r8  r   tracerr3   r4   	add_index  s    z)LoopBodyBlock.__init__.<locals>.add_indexc                       sx   e Zd Zd_eejd fddZd fdd	Z fdd	Z	 fd
dZ
eedfddZefddZdS )z/LoopBodyBlock.__init__.<locals>.CaptureIndexingCaptureIndexing)r`   r:   c                    s    |d|}| j ||S )Nrs   )_innerr  )r   r`   r:   rE  r3   r4   r    s    z4LoopBodyBlock.__init__.<locals>.CaptureIndexing.loadNc                    s    |d|}| j ||||S Nr.  )rG  r   )r   r`   r:   r  r  rH  r3   r4   r     s    z5LoopBodyBlock.__init__.<locals>.CaptureIndexing.storec                    s"    |d|}| j ||||||S rI  )rG  r  )r   r`   r   r  r  r:   r  rH  r3   r4   r    s    z9LoopBodyBlock.__init__.<locals>.CaptureIndexing.reductionc                    s8   t |ttjfr tt||S  |d}| j||S )Nr  )r,   rg  r   r   r(   rm  rG  r]  )r   r:   r   rH  r3   r4   r]    s    
z:LoopBodyBlock.__init__.<locals>.CaptureIndexing.index_expr)masked_bodyc                    sH    fdd}j |d}tj |g   j j|< d|| |fi S )zb
                Recursively capture the masked out body in another LoopBodyBlock
                c                    s   t j|  |S rK   )r)   r(   r  )r  r  Zsubblockr3   r4   shim)  s    zDLoopBodyBlock.__init__.<locals>.CaptureIndexing.masked.<locals>.shimZmasked_subblockrA  )r  r;  r1  r/  rB  )Z
mask_proxyrJ  Zother_proxyrL  r`   rC  rK  r4   r  #  s    z6LoopBodyBlock.__init__.<locals>.CaptureIndexing.maskedc                    s>    fdd}j   dj |d  | fi   S )z
                Flow data from tensors into indexing formulas.
                Introduce a call_module to update the indexing.
                c                    s    j tj|  d S rK   )r  r?  r)   r(   indirect_indexing)Znew_var)r   rb   r3   r4   set_indirect:  s    zWLoopBodyBlock.__init__.<locals>.CaptureIndexing.indirect_indexing.<locals>.set_indirectrA  set_)r  r<  rB  r;  )Zindex_proxyrN  rC  ra   r4   rM  3  s    
zALoopBodyBlock.__init__.<locals>.CaptureIndexing.indirect_indexing)N)r   r   r   r`   r   r   r   r  r   r  r]  r   r   r  rM  r3   rE  r   rD  r3   r4   rF    s   rF  )Z
tracer_clsplaceholderr(   r3   r   )SimplifyIndexingr  )N)r  r)   ZWrapperHandlerr/   r0   ZTracerZGraphr  rg   rB  r   rR  r   rv   )r   r  rZ  r   rF  Z	proxy_opsrR  r3   rP  r4   r    s    :
4zLoopBodyBlock.__init__c                 C   s"   | j }| jj}t||t S rK   )rg   r  r&  r$  r+  r)   Zget_ops_handlerr)  r3   r3   r4   r@  Q  s    zLoopBodyBlock.__call__r5  c              
   C   s8   t j| jj| jj}tdd|	 
dd| dS )Nz;[^\n]*r  zdef forward(zdef r  )r/   r0   ZGraphModuler  r&  rg   coderesubstripr  )r   r`   rS  r3   r3   r4   r4  W  s    zLoopBodyBlock.debug_strN)r5  )r   r   r   r   rP  r   r   r   r  r@  r4  r3   r3   r3   r4   r1    s   Pr1  )T)TFN)FN)r   r  rV  rX  loggingrT  textwrapcollectionsr   r   enumr   r   inspectr   typingr   r   r	   r
   r   r   r   r   r   Zunittest.mockr   r   r   r   Ztorch.fxr/   Ztorch.utils._pytreeutilsZ_pytreer~  Ztorch._prims_commonr   r   r   r   Z%torch.fx.experimental.symbolic_shapesr   r  r   r   Zcodegen.commonr   Zcuda_propertiesr   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   Zvirtualizedr(   r)   	getLoggerr   rY  r   r  r6   rH   rJ   rO   rX   r^   rt   r   r   ZFunctionr   r   r   r   r   r   r  r   r   r   r   r  r	  r
  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rb  r   rq   r!  r  r2  r   r.   r5  r8  r3  r]  r^  rn  ro  rl  r  r  r  r  r  r  r  r*  r  rl   rg  re  r  r  r   r  r  r  r  r  r  r-   r  r0   ZInterpreterr$  rP  r1  r3   r3   r3   r4   <module>   s.  ,,
&
#
3	E    1
3;+; !<1g#[8Y	 ^4g  .$ k     /?=*238![S