a dsa@sddlmZmZddlmZddlZddlmZddlZddl Z ddl m m Z ddlZddlZddlmZddlmZddlmZddlmZmZdd lmZddlZejZGd d d ZeZd d Z ddZ!ddZ"ej#dddZ$d1ej#dddZ%ej#eej#ej#fdddZ&ddZ'ddZ(ej)e*dd d!Z+d"d#Z,e-dd$d%Z.d2ej#eej#ej#fdd'd(Z/d3ejj#e0e0d+d,d-Z1d4d/d0Z2dS)5) is_sym_node py_sym_types)hint_intN defaultdict) graph_drawer)Tuple) fx_graph_cseget_aten_target)configc@seZdZddZdS)InvalidNodeBasecCsdS)Nz Invalid Node)selfrrf/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_functorch/partitioners.py__repr__szInvalidNodeBase.__repr__N)__name__ __module__ __qualname__rrrrrr sr c sXt}i|D] }||j}|j|_||<q|jD]}||vrHq8q8|jdkr\t|<q8|jdkrt |j |j fd}fdd|D}t |rt|<q8| |fdd|<q8|jdkr| |fd d|<q8|jd kr8q8g}|D]F}t|tjr.|vrtd |d ||q||q|||||S) a Given a graph, extracts out a subgraph that takes the specified nodes as inputs and returns the specified outputs. This includes specifying non-placeholder nodes as inputs. The general strategy is to initialize all inputs with proxies as we encounter them, and trace through the graph, only keeping values which take in valid proxies. Then, all dead code is eliminated. placeholder call_functionrcs&g|]}t|tjrt|tqSr) isinstancefxNoder ).0xenvrr 9z6_extract_graph_with_inputs_outputs..cs|SNrrrrr=rz4_extract_graph_with_inputs_outputs..Zget_attrcs|Sr rr!rrrr"?routputzNode z couldn't be found in env)rZGraphrnamemetanodesop InvalidNodepytree tree_flattenargskwargsanyZ node_copyrr RuntimeErrorappendr#eliminate_dead_codeZlint) Z joint_graphinputsoutputs new_graphnodenew_nodeall_argsZ output_valuesrrrr"_extract_graph_with_inputs_outputssB            r7cCs|jdkod|jvSNrtangentsr'targetr4rrr _is_primalQsr=cCs|jdkod|jvSr8r:r<rrr _is_tangentUsr>) joint_modulecCs<tdd|jjDd}|d|}||d}||fS)NcSsg|]}|jdkr|jqSr#)r'r+rr4rrrrZrz,_extract_fwd_bwd_outputs..r)r)r*graphr&)r?num_fwd_outputsr2 fwd_outputs bwd_outputsrrr_extract_fwd_bwd_outputsYs  rFrcCst||d\}}ttt|jj}ttt|jj}t|j||||}t|j||||} | jD]\} | jdkrf| j sf|D]} | j | j kr~| | qq~|D]} | j | j kr| | qfqqft|j||||}t|j||||} t ||} t || }| |fS)NrCr)rFlistfilterr=rBr&r>r7r'usersr$remover GraphModule)r? saved_valuessaved_sym_nodesrCrDrE primal_inputsZtangent_inputsZ fwd_graphZ bwd_graphr4Z saved_valueZ saved_symZ fwd_moduleZ bwd_modulerrr_extract_fwd_bwd_modules`s(       rP)r?returnc s8ttt|jj}t||d\}}t|j||}dd|jDg}g}|jjD]} | jvr`qPt| rt| | qPd| j vr| j dkr| j } t dd| DsJ| D]} | | qqPfdd | j D} d| j vrt d d| Dr| D]} | | qqP| | qPtt|}tt|}t||||d S) a Partitions the :attr:`joint_module` in a manner that closely resembles the behavior observed in the original ``.forward()`` and ``.backward()`` of the callable, i.e., the resulting forward graph contains those operators that are executed in the original ``.forward()`` callable passed to :func:`aot_function`. The default partitioner collects the operators that are between the forward inputs and the forward outputs. This helps in finding the tensors which have to be stashed for the backward pass. These stashed tensors become the output of the generated forward graph. The remaining operators are then placed in the backward graph. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. Returns: Returns the generated forward and backward Fx graph modules. rGcSsh|]}|jdkr|jqSr@r'r$rArrr rz$default_partition.. tensor_metarcss|]}|jtjkVqdSr )r;operatorgetitemruserrrr rz$default_partition..csg|]}|jvr|qSr)r$rnZforward_node_namesrrrrz%default_partition..css|]}t|VqdSr rrZrrrrYrrNrC)rHrIr=rBr&rFr7r$rr/r%r'rJallsetrP) r? _joint_inputsrCrOrDrEforward_only_graphrMrNr4rJrXZbackward_usagesrr\rdefault_partitions6       rccCsd}|D] }||9}q|SNr r)rsirrr_prods rgcCsjtjdtjdtjdtjdtjdtjdtjdtjdtj dtj dtj dtj di }||vr^t d||||S)Nr zDon't know the size of dtype )torchfloatfloat16Zbfloat16float32float64intZint8int16int32int64Zuint8boolNotImplementedError)numeldtypesizesrrr_tensor_nbytess  ry)r4rQcCsd|jvrt|jd}t|tr"dSt|ttfrBtdd|DSt|tjrbtt | |j St dt |d|jvr|jd}ttt|j}|j }ndSt||S)Nvalr css.|]&}t|tjrtt||jVqdSr )rrkTensorryrrvrwrZrrrrYrz_size_of..zUnknown metadata type rTr)r%rrrHtuplesumrkr{ryrrvrwr.typergmapZ to_size_hintshape)r4rzmetadatarvrwrrr_size_ofs      rcCs\ddlm}|t}|jD]"}|jdkr||jjd7<qtt| dddddS) Nrrrr cSs|dSrdrr!rrrr"rz_count_ops..Tkeyreverse) collectionsrrpr&r'r;rprintsorteditems)rBrZcntr4rrr _count_opss    rcCslg}ttjjD]V}ttjj|}t|tjjs2q|D]*}t||}tj j |j vr:| |qq:q|Sr ) dirrkopsatengetattrrZ_opsZOpOverloadPacketZ overloadsTagZ pointwisetagsr/)r attr_nameZopoverloadpacketoverloadZ op_overloadrrr pointwise_opss   rnvfuserc,Ksz ddl}Wn.ty:}ztd|WYd}~n d}~00|j||j}tjrht|}||_|j} i|jj D]} | | j <qz fdd} | |\} } }dd}t | dkrt || dSt |jj D]>} | vrd| _n*td | _| jD]}t| j|jd | _qqtjjtjj}jjjjjjjjjjjj j!j"j#j$j%j&j'j(j)j*j+j,j-j.j/j0j1j2j3j4j5j6j7j8j9j:j;j<j=j>j?j@jAjBjCjDjEjFjGjHjIjJjKjLjMjNjOjPjQjRjSjTjUjVjWjXjYjZt[j\j]j^j_j`gK}j]j^jagd kr(||j|jbjcj`jd|je|jTjejf|jgjhjijjjkjljmjnjojpjqjrjsjtjujvjwjxjyjzj{j|j}g 7}jkjmjij~|jgjljzg7|jg7}|7}|t7} durLt nt| jjjg}jjjjjjjjjjjjjg }|| t|Btrd d |jj D}|dd D}td|td fdd fdd}fddfddtdfdd }| | j D]>} | jdkrbqL| | vr j| j ddtjdqL| jd krd!| jvr jd"| j dtjd|| r| vr jd"| j dtjdd#| jvod$| jvpd#| jvot| jd#tj }t| r,d }n|r:tj}n|| } j| j d| j d%|d| jD]$} j| j d%|j dtjdqdqL| d"d\}}|\} t} fd&d'|DD]$\}| fd(d'|Dqt} |D]>\}!}"|!dd)|"dd*ksJ|!dd)}#| |#qd+d,t|jj Dtfd-d'| Dfd.d/d0}$ttd1d/|$}%ttd2d/|$}$t||$|% d3\}&}'trztd4tTd5d6|$Dd d7d |&jj D}(d8d |'jj D})|(|)@}*tt}+|&jj D]8} | j |*vr4t| jd9r4|+t| jjd 7<qtd:t |*d;t |(d;t |)tdd?|&|'fS)@ay Partitions the joint graph such that the backward recomputes the forward. Recomputing helps in trading off memory bandwidth with computation. To create the fwd and bwd graph, we copy the joint graph, manually set the outputs to just original forward or backward outputs. And then we run the resulting graphs through dead code elimintation. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. _joint_inputs: The inputs to the joint graph. This is unused. compiler: This option determines the default set of recomputable ops. Currently, there are two options: ``nvfuser`` and ``inductor``. recomputable_ops: This is an optional set of recomputable ops. If this is not None, then this set of ops will be used instead of the default set of ops. num_fwd_outputs: The number of outputs from the forward graph. Returns: Returns the generated forward and backward Fx graph modules. rNzANeed networkx installed to perform smart recomputation heuristicscst|jjD]@}|jdkr0d|jvr0||vr|jD]}|q>qttt |jj}t |d\}}t |j||}fdd|jDfdd|jjD}||fS)Nrr9rGcs h|]}|jdkr|jqSr@rRrA name_to_noderrrSQs zNmin_cut_rematerialization_partition..classify_nodes..cs h|]}|vr|vr|qSrrrArequired_bw_nodesrequired_fw_nodesrrrSSs) r`rBr&r'r;addrJrHrIr=rFr7)r?r4rXrOrD_rbunclaimed_nodes)rrCrrclassify_nodesEs   z;min_cut_rematerialization_partition..classify_nodescSs6t|tjsdSd|jvp4d|jvo4t|jdtjS)NFrTrz)rrrr%rkr{r!rrris_tensor_nodeYs z;min_cut_rematerialization_partition..is_tensor_noderGgeAr inductorcSs.h|]&}|jdkrt|jdrt|jjqS)r_overloadpacket)r'hasattrr;strrrArrrrSsz6min_cut_rematerialization_partition..cSsh|] }t|qSr)rrrfrrrrSrz#Ops banned from rematerialization: Fcsb|h}t|dkr^|}|jD]:}|vr<||s<dS|vr t|vr ||q qdS)NrTF)lenpoprJr r)r4Z cur_nodescurrX) is_fusiblerview_opsrris_materialized_backwardss  zFmin_cut_rematerialization_partition..is_materialized_backwardscsr|jdkot|vS|jdkr(dSt|vr8dS|jtjkrHdS|jjjjjfvrbdS|rndSdkr|jt j krdSt dd|j D}t |}|d|kSdS)NrFTrcss"|]}t|tjrt|VqdSr )rrrrrrrrrYrzQmin_cut_rematerialization_partition..ban_recomputation..rh)r'r r;rUrVlift_fresh_copydefaultZ lift_fresh dist_from_bwr Zmax_dist_from_bwr}r+r)r4Zinput_tensors_sizeZ output_size)AGGRESSIVE_RECOMPUTATIONrcompilerrrecomputable_opsunrecomputable_opsrrban_recomputations"   z>min_cut_rematerialization_partition..ban_recomputationcst|vot|vSr )r )ab) fusible_opsrrrsz7min_cut_rematerialization_partition..is_fusiblecs*jdkrdStfddjD S)NrTc3s|]}|VqdSr rrW)rr4rrrYrzOmin_cut_rematerialization_partition..is_materialized..)r'r_rJr<)rr<ris_materializeds z.is_materialized)rQcs>t|}t|dtt|jdd}|r2|S|dSdS)Ng?dr ri)rrpmaxminr)r4Zmem_sz)rrrget_node_weights z.get_node_weightr#Z_inZsink)capacityrZprimalssourcerzrTZ_outc3s|]}||fVqdSr rrZ)nx_graphrrrYrz6min_cut_rematerialization_partition..c3s|]}|vr|fVqdSr r)rv) non_reachableurrrYrcSsi|]\}}||qSrr)ridxr4rrr rz7min_cut_rematerialization_partition..c3s|]}|VqdSr rrArrrrYrcs|Sr rr!)node_idxrrr"rz5min_cut_rematerialization_partition..)rcSst|Sr r]r[rrrr"rcSs t| Sr r]rrrrr"rr^z Theoretical Activations Stored: cSsg|] }t|qSr)rrrrrr rz7min_cut_rematerialization_partition..cSsh|]}|jdkr|jqSrrRrArrrrS rcSsh|]}|jdkr|jqSrrRrArrrrSrrz# remat/fw/bw: /zCount of Ops Rematerialized: cSs|dSrdrr!rrrr"rTr)Znetworkx ImportErrorr.rBr0Z recompiler Zcser r&r$rrcreversedrrprJrrkrrprimsrsubdivatan2mulrpow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltabsZ bitwise_notceilfloorfracnegZreluroundZsilutruncloglog10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtZrsqrtZ reciprocalZsigmoidZsoftplus thresholdZthreshold_backwardclampwhereZlerpZaddcmulZgeluZ gelu_backwardr}meanZ_grad_sum_to_sizeZ sum_to_sizeZamaxtoZtype_asrUrVZsqueezeZ unsqueezeZrsubZ_to_copyaliasZconvert_element_typecloneZ full_likevarZstdZbroadcast_in_dimselectZpermuteZ _unsafe_viewviewexpandsliceZreshapeZbroadcast_tensorsZ scalar_tensorZonesZ new_zerosrZarangeZtriuZvar_meanisinfr-fullZ as_stridedzerosZargmaxmaximumtindexrr`Znative_dropoutZ rand_likeZ randn_likemmZ convolutionZconvolution_backwardZbmmZaddmmZupsample_bilinear2dZ_softmaxZ_softmax_backward_dataZnative_layer_normZnative_layer_norm_backwardZnative_batch_normZnative_batch_norm_backwardZ_native_batch_norm_legitAOT_PARTITIONER_DEBUGrZDiGraphr'Zadd_edgemathinfr;r%rr{rZ minimum_cutupdate enumeraterrHrIrPrrrrr),r?rarrrCZnxeZfx_gZ cse_graphZ full_bw_graphr4rZorig_fw_outputsrrrrXrZdefault_recomputable_opsZ random_opsZcompute_intensive_opsZjoint_module_opsZ ops_ignoredrrZis_non_tensor_nodeZweightZ cut_value partitionZ reachableZcutsetZnbrsZ cut_nodesZnode_inZnode_outZ node_namerMrNZ fw_moduleZ bw_moduleZfw_module_nodesZbw_module_nodesZ remat_nodescountsr)rrrrrrrrrrrCrrrrrrr#min_cut_rematerialization_partitions       1 $  8   !        &  "  &rfx_graphT)tracedfnamefignamec Cs|r0t|j}t||}|jjD] }i|_q$tj |\}}|sHd}t d||t ||}| } t| d|d||dS)Nz.svgzWriting FX graph to file: Zwrite_.)copydeepcopyrBrrLr&r%ospathsplitextrrZ FxGraphDrawerZget_main_dot_graphrlstrip) rrrZ clear_metar3r4baseextgrrrr draw_graphs    r"full_graph.pngcCst||t||Sr )r"rc)rBZ joint_inputs file_namerrrdraw_joint_graph)s r%)r)rN)rT)r#)3Z"torch.fx.experimental.proxy_tensorrrZ%torch.fx.experimental.symbolic_shapesrrkZtorch.fxrrUr Ztorch.utils._pytreeutilsZ_pytreer)rrrrZtorch.fx.passesrtypingrZ compile_utilsr r r functoolsZdebug_partitionerr r r(r7r=r>rLrFrPrcrgryrrprr lru_cacherrrr"r%rrrrsP      4! F