a d7@sFddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z ddlZddlmZddlmZddlmZddlmZddlmZdd lmZmZmZdd lmZdd lmZm Z m!Z!ddl"m#m$Z%e&e'Z(d d Z)eddZ*eej+e dddZ,d0ddZ-ddZ.eej+e dddZ/Gdddej0Z1eej+e dddZ2eddZ3d1d d!Z4ej5j6Z6e6j7e6j8e6j9e6j:e6j;e6je6j?e6j@e6jAe6jBe6jCe6jDjEe6jDjFe6jGe6jHe6jIe6jJe6jKe6jLe6jMhZNeeNZNed"d#ZOd2e e ejPfe e eQd$d%d&ZRd'd(ZSdaTd)d*ZUd+d,ZVd3d.d/ZWdS)4N)contextmanager)partial)CallableOptionalTupleUnion)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partitioncCs6|jjD] }|jtjjjkrtjjj|_q||SN) graphnodestargettorchopsaten_to_copyto recompile)fx_gnoderc/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_functorch/compilers.py _canonicalize s  r!c cs6tjd}zdVWtj|ntj|0dS)NF)r_CZ_jit_set_autocast_mode)Zold_jit_autocast_flagrrr _disable_jit_autocast(s r#)rreturncCs0tt||jjD]F}|jtjjjkrt |j dkrt |j dkrd|j vrtjjj |_q|jjD]<}i}|j D]"\}}t|tjr|j}|||<q|||_ qj|j|tj|}tj|jtj|}tj|}tdd|Ds ||Wdn1s"0Y|S)a Compiles the :attr:`fx_g` with Torchscript compiler. .. warning:: This API is experimental and likely to change. Args: fx_g(fx.GraphModule): The input Fx graph module to be compiled. Returns: Torch scripted model. r dtypecss|]}t|tjjVqdSr) isinstancerZ _subclassesZ FakeTensor).0trrr ^zts_compile..N)r#rrrrrrrrlenargskwargsritemsr&devicetypeZlintrjitscriptr"Z_jit_pass_remove_mutationfreezeevalZoptimize_for_inferenceany)rinpsrZ new_kwargskvfrrr ts_compile1s8          (r:TcCst|jt|||d|S)N) clear_meta)printcoder)r_namer;rrr _draw_graph_compilecs r@cCsttt|dS)Nr?)rrr@rArrr draw_graph_compileis rBcCs|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. rrr>rrr nopos rDcs(eZdZfddZfddZZS)DebugInterpretercs$t|jg|R|_tj|dSr)r modulesymbol_mappingsuperrun)selfr, __class__rr rI|szDebugInterpreter.runc sddlfddfddfddfdd }t|}d |jvrt|jd \}}t|\}}t|t|ksJt|d t|ttt|||D].\}} t | t j sq||| fd d q|S)NrcsBt|ts|S|jjj}t|jdks:J|t |S)Nr) r&rexpandrexprZxreplacerGr+Z free_symbolsint)nir)rJsympyrr subst_symints  z/DebugInterpreter.run_node..subst_symintcstfdd|DS)Nc3s|]}|VqdSrr)r'rPrSrr r)r*zHDebugInterpreter.run_node..subst_symint_tuple..)tuple)ZnisrTrr subst_symint_tuplesz5DebugInterpreter.run_node..subst_symint_tuplecsT|dkrPt|jD]4}||||kr||dkrdSqdS)Nrr FT)Znumelrangendimstridesize)abidxrTrr check_significant_stridess *z.check_significant_stridesc st|s J|j|jks6J|d|jd|j||kszJ|d|d|d|||}|sJ|d|d|d|dS)Nz:  != z aka )callabler%rZrY)nvrvdescZ same_strides)r^rVrr checks ** z(DebugInterpreter.run_node..checkvalr_csddjS)Nzoutput z where )rGr)irJrr r*z+DebugInterpreter.run_node..) rRrHrun_nodemetapytreeZ tree_flattenr+ziprWr&rTensor) rJnrdrQZn_valsZn_specZr_valsZr_specrarbrK)r^rfrJrSrVrRr rhs    * zDebugInterpreter.run_node)__name__ __module__ __qualname__rIrh __classcell__rrrKr rE{s rEcCs t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )rErIrCrrr debug_nopsrrcCs(t|tj|}tj|}|Sr)rrr1r2r3r4)rr>r9rrr simple_ts_compiles rscCst|t|dS)N)static_argnums)r rs)r9rtrrr nnc_jitsrucCst|j|Sr)r<r=rCrrr print_compiles rv)fnrtcKsLtttt|d}||t|tjjr8t|fi|St |fi|SdS)a Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one ore more arguments. Must return one or more Tensors. static_argnums (Optional[Tuple[Int]]): An option tuple of ints to mark the arguments of the function as static. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser. ) fw_compiler bw_compiler partition_fndecompositionsrtN) r:rdefault_decompositionsupdater&rnnModuler r )rwrtr-configrrr memory_efficient_fusions rcCsH|dtddd|Ddddlm}||t||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = cSsg|]}|j|jfqSr)shaper%)r'rfrrr r*z!debug_compile..a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderr<rrZcudar:)rr6rrrr debug_compiles   rc Csg}t|d}t|}g}|D]}t|dkrD|}|t}nX|\}}}} } | tjtjtj tj tjtj tt hvrtj dd|| | d}ntj|| | d}||q"Wdn1s0Y|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r%r/N)openpickleloadr+randomZrandrrOint32int64boolZuint8floatrandintappend) Zinput_data_pathinputsr9Z inputs_metarir0inputrrYr%r/rrr get_inputs1s.    *rc sbddlm}fddfddfdd}fd d }fd d }||||||td S)aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedcsg}t|dkrBt|dtrB||d7}||d7}|S|D]P}t|tksbt|tkrt|t|fqF|t||j||j |j fqF|S)Nrr ) r+r&rUr0rOrrrrYr%r/)r, input_metaarg)get_input_metarr rbsz(_save_fx_default..get_input_metacs@t|jjdkr6ttjdd|dtddSt|}|j t j j | |}tjd}|std|ddd|dt t|tddd|dtdd|dtddr./z.inputwbz.pt)r+rrlogloggingWARNING graph_indexcopydeepcopyZ set_codegenrfxZCodeGenrospathexistsmakedirsrrdumprsave)Z gm_to_saver, type_namegmrZisExist) current_namedump_example_input folder_namerrr graph_saver_helperqs8 22z,_save_fx_default..graph_saver_helpercs||d|S)NZforwardr)rZfw_argsrrr graph_saver_forwards z-_save_fx_default..graph_saver_forwardcs||dtd7a|S)NZbackwardr )r)rZbw_argsrrr graph_saver_backwards z._save_fx_default..graph_saver_backwardcs||dt||S)NZjoint)r)rZ joint_argsrrr graph_saver_joints z+_save_fx_default..graph_saver_joint)rxryrzr{)Zfunctorch.compilerr|) rrrrZexample_inputsrrrrr)rrrrrr _save_fx_defaultPs  "   rFcCsdatt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrrrr graph_dumper_aots r)T)N)N)F)Xrrrrr contextlibr functoolsrtypingrrrrrrZtorch.fxrZtorch.nnr~Z torch._decompr Z%torch.fx.experimental.symbolic_shapesr Z aot_autogradr r rZ compile_utilsrZ partitionersrrrZtorch.utils._pytreeutilsZ_pytreerj getLoggerrnrr!r#Z GraphModuler:r@rBrDZ InterpreterrErrrsrurrdetachZ gelu_backwardZleaky_relu_backwardZsigmoid_backwardZthreshold_backwardZhardtanh_backwardZhardsigmoid_backwardZhardswish_backwardZ tanh_backwardZ silu_backwardZ elu_backwardZcudnn_batch_normZcudnn_batch_norm_backwardZ masked_fillZScalarrlZeluZ leaky_reluZhardtanhZ hardswishZ hardsigmoidZ conj_physicalZ is_same_sizer|rvrrOrrrrrrrrrr s          1  3     +\