a dȕ@snddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlZddlmZddlmZddlmZddlmZmZdd lmZmZeeZe d Ze Z!gZ"d Z#e!rgd Z"d $dde"DZ#GdddZ%ddZ&GdddZ'e(dddZ)ddZ*dZ+ddZ,dZ-e-ddfe-dd fd!Z.d"d#Z/d$d%Z0dTe1d&d'd(Z2dUd)dZ3dVd*d Z4d+d,Z5d-d.Z6e1d&d/d0Z7Gd1d2d2e8Z9e1d&d3d4Z:dWd6d7Z;dXd8d9Zd>d?Z?dYd@dAZ@dZdBdCZAd[dDdEZBd\dFdGZCejDeCdHdIZEd5ZFdJdKZGdLdMZHe1d&dNdOZIedPdQZJedRdSZKdS)]N)Counter) import_module) TemporaryFile)is_float_dtype)config)lookup_backendregister_debug_backend) clone_inputs get_debug_dirztorch._inductor.config)z6//caffe2/fb/custom_ops/sparsenn:sparsenn-all_operatorsz1//caffe2/torch/fb/sparsenn:sparsenn_operators_gpuz-//caffe2/torch/fb/sparsenn:sparsenn_operatorsz///deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpuz+//deeplearning/fbgemm/fbgemm_gpu:sparse_ops cCsg|]}d|dqS)ztorch.ops.load_library("z").0xrrb/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/_dynamo/debug_utils.py 'rc@s&eZdZddZddZd ddZdS) BuckTargetWritercCstj|\|_|_|jdd|_|jddd|j|_|j|jdd|_|jdd|_|j}||dddd}d|d |j|_dS) N.pyr /.zfbcode.zfbcode/z//:) ospathsplitsubdirpy_filereplacetargetfind cmd_line_path)selffilenametmprrr__init__+szBuckTargetWriter.__init__c Cs>dddtD}td|jd|jd|d|jd S) Nr cSsg|]}d|dqS)z "z",rrrrrr:rz*BuckTargetWriter.build..za load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") python_binary( name="z", srcs = ["z"], compile = False, deps = [ "//caffe2:torch", "//caffe2/functorch:functorch", "//triton:triton", ], cpp_deps = [ z ], main_module = "z", ) )join extra_depstextwrapdedentr!rr)r$Zextra_cpp_depsrrrbuild9s zBuckTargetWriter.buildTcCsttj|jd}t|d}||Wdn1s>0Yddd|jg}|rpt dd||S)NZTARGETSwZbuck2runz@mode/dev-nosanzDFound an example that reproduces the error. Run this cmd to repro -  ) rrr(ropenwriter,r#logwarning)r$ print_msgZ target_filefdcmdrrrr1Ps ,zBuckTargetWriter.writeN)T)__name__ __module__ __qualname__r'r,r1rrrrr*srcCsLtjtd}|dur.tdt}tj|sHtj |dd|S)Nminifierz /minifier_Texist_ok) rrr(r tempfile gettempdirgetpassgetuserexistsmakedirs)rrrr minifier_dir]s  rCc@seZdZejjejjejjejjejj ejj ejj ejj ejj ejjejjejjejjejjejjejjejjejjejjejjejjgZeddZeddZdS)NNModuleToStringcCsPt}|D] \}}t|tjvr||qt|dkrLtd|dS)Nrz+We have not tested reprs of some modules - T) setnamed_childrentyperD safe_reprsaddlenr2r3)gmZ cant_convert_modulerrrcan_convert_to_strings  z&NNModuleToString.can_convert_to_stringc Csddlm}d}td}|D]V\}}|}t|d}|durZ|jrZ|d}||dd|d|d 7}q"|j D]\}} | durqt | rd t | jd | jd } nd t | jd | jd } | jr| d} ||dd|d| d7}q|j D]b\} } | dur&qdt | jd | jd} | jrR| d} ||dd| d| d 7}q|||jdd 7}|S)Nr) _addindentz z from torch.nn import * class Repro(torch.nn.Module): def __init__(self): super().__init__() z.cuda()zself.z = r z torch.randn(z, dtype=)ztorch.randint(1, size=zself.register_buffer('z', z) ztorch.nn.Parameter(torch.randn(z)))Ztorch.nn.modules.modulerOr*r+rF__repr__next parametersis_cuda_buffersitemstorchis_floating_pointlistshapedtype _parameterscode) rKrOtab model_str module_namerMZ module_strZ example_paramZ buffer_namebufferZ tensor_str param_nameparamrrrconverts<          " zNNModuleToString.convertN)r7r8r9rYnnZLinearZConv1dZConv2dZConv3dZ BatchNorm1dZ BatchNorm2dZ BatchNorm3dZ LayerNormZDropoutZSoftmaxZReLUZGELUZIdentityZ MaxPool2dZ EmbeddingZTanhZConvTranspose1dZGLUZLSTMZFlattenZAdaptiveAvgPool2drH staticmethodrNrfrrrrrDfs4 rDcCstjsdSd}zJtjddgtjd}|jd}d dd |D}||d7}Wnt yv|d 7}Yn0t d d t tj D}|d 7}|D]\}}|d|d|d7}q|d7}|S)Nz:# torch.cuda.is_available()==False, no GPU info collected z# CUDA Info: Znvccz --version)stdoutr r cSs g|]}|dvrd|dqS))r #  r)rsrrrrrz-_cuda_system_info_comment..z# nvcc not found css|]}tj|VqdSN)rYcudaZget_device_name)rirrr sz,_cuda_system_info_comment..z# GPU Hardware Info: rjz : rk)rYrnZ is_available subprocessr.PIPEridecoderr(FileNotFoundErrorrrange device_countrX)raZcuda_version_outZcuda_version_linesZ gpu_namesnamecountrrr_cuda_system_info_comments(    ryc CsRddl}ddl}tdt|jjdt|jjdt|j jdS)Nrzzimport torch._dynamo.config import torch._inductor.config import torch._functorch.config torch._dynamo.config.load_config(z%) torch._inductor.config.load_config(z&) torch._functorch.config.load_config(z ) ) Ztorch._functorch.configZtorch._inductor.configr*r+reprZ_dynamorZ save_configZ _inductorZ _functorch)rYrrrgenerate_config_stringsr{z*# REPLACEABLE COMMENT FOR TESTING PURPOSESc Cstdtdtdtd}|dtjjd7}ttjdrT|dtjj d7}ttjdrt|d tjj d 7}|t 7}|t |7}|d d d |Dd7}|d7}d}tjrd}|dt|d7}|S)Nz import torch from torch import tensor, device import torch.fx as fx from torch._dynamo.testing import rand_strided from math import inf from torch.fx.experimental.proxy_tensor import make_fx  r z z# torch version: rnz# torch cuda version: git_versionz# torch git version: z zargs = cSs,g|]$}t|jt||j|jjfqSr)tupler\strider]devicerGrarrrr rz2generate_compiler_repro_string..zEargs = [rand_strided(sh, st, dt, dev) for (sh, st, dt, dev) in args] realZsymbolicz$mod = make_fx(Repro(), tracing_mode=z )(*args) )r*r+r{TEST_REPLACEABLE_COMMENT extra_importsrYversion __version__hasattrrnr}ryrDrfrZdynamic_shapesrz)rKargsraZ tracing_moderrrgenerate_compiler_repro_strings2     rzo from torch._inductor.compile_fx import compile_fx_inner from torch._dynamo.debug_utils import same_two_models compile_fx_innerinductor_failsinductor_accuracy_fails)inductorZinductor_accuracycCs tjtd}tj|s*tj|ddtj|t|jjd}t dt|jjd|t |d}t ||||Wdn1s0Yt }tj|d}z0t||t d |trt|Wn$tyt d |Yn0dS) N checkpointsTr;rWriting checkpoint with nodes to r-repro.pyz&Copying repro file for convenience to zNo write permissions for )rrr(rCrArBrJgraphnodesr2r3r0save_graph_reprogetcwdshutilcopyfileuse_buckrr1OSError)rKr compiler_namer file_namer5curdirZ repro_pathrrrdump_compiler_graph_state&s"  , rc Csd}|D]}|jrd}qqd|vr.|d|t|||t|dd|vrz|tdt|dd n&|tdt|dd |d dS) Nr z>torch.cuda.synchronize() # Ensures that segfaults are surfacedrz!import torch._inductor.overrides r _accuracyz compiled = rz(mod, args) class AccuracyError(Exception): pass if not same_two_models(mod, compiled, args, only_fwd=True): raise AccuracyError("Bad accuracy detected") zA(mod, args) ref = compiled(args)  )rVr1rCOMPILER_REPRO_OPTIONSr*r+)r5rKrrZ sync_lineargrrrr:s4    rrc Cs|dur i}tjtd}tj|s8tj|ddtj|ttddd}t |dz}t ||}|dur| t |}| |t|d} | tdtd | d | td | d Wdn1s0Ytj} i| |} tt} } tr,t|j d d} nd|g} tj| || | | d}||jdkr| d| dttj| dddttj| ddddSd S)NZisolateTr;rr-rPz from z import rz if zk(mod, args): exit(1) else: exit(0) F)r4python)cwdristderrenvrzutf-8z>> )prefix) rrr(rrArBstruuiduuid4r0rr rr1rr*r+r7environcopyrrrrqPopenwait returncodeseekprintindentreadrs)fx_grrrZ patch_coderrr5Z repro_codeZfail_fnZnew_envrirr6prrr isolate_fails]sb $     "     rc sd|D]}|jrdqqfdd}ddlm}z4||}t|ttfsPJtdd|DrfJWnty|YdS0|z|||}|||WnVty}z<|dur|t|vrWYd}~dSt t|WYd}~dSd}~00dS) NFTcsrtjdSrm)rYrnZ synchronizerZhas_cudarrsyncszinductor_fails..syncrrcSsg|]}t|ttfqSr) isinstancer~r[rrrrrrz"inductor_fails..) rVtorch._inductor.compile_fxrrr~r[any Exceptionrzr) rr check_strrrrresultZ compile_moderrrrs0      cCsddlm}t|||S)Nrr)rrbackend_aot_accuracy_fails)rrrrrrrrs cCstjtdS)Nminifier_launcher.py)rrr(rCrrrrget_minifier_repro_pathsrc Cst}td|tr&t|z8t|d}||Wdn1sR0YWn8ty}z t|t d|WYd}~n d}~00dS)NzWriting minified repro to r-z(Could not write to {minified_repro_path}) rr2r3rrr1r0r exceptionNotImplementedError)contentsZminified_repro_pathr5rrrrhelper_for_dump_minifys  , rc CsNtjdkrdnd}tdt||dtd|d|d|d }t|S) NrPrrz isolate_fails_code_str = None z% from functools import partial from z import ( isolate_fails, dump_compiler_graph_state, ) from functorch.compile import minifier env_variables = {"CUDA_VISIBLE_DEVICES": "zk"} minifier( mod, args, module_fails=partial(isolate_fails, env=env_variables, compiler_name="zh", patch_code=isolate_fails_code_str), dump_state=partial(dump_compiler_graph_state, compiler_name="z"), ) )rYrnrvr*r+rr7r)rKrrZfavored_devicerrrrdump_to_minifys rc@s eZdZdS) AccuracyErrorN)r7r8r9rrrrrsrcstfdd}|S)a\ Minifier for Fx Graph modules after Aot Autograd has finished. We wrap both forward and backward call separately with the backend compiler_fn - like inductor or nvfuser. Intercepting after Aot Autograd presents neat abstration, where all the params are lifted as graph inputs, making it easy to save the graph as a string. csxddlmtjfi|tjtjdvs8Jdfdd}tjdkrj|}d|_ n }|S)Nr)FakeTensorModedynamoaotNc sVfdd|D}tjdkr6tt|tjdkrdkrPtddurbt|rtdt t|d tt|d t d n|SnzdurԈ|WSt yP}zZtjd krt t|n tjd kr0tt|t d WYd}~n d}~00dS)ah Aot Autograd fw_compiler and bw_compiler can have fake tensors. So, example_inputs can be fake tensors. We can call compiler_fn (which is inductor or nvfuser) with fake tensors but the actualy compiled_fn should be called with real tensors. Therefore, the actual invocation is deffered. csg|]}|qSr)Z from_tensorrZ fake_moderrrrz`wrap_compiler_debug..debug_wrapper..deferred_for_real_inputs..rRrz4Accuracy minification is supported for inductor onlyNz*Accuracy failed for the AOT Autograd graphrzBad accuracy detectedrrPZ CompilerError) r repro_levelrfx GraphModulerrr2r3rrrerror)Z real_inputsZcopy_tensor_attrsr)r compiler_fnrexample_inputsrKinner_compiled_fn orig_graphrrdeferred_for_real_inputs s\                zLwrap_compiler_debug..debug_wrapper..deferred_for_real_inputsrT) Ztorch._subclassesr functoolspartialrdeepcopyrr repro_after _boxed_call)rKrkwargsrZ compiled_fnrunconfigured_compiler_fn)rrrrKrrr debug_wrappers  F  z*wrap_compiler_debug..debug_wrapper)rwrapsrrrrrrwrap_compiler_debugs WrFcCsddlm}ddlm}m}m}t|}t|}t ||D]\}} | | j q<|}t |drj| dt|dd} t|d d} t |d s| dus| dur||}| dur| |_| dur| |_||} |r| S|| r|| } | ||| d|S) zR Runs a forward and possibly backward iteration for a given mod and args. r)make_boxed_funcr)collect_resultsreduce_to_scalar_lossrequires_bwd_pass zero_gradTnamed_parametersN named_buffersr)Ztorch._functorch.aot_autogradrtestingrrrrrr zipZrequires_grad_ requires_gradrrgetattrrrZbackward)rKronly_fwdrrrrnew_argsZnargrZorig_named_parametersZorig_named_buffersoutZlossrrrrun_fwd_maybe_bwdZs8       rc Csddlm}ddlm}m}ddlm}t||rF|||_|||_ t||rd|||_|||_ t |||}z(t t |t|\} } t | | |} Wn tytdd} Yn0zt |||} Wn2ty} ztdWYd} ~ dSd} ~ 00||| | tjdd }|S) z. Check two models have same accuracy. r)OptimizedModule)"named_buffers_for_optimized_module%named_parameters_for_optimized_module)samezCould not generate fp64 outputsNzWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graph.T)ZtolZ equal_nan)Z eval_framerrrrutilsrrrrr cast_to_fp64rrr rr2r3rrZrepro_tolerance)rKZopt_gmrrrrrrrefZ fp64_modelZ fp64_examplesZfp64_refresrZpassingrrrsame_two_modelss6            rcCs|jjD]b}|jdkr|jtjjjjkrt |j dks:Jt |j dr|j dtj kr|j dtj f|_ q|j ||S)N call_functionrPrr)rropr!rYopsZprimsZconvert_element_typedefaultrJrrfloat64ZlintZ recompile)modelnoderrr!cast_convert_element_type_to_fp64s  rcsBddlm}|}tjkr(t|}|fdd|}||fS)Nr)tree_mapcs"t|tjr|r|S|Srm)rrYZTensorrZto)rr]rrs   zcast_to..)Ztorch.utils._pytreerrrYrr)r]rinputsrrr rcast_tos    r cCsttj||Srm)r rYr)rr rrrrsrcCs|tdtd}tjdks$|r:tdtd}tdtdtdtd d d |Dd |d |d|dS)zH Generate a repro string for backend-agnostic minified version. z& with torch.cuda.amp.autocast(enabled=zY): ref = run_fwd_maybe_bwd(mod, args) res = run_fwd_maybe_bwd(opt_mod, args) rRzk mod.eval() opt_mod.eval() class AccuracyError(Exception): pass with torch.cuda.amp.autocast(enabled=z): assert same_two_models(mod, mod, args), "Eager itself failed" if not same_two_models(mod, opt_mod, args): raise AccuracyError("Dynamo failed") a  from math import inf import torch from torch import tensor, device import torch.fx as fx import torch._dynamo from torch._dynamo.testing import rand_strided from torch._dynamo.debug_utils import run_fwd_maybe_bwd from torch._dynamo.debug_utils import same_two_models r|r args = cSs0g|](}t|jt||j|jj|jfqSrr~r\rr]rrGrrrrrrrz3generate_dynamo_fx_repro_string..^ args = [rand_strided(sh, st, dt, dev).requires_grad_(rg) for (sh, st, dt, dev, rg) in args] z2 mod = Repro() opt_mod = torch._dynamo.optimize("z ")(mod) z ) r*r+rYis_autocast_enabledrrr{rr)rarrcheck_accuracyZrun_coderrrgenerate_dynamo_fx_repro_strings8    rc Cst}tjtd}tj|s4tj|ddtj|dt|jjd}t dt|jjd|t |}t |d$}|t||||Wd n1s0Ytj|d } t d |d | d trt| t|| d S)z, Saves the repro to a repro.py file rTr;Z minified_z _nodes.pyrrr-NrzCopying z to z for convenience)rrrr(rArBrJrrr2r3rDrfr0r1rrrrr) rKrrrrrrrar5Z latest_reprorrrdump_backend_repro_as_files$   " rcCst|sJt||||S)a  Dumps the dynamo graph to repro the issue. 1) It tries to convert Fx GraphModule to a string. If we can, it writes to a repro.py file. 2) If we can't convert Fx GraphModule to a string, we use to_folder to save the module and save a tar file. )rDrNr)rKrrrrrrdump_backend_statedsrc CsZz|t|t|}Wn0tyH}ztdWYd}~dSd}~00t|||| S)NzWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graphF)rrr rr2rr)rKrrr compiled_gmrrrrbackend_accuracy_failsqsrTrc Csddlm}z|||}t|t|WdStyz}z8t|}|d||dkrdWYd}~dSWYd}~dSd}~00dS)a. Minifier uses this function to identify if the minified graph module fails with the same error. One caveat is that minifier can potentially go into a wrong direction when the resulting graph module fails for a different reason. To avoid this, we save the string for the original exception and check similarity between new and old exception. They can be somewhat different in some cases, when the exception string depends on the failing node information. So, we have a loose similarity metric to guide the minifier path. r)SequenceMatcherFNg?T)difflibrrr rrZratio)rKrr orig_failurerrrZ new_failurerrr backend_failss  rcCst|}d}tjdkrd}|dur.tdnd}tdtdtdtd d d |Dd |d |d|d|dt d}t |dS)Ndynamo_minifier_backendrR dynamo_accuracy_minifier_backendad raise RuntimeError( 'Compiler name is None - this likely means that a custom compiler ' 'was called by torchdynamo. Please remove this error, import your ' 'custom compiler function, and replace the compiler_name="None" ' 'line below to compiler_name=' ) r a- import os from math import inf import torch from torch import tensor, device import torch.fx as fx import functools import torch._dynamo from torch._dynamo.debug_utils import run_fwd_maybe_bwd from torch._dynamo.backends.registry import lookup_backend from torch._dynamo.testing import rand_strided r|r r cSs0g|](}t|jt||j|jj|jfqSrrrrrrrrz/dump_to_minify_after_dynamo..rz mod = Repro() # Setup debug minifier compiler torch._dynamo.debug_utils.MINIFIER_SPAWNED = True compiler_fn = lookup_backend("z") zR dynamo_minifier_backend = functools.partial( compiler_fn, compiler_name="zj", ) opt_mod = torch._dynamo.optimize(dynamo_minifier_backend)(mod) with torch.cuda.amp.autocast(enabled=z): opt_mod(*args) ) rDrfrrr*r+r{rrrYrr)rKrrraZminifier_backendZcustom_compiler_errorrrrrdump_to_minify_after_dynamos@      !&rcs"tfdd}|_|S)a A minifier decorator that wraps the TorchDynamo produced Fx graph modules. As opposed to wrap_compiler_debug, this wrapper intercepts at the TorchDynamo produced Fx Graph Module. This makes it backend-agnostic to some level, e.g., it is useful for minifying issues related to Aot Autograd tracing. If an error is found, we minify and save the minified repro in repro.tar.gz. c stjfi|}tjdvs JtjdkrrtjdkrBt||tjdkr|t||}t|||rt dtt |t|j |td}tjtd|_|nz|t||}t||Wntyn}zt dtjd krtjtd }|t |t|j |n(tjd krFtt |t|j |tjtd|_WYd}~n d}~00n |||}|S) NrrrrRzXAccuracy failed for the TorchDyanmo produced graph. Creating script to minify the error.zBad accuracy detected.rDCompiled Fx GraphModule failed. Creating script to minify the error.rrrP)rrrrrrrrrr2r3rrrrrrr(rCZ minifier_pathrrr)rKrrrrexc dump_state_fnrrrrs^        z)wrap_backend_debug..debug_wrapper)rrZ_torchdynamo_orig_callablerrrrwrap_backend_debugs 7r"c Csddlm}t|}z |||}t||tdWn~ty}zft|}tdt j t |d}|t |t|j|t j t||d} |||| |dWYd}~n d}~00|S)Nrr:zNo issue was detectedrr)rrZ module_failsZ dump_state)functorch.compiler:rr ValueErrorrrr2r3rrrrrrrrr) rKrrr:rrr rr!fails_fnrrrr(s6    rcCsddlm}t|}|t|||tjdrtdt j t |dd}t j t|tjd}|t |t|j||||||dn td |S) Nrr#rz2Accuracy failed for the TorchDynamo produced graphT)rr)rrr$z*Input graph does not fail accuracy testing)r%r:revalrrZrepro_forward_onlyr2r3rrrrrrrrr)rKrrr:rr!r'rrrrIs0    r)NN)N)N)F)F)F)F)F)F)Lrrr?loggingrrrqr*r collectionsr importlibrr=rrYZtorch.fxrZtorch._prims_commonrr rZbackends.registryrr rr r getLoggerr7r2Zinductor_configZ is_fbcoderr)rr(rrCrD lru_cacheryr{rrZINDUCTOR_IMPORTrrrrrrrrrrrrrrrrr rrrrrrrZMINIFIER_SPAWNEDrrr"rrrrrrs       3 ] & #< # d ( 0 < R ?G