a
    dp                     @   sj   d dl Z d dlmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ deeeedd	d
ZdS )    N)AnyCallable)_distributed_available)TrainerStatus)_TunerExitException)rank_zero_warnz
pl.Trainer)trainer
trainer_fnargskwargsreturnc              
   O   sb  zD| j jdur0| j jj|g|R d| i|W S ||i |W S W n tyx   |   |   tj| j_	d| j_
Y n ty } zFtd | jstj| j_	| d| | jD ]}|d qW Y d}~nd}~0  ty\ } zptj| j_	t r| jdkr| j t  | d| | jD ]}|d q$|   d| j_
 W Y d}~n
d}~0 0 dS )ap  
    Error handling, intended to be used only for main trainer function entry points (fit, validate, test, predict)
    as all errors should funnel through them

    Args:
        trainer_fn: one of (fit, validate, test, predict)
        *args: positional arguments to be passed to the `trainer_fn`
        **kwargs: keyword arguments to be passed to `trainer_fn`
    Nr   z;Detected KeyboardInterrupt, attempting graceful shutdown...Zon_exceptionfailed   )ZstrategylauncherZlaunchr   Z_call_teardown_hookZ	_teardownr   FINISHEDstatestatusZstageKeyboardInterruptr   interruptedZINTERRUPTEDZ_call_callback_hooksloggersfinalizeBaseExceptionr   Z
world_sizeZreconciliate_processes	traceback
format_exc)r   r	   r
   r   	exceptionlogger r   g/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py_call_and_handle_interrupt   s4    
"


 

r   )r   typingr   r   Zpytorch_lightningplZ&lightning_fabric.utilities.distributedr   Z pytorch_lightning.trainer.statesr   Z&pytorch_lightning.utilities.exceptionsr   Z%pytorch_lightning.utilities.rank_zeror   r   r   r   r   r   <module>   s   