a
    d~                     @   s   d dl Z d dlmZ d dlmZmZmZ d dlmZ	 d dl
Zd dlmZ d dlmZ d dlmZ d dlmZmZmZmZ d dlmZ d d	lmZ G d
d deZdS )    N)SimpleQueue)AnyCallableOptional)_XLA_AVAILABLE)_rank_teardown)move_data_to_device)
_FakeQueue_GlobalStateSnapshot_MultiProcessingLauncher_WorkerOutput)	TrainerFn)rank_zero_debugc                
       s   e Zd ZdZddd fddZeeddd	Zdd
ee	e
d e	e	dddZdee
d ee	e	ee
e ddddZde	e
d dddZ  ZS )_XLALaunchera  Launches processes that run a given function in parallel on XLA supported hardware, and joins them all at the
    end.

    The main process in which this launcher is invoked creates N so-called worker processes (using the
    `torch_xla` :func:`xmp.spawn`) that run the given function.
    Worker processes have a rank that ranges from 0 to N - 1.

    Note:
        - This launcher requires all objects to be pickleable.
        - It is important that the entry point to the program/script is guarded by ``if __name__ == "__main__"``.

    Args:
        strategy: A reference to the strategy that is used together with this launcher
    zpl.strategies.TPUSpawnStrategyN)strategyreturnc                    s$   t sttt t j|dd d S )Nfork)r   start_method)r   ModuleNotFoundErrorstrsuper__init__)selfr   	__class__ s/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/strategies/launchers/xla.pyr   2   s    z_XLALauncher.__init__)r   c                 C   s   dS )NTr   )r   r   r   r   is_interactive_compatible7   s    z&_XLALauncher.is_interactive_compatible)trainerz
pl.Trainer)functionargsr   kwargsr   c          	      O   sp   t | j}| }ddlm  m} |j| j|||||f| j	j
| jd | }|du r^|S | || |jS )a  Launches processes that run the given function in parallel.

        The function is allowed to have a return value. However, when all processes join, only the return value
        of worker process 0 gets returned from this `launch` method in the main process.

        Arguments:
            function: The entry point for all launched processes.
            *args: Optional positional arguments to be passed to the given function.
            trainer: Optional reference to the :class:`~pytorch_lightning.trainer.trainer.Trainer` for which
                a selected set of attributes get restored in the main process after processes join.
            **kwargs: Optional keyword arguments to be passed to the given function.
        r   N)r    Znprocsr   )mpZget_contextZ_start_methodr   Z)torch_xla.distributed.xla_multiprocessingZdistributedZxla_multiprocessingspawn_wrapping_function	_strategyZnum_processesgetZ _recover_results_in_main_processZtrainer_results)	r   r   r   r    r!   contextreturn_queuexmpZworker_outputr   r   r   launch;   s    z_XLALauncher.launch)process_idxr   r   r    r!   r(   global_statesr   c           	      C   sN   ||i |}|d ur"|  ||}| jjdkr>|t|d t| jj d S )Nr   cpu)_collect_rank_zero_resultsr%   
local_rankputr   r   )	r   r+   r   r   r    r!   r(   r,   resultsr   r   r   r$   Y   s    z_XLALauncher._wrapping_functionr   )r   r1   r   c                 C   s   t d |j}|r"t|dr"|jnd }|j }d }|jjtj	krbt
j|jd}| jj|| | jjdkrrd S t }| || t|||j||S )Nz'Collecting results from rank 0 process.best_model_pathz
.temp.ckptr   )r   checkpoint_callbackhasattrr2   Zlightning_module
state_dictstatefnr   ZFITTINGospathjoinZdefault_root_dirr%   Zcheckpoint_ioZsave_checkpointr/   r	   Zadd_to_queuer   )r   r   r1   r3   r2   r5   Zweights_pathextrar   r   r   r.   o   s$    
z'_XLALauncher._collect_rank_zero_results)N)__name__
__module____qualname____doc__r   propertyboolr   r   r   r   r*   intr   r
   r$   r.   __classcell__r   r   r   r   r   "   s"    ( r   )r8   Zmultiprocessing.queuesr   typingr   r   r   Ztorch.multiprocessingmultiprocessingr"   Zpytorch_lightningplZ!lightning_fabric.accelerators.tpur   Z)lightning_fabric.strategies.launchers.xlar   Zlightning_fabric.utilitiesr   Z6pytorch_lightning.strategies.launchers.multiprocessingr	   r
   r   r   Z pytorch_lightning.trainer.statesr   Z%pytorch_lightning.utilities.rank_zeror   r   r   r   r   r   <module>   s   