a
    d\                     @   sJ   d dl Z d dlZd dlmZ d dlmZ d dlmZ dd Zd	ddZ	dS )
    N_get_device_index)autocast)ExceptionWrapperc                 C   sx   t | tjr| S t | ttfrBtt| D ]}t |tjr(|  S q(t | trttt|  D ]}t |tjrZ|  S qZd S )N)	
isinstancetorchZTensorlisttuplemap	get_a_vardictitems)objresult r   i/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/nn/parallel/parallel_apply.pyr      s    


r   c           
         sz  t | t |ksJ |dur2t | t |ks@J ni ft |  }|dur^t | t |kslJ ndgt |  }dd |D }dd |D }t i t t  d
fdd	 t | dkr fddtt| ||||D }|D ]}|  q|D ]}|	  q n& d	| d	 |d	 |d	 |d	 |d	  g }t
t |D ],}| }	t|	trh|	  ||	 qH|S )a~  Applies each `module` in :attr:`modules` in parallel on arguments
    contained in :attr:`inputs` (positional) and :attr:`kwargs_tup` (keyword)
    on each of :attr:`devices`.

    Args:
        modules (Module): modules to be parallelized
        inputs (tensor): inputs to the modules
        devices (list of int or torch.device): CUDA devices

    :attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and
    :attr:`devices` (if given) should all have same length. Moreover, each
    element of :attr:`inputs` can either be a single object as the only argument
    to a module, or a collection of positional arguments.
    Nc                 S   s   g | ]}t |d qS )Tr   .0xr   r   r   
<listcomp>/       z"parallel_apply.<locals>.<listcomp>c                 S   s   g | ]}t j|qS r   )r   cudacurrent_streamr   r   r   r   r   0   r   c                    s\  t  |d u rt| }|d u r2t j|}zt j| t j|\ t d2 t	|t
tfsp|f}||i |}W d    n1 s0    Y  W d    n1 s0    Y  W d    n1 s0    Y   || < W d    n1 s0    Y  W nP tyV   & td| |d| < W d    n1 sH0    Y  Y n0 d S )N)Zenabledzin replica {} on device {})where)r   Zset_grad_enabledr   Z
get_devicer   r   devicestreamr   r   r   r	   	Exceptionr   format)imoduleinputkwargsr   r   output)autocast_enabledgrad_enabledlockresultsr   r   _worker5   s"    
(h*
zparallel_apply.<locals>._worker   c              
      s4   g | ],\}\}}}}}t j ||||||fd qS ))targetargs)	threadingThread)r   r   r   r    r!   r   r   )r'   r   r   r   I   s   r   )NN)lenr+   Lockr   Zis_grad_enabledZis_autocast_enabled	enumeratezipstartjoinranger   r   reraiseappend)
modulesinputsZ
kwargs_tupZdevicesZstreamsthreadsthreadoutputsr   r"   r   )r'   r#   r$   r%   r&   r   parallel_apply   s:    

&r;   )NN)
r+   r   Ztorch.cuda._utilsr   Ztorch.cuda.ampr   Ztorch._utilsr   r   r;   r   r   r   r   <module>   s   