a
    d                     @   s   d Z ddlZddlZddlmZ zddlmZ W n eyF   dZY n0 ddlm	Z	 dd Z
dd	d
Zdd Zdd ZdddZdd Zdd Zdd ZdS )zZ Distributed training/validation utils

Hacked together by / Copyright 2020 Ross Wightman
    N)distributed   )unwrap_modelc                 C   s&   |   }tj|tjjd || }|S )Nop)clonedist
all_reduceReduceOpSUM)Ztensornrt r   _/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/utils/distributed.pyreduce_tensor   s    r   Fc                 C   sb   t | jddD ]L\}}d|v s(d|v r|rNtjj|tjjd |t| }qtj	|d qd S )NT)ZrecurseZrunning_meanZrunning_varr   r   )
r   Znamed_bufferstorchr   r	   r   r
   r   float	broadcast)model
world_sizereduceZbn_nameZbn_bufr   r   r   distribute_bn   s    r   c                 C   s
   | j dkS Nr   )rankargsr   r   r   is_global_primary&   s    r   c                 C   s
   | j dkS r   )
local_rankr   r   r   r   is_local_primary*   s    r   c                 C   s   |rt | S t| S )N)r   r   )r   localr   r   r   
is_primary.   s    r    c                   C   s<   dt jv rtt jd dkS dt jv r8tt jd dkS dS )N
WORLD_SIZEr   SLURM_NTASKSFosenvironintr   r   r   r   is_distributed_env2   s
    

r'   c                  C   s   d} dD ] }|t jv rtt j| }  q*qd}dD ] }|t jv r2tt j| } qTq2d}dD ] }|t jv r\tt j| } q~q\| ||fS )Nr   )
LOCAL_RANKZMPI_LOCALRANKIDZSLURM_LOCALIDZOMPI_COMM_WORLD_LOCAL_RANK)RANKZPMI_RANKSLURM_PROCIDZOMPI_COMM_WORLD_RANKr   )r!   ZPMI_SIZEr"   ZOMPI_COMM_WORLD_SIZEr#   )r   vZglobal_rankr   r   r   r   world_info_from_env:   s     


r,   c                 C   s   d| _ d| _d| _d| _t| dd}t| dd}t rdtjv rt \| _| _| _t	| jtjd	< t	| jtjd
< t	| jtjd< t
j j||| j| jd n6t \| _}}t
j j||d t
j  | _t
j  | _d| _ t
j r| j rd| j }nd}t
j| nd}|| _t
|}|S )NFr   r   dist_backendZnccldist_urlzenv://r*   r(   r)   r!   )backendinit_methodr   r   )r/   r0   Tzcuda:%dzcuda:0cpu)r   r   r   r   getattrr'   r$   r%   r,   strr   Zinit_process_groupZget_world_sizeZget_rankZcudaZis_availableZ
set_devicedevice)r   r-   r.   _r4   r   r   r   init_distributed_deviceP   sF    

r6   )F)F)__doc__r$   r   r   r   Zhorovod.torchZhvdImportErrorr   r   r   r   r   r   r    r'   r,   r6   r   r   r   r   <module>   s    


