a
    þd\  ã                   @   s–   d dl Z d dlZd dlmZ zd dlmZ W n eyB   dZY n0 dd„ Zdd„ Z	ddd„Z
d	d
„ Zdd„ Zdd„ Zdd„ Zddd„Zddd„ZdS )é    Nc                 C   s
   | j dkS ©Nr   )Úrank©Úargs© r   ú]/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/training/distributed.pyÚis_global_master   s    r   c                 C   s
   | j dkS r   )Ú
local_rankr   r   r   r   Úis_local_master   s    r
   Fc                 C   s   |rt | ƒS t| ƒS ©N)r
   r   )r   Úlocalr   r   r   Ú	is_master   s    r   c                  C   s@   ddg} ddg}t dd„ | D ƒƒs4t dd„ |D ƒƒr8dS dS d S )	NÚOMPI_COMM_WORLD_RANKÚOMPI_COMM_WORLD_SIZEÚPMI_RANKÚPMI_SIZEc                 S   s   g | ]}|t jv ‘qS r   )ÚosÚenviron)Ú.0Úvarr   r   r   Ú
<listcomp>   ó    z$is_using_horovod.<locals>.<listcomp>TF)Úall)Z	ompi_varsZpmi_varsr   r   r   Úis_using_horovod   s
    $r   c                   C   s<   dt jv rtt jd ƒdkS dt jv r8tt jd ƒdkS dS )NÚ
WORLD_SIZEé   ÚSLURM_NTASKSF©r   r   Úintr   r   r   r   Úis_using_distributed#   s
    

r   c                  C   sˆ   d} dD ] }|t jv rtt j| ƒ}  q*qd}dD ] }|t jv r2tt j| ƒ} qTq2d}dD ] }|t jv r\tt j| ƒ} q~q\| ||fS )Nr   )Ú
LOCAL_RANKZMPI_LOCALRANKIDZSLURM_LOCALIDZOMPI_COMM_WORLD_LOCAL_RANK)ÚRANKr   ÚSLURM_PROCIDr   r   )r   r   r   r   r   )r	   ÚvZglobal_rankÚ
world_sizer   r   r   Úworld_info_from_env+   s     


r%   c                 C   s–  d| _ d| _d| _d| _| jrtd us.J dƒ‚t ¡  tt ¡ ƒ| _t ¡ | _t ¡ | _d| _ t	| jƒt
jd< t	| jƒt
jd< t	| jƒt
jd< n´tƒ rDd	t
jv rtƒ \| _| _| _t	| jƒt
jd< t	| jƒt
jd< t	| jƒt
jd< tj j| j| j| j| jd
 n:tƒ \| _}}tj j| j| jd tj  ¡ | _tj  ¡ | _d| _ tj ¡ r~| j rl| jsld| j }nd}tj |¡ nd}|| _t |¡}|S )NFr   r   zHorovod is not installedTr    r!   r   r"   )ÚbackendÚinit_methodr$   r   )r&   r'   zcuda:%dzcuda:0Úcpu)Údistributedr$   r   r	   ÚhorovodÚhvdÚinitr   ÚsizeÚstrr   r   r   r%   ÚtorchZinit_process_groupZdist_backendZdist_urlZget_world_sizeZget_rankZcudaZis_availableZno_set_device_rankZ
set_deviceÚdevice)r   Ú_r0   r   r   r   Úinit_distributed_device?   sV    

üþ
r2   c                 C   sF   | j rtj||dS | j|kr&|g}nd g}tj||d |d S d S )N)Z	root_rank)Úsrcr   )r*   r+   Úbroadcast_objectr   ÚdistZbroadcast_object_list)r   Úobjr3   Úobjectsr   r   r   r4   u   s    
r4   c                 C   s8   | j rt |¡S dd„ t| jƒD ƒ}t ||¡ |S d S )Nc                 S   s   g | ]}d ‘qS r   r   )r   r1   r   r   r   r   ‡   r   z%all_gather_object.<locals>.<listcomp>)r*   r+   Zallgather_objectÚranger$   r5   Úall_gather_object)r   r6   Údstr7   r   r   r   r9   ‚   s
    
r9   )F)r   )r   )r   r/   Ztorch.distributedr)   r5   Zhorovod.torchr+   ÚImportErrorr   r
   r   r   r   r%   r2   r4   r9   r   r   r   r   Ú<module>   s   

6
