a
    dk\                  	   @   s  d dl Z d dlZd dlZd dlmZ d dlmZmZmZm	Z	 d dl
Z
ddlmZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZ d
dlmZmZ g fddZe je	ee eeef f dddZ e jeeef dddZ!e je	ee eeef f dddZ"d#e jeeef e#e	e jeeef f dddZ$ee eeef dddZ%ee je	e jeeef f dddZ&dd  Z'G d!d" d"Z(dS )$    Nliteral_eval)AnyDictListTuple   )SageMakerConfig)DYNAMO_BACKENDS)DynamoBackendPrecisionTypeis_ipex_availableis_npu_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)is_port_in_usemerge_dicts   )DistributedTypeSageMakerDistributedTypec                 C   sD   | |\}}t|  D ]$\}}|t| v rt||| q|S )z4
    Filters out all `accelerate` specific args
    )parse_known_argsvarsitemskeyssetattr)argsparserZdefault_argsnew_args_keyvalue r!   `/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/accelerate/utils/launch.py_filter_args%   s
    r#   )r   returnc              	   C   s"  g }| j r| jrtd| j s:|tj | jr:|d || j || j t	j
 }t| jph| j|d< | jdkr| jdurt r| j|d< nt r| j|d< n
| j|d< | jd	kr| j|d
< t| j|d< nB| jd	kr| jdur| jnd|d
< | jdurt| jnd|d< zt| j }W n4 ty^   td| j  dt  dY n0 t||d< zt| j }W n0 ty   td| j  dt dY n0 |j|d< | j|d< t| j |d< t| j!|d< t| j"|d< t# rt| j$ |d< t| j% |d< ||fS )zz
    Prepares and returns the command list and an environment with the correct simple launcher environment variables.
    0--module and --no_python cannot be used togetherz-mZACCELERATE_USE_CPUallNZE_AFFINITY_MASKASCEND_RT_VISIBLE_DEVICESCUDA_VISIBLE_DEVICESr   ZMASTER_ADDRZMASTER_PORTz	127.0.0.1Z29500Unknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONUnknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMICOMP_NUM_THREADSZACCELERATE_USE_IPEXZACCELERATE_USE_XPU)&	no_pythonmodule
ValueErrorappendsys
executabletraining_scriptextendtraining_script_argsosenvironcopystrcpuZuse_cpugpu_idsr   r   num_machinesmain_process_ipmain_process_portnum_processesr   mixed_precisionlowerlistr   dynamo_backendupperr
   r    dynamo_modedynamo_use_fullgraphdynamo_use_dynamicnum_cpu_threads_per_processr   ZipexZuse_xpu)r   cmdcurrent_envrG   rJ   r!   r!   r"   prepare_simple_launcher_cmd_env0   sV    





"

rR   c           
   	   C   s
  t | d}t | d}t | d}t | d}|dkrt| dt||  t| dt| t| dt| j t | d	d
rt| dt| t| dt| qt| d| d|  n(t| dt| |durt| dt| |du rd}t|rtd| d| jr| jrt	dn*| jr(t| dd n| jr<t| dd t
j }t | dd}|dkr| jdurt sz||d< nt r||d< n||d< | j }zt|}W n. t	y   t	d| dt  dY n0 t||d< zt| j }W n0 t	y(   t	d | j  dt dY n0 |j|d!< | j|d"< t| j|d#< t| j|d$< | jr6d%|d&< t| j|d'< t| j |d(< t| j|d)< | j durt| j |d*< | j!durt| j!|d+< | j"durt| j"|d,< | j#dur t| j#|d-< t| j$ |d.< t| j% |d/< t| j& |d0< | j'rd1}	d%|d2< t| j(||	d3 < t| j)||	d4 < t| j*||	d5 < | j+durt| j+||	d6 < | j,durt| j,||	d7 < | j-durt| j-||	d8 < | j.durt| j.||	d9 < t| j/|d:< |S );z_
    Prepares and returns an environment with the correct multi-GPU environment variables.
    rF   rC   rD   rE   r   nproc_per_nodennodes	node_ranksame_networkFmaster_addrmaster_portrdzv_endpoint:N<s  3Tried to launch distributed communication on port `&  `, but another process is utilizing it. Please specify a different port (such as using the `----main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.r%   r5   Tr4   rB   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   trueZACCELERATE_USE_FSDPZFSDP_SHARDING_STRATEGYZFSDP_OFFLOAD_PARAMSZFSDP_MIN_NUM_PARAMSZFSDP_AUTO_WRAP_POLICYZFSDP_TRANSFORMER_CLS_TO_WRAPZFSDP_BACKWARD_PREFETCHZFSDP_STATE_DICT_TYPEZFSDP_FORWARD_PREFETCHZFSDP_USE_ORIG_PARAMSZFSDP_SYNC_MODULE_STATESZMEGATRON_LM_ZACCELERATE_USE_MEGATRON_LMZ	TP_DEGREEZ	PP_DEGREEZGRADIENT_CLIPPINGZNUM_MICRO_BATCHESZSEQUENCE_PARALLELISMZRECOMPUTE_ACTIVATIONSZUSE_DISTRIBUTED_OPTIMIZERr3   )0getattrr   r@   intmachine_rankr   ConnectionErrorr5   r4   r6   r=   r>   r?   rB   r   r   rG   rH   r   rI   r   rJ   rK   r
   r    rL   rM   rN   Zuse_fsdpZfsdp_sharding_strategyZfsdp_offload_paramsZfsdp_min_num_paramsZfsdp_auto_wrap_policyZ"fsdp_transformer_layer_cls_to_wrapZfsdp_backward_prefetch_policyZfsdp_state_dict_typeZfsdp_forward_prefetchZfsdp_use_orig_paramsZfsdp_sync_module_statesZuse_megatron_lmZmegatron_lm_tp_degreeZmegatron_lm_pp_degreeZmegatron_lm_gradient_clippingZmegatron_lm_num_micro_batchesZ megatron_lm_sequence_parallelismZ!megatron_lm_recompute_activationsZ%megatron_lm_use_distributed_optimizerrO   )
r   rF   rC   rD   rE   rQ   rB   rG   rJ   prefixr!   r!   r"   prepare_multi_gpu_envg   s    









 "

rd   c           	   	   C   s0  t | d}t | d}t | d}t | d}d}| jdu rFt| dtd  |dkrL| jtd krLd	d
g}|dt| jdt| jg | jdur|dt| jg n:| jdur|dt| jg n|dt| j	| j
 g |dt|g | jr| jrtdn&| jr |d n| jr2|d || j || j n|dkr| jtd krt| dt||  t| dt| t| dt| j t | ddrt| dt| t| dt| nt| d| d|  n*t| dt| |durt| dt| |du rd}t|r6td| d| jrP| jrPtdn*| jrft| d d! n| jrzt| d"d! tj }t | d#d$}|d$kr| jdurt s||d%< n||d&< zt| j }W n4 ty   td'| j  d(t  d)Y n0 td*tj d)|d*< t||d+< t| j! |d,< d-|d.< | j"dur^t| j"|d/< | j#durxt| j#|d0< | j$durt| j$ |d1< | j%durt| j% |d2< | j&durt| j& |d3< | j'durt| j' |d4< | j(durt| j( |d5< | j)dur(t| j)|d6< ||fS )7zt
    Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
    rF   rC   rD   rE   Ndeepspeed_multinode_launcherr   r   Z	deepspeedz--no_local_rankz
--hostfilez
--launcherz	--excludez	--includez
--num_gpusz--master_portr%   z--modulez--no_pythonrS   rT   rU   rV   FrW   rX   rY   rZ   r[   r\   r]   r5   Tr4   rB   r&   r)   r'   r*   r+   r,   
PYTHONPATHr-   ZACCELERATE_CONFIG_DS_FIELDSr^   ZACCELERATE_USE_DEEPSPEEDZACCELERATE_DEEPSPEED_ZERO_STAGEZ&ACCELERATE_GRADIENT_ACCUMULATION_STEPSZACCELERATE_GRADIENT_CLIPPINGZ-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICEZ)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEZACCELERATE_DEEPSPEED_ZERO3_INITZ+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODELZ ACCELERATE_DEEPSPEED_CONFIG_FILE)*r_   re   r   r   r;   r@   Zdeepspeed_hostfileZdeepspeed_exclusion_filterZdeepspeed_inclusion_filterrF   rC   r5   r4   r6   r7   r:   r<   r`   ra   r   rb   r=   r>   r?   rB   r   r   rG   rH   rI   env_var_path_addpathabspathZ'deepspeed_fields_from_accelerate_configZ
zero_stageZgradient_accumulation_stepsZgradient_clippingZoffload_optimizer_deviceZoffload_param_deviceZzero3_init_flagZzero3_save_16bit_modelZdeepspeed_config_file)	r   rF   rC   rD   rE   rP   rQ   rB   rG   r!   r!   r"   prepare_deepspeed_cmd_env   s    
















rj   F)r   rQ   podr$   c                 C   s>   | j dkr"| jrd|d< nd|d< |r6| j| _| j| _| |fS )zY
    Prepares and returns an environment with the correct TPU environment variables.
    Zbf161ZXLA_DOWNCAST_BF16ZXLA_USE_BF16)rG   Zdowncast_bf16Ztpu_vmZvmZtpu_nameZtpu)r   rQ   rk   r!   r!   r"   prepare_tpu;  s    

rm   )nargsr$   c                 C   s   t | dk ri S dd }t }|| \}}t|D ]p\}}|dr6d }|d t |k rx||d  drtdntd|d u r|j||d q6|j||d q6d	d
 || j	
 D S )Nr   c                 S   s@   z$t | } | d | kr t| W S | W S  ty:   |  Y S 0 d S )Nr   )floatr`   r6   )sr!   r!   r"   _infer_typeR  s    
z+_convert_nargs_to_dict.<locals>._infer_type)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)actionc                 S   s&   i | ]\}}||d v rt |n|qS ))TrueFalser   ).0r   r    r!   r!   r"   
<dictcomp>q  s   z*_convert_nargs_to_dict.<locals>.<dictcomp>)lenargparseArgumentParserr   	enumerate
startswithr6   add_argument
parse_args__dict__r   )rn   rq   r   r   unknownindexargumentrt   r!   r!   r"   _convert_nargs_to_dictM  s,    

r   )sagemaker_configr   r$   c                 C   s2  t d | jtjd< | jd ur,| jtjd< n6|jd urZ|jd urZ|jtjd< |jtjd< ntdtj	|j
}|sxd}tj|j
}|dstd	| d
t d t|j}zt|j }W n2 ty   td|j  dt  dY n0 zt|j }W n0 ty:   td|j  dt dY n0 dt||j|jt|jt|j| jjd}d }| jtjkrddddiii}d }	| j d ur*t d| j  d i }	t!| j P}
t"|
D ]6\}}|dkrڐq|#d}|d $ |	|d < qW d    n1 s0    Y  t d|	  d }| j%d urt d| j% d g }t!| j%\}
t"|
D ]B\}}|dkrzqd|#d}|d |d $ d}|&| qdW d    n1 s0    Y  t d|  t d | j'||| j(| j)| j*| j+| j,| j-| j.d||||d }| j/d ur*t0| j/|}||	fS )!Nz(Configuring Amazon SageMaker environmentZAWS_DEFAULT_REGIONZAWS_PROFILEZAWS_ACCESS_KEY_IDZAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profiler,   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersr*   r+   r.   r^   )ZACCELERATE_USE_SAGEMAKERr-   r/   r0   r1   r2   Z%ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPEZsmdistributedZdataparallelZenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameZinstance_countZinstance_typeZdebugger_hook_configdistributionhyperparametersenvironmentZmetric_definitions)1printZregionr=   r>   ZprofileZaws_access_key_idZaws_secret_access_keyEnvironmentErrorrh   dirnamer:   basenameendswithr6   r   r<   r   rG   rH   rI   r   rJ   rK   r
   r@   r    rL   rM   rN   distributed_typer   ZDATA_PARALLELZsagemaker_inputs_fileopenr|   splitstripZsagemaker_metrics_filer7   r   Ziam_role_namer   r   r   r   rC   Zec2_instance_typeZadditional_argsr   )r   r   r   r   r   rG   rJ   r   r   Zsagemaker_inputsfileilinelZsagemaker_metricsZmetric_dictr!   r!   r"   prepare_sagemager_args_inputsw  s    



"


8


.r   c                 C   s6   dd t j| ddD }|t| d|S )z
    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
    caller to set it in os.environ.
    c                 S   s   g | ]}t |d kr|qS )r   )ry   )rw   pr!   r!   r"   
<listcomp>      z$env_var_path_add.<locals>.<listcomp> rZ   )r=   r>   getr   r7   r@   join)Zenv_var_nameZpath_to_addpathsr!   r!   r"   rg     s    rg   c                   @   s"   e Zd ZdZd	ddZdd ZdS )
PrepareForLaunchai  
    Prepare a function that will launched in a distributed setup.

    Args:
        launcher (`Callable`):
            The function to launch.
        distributed_type ([`~state.DistributedType`]):
            The distributed type to prepare for.
        debug (`bool`, *optional*, defaults to `False`):
            Whether or not this is a debug launch.
    NOFc                 C   s   || _ t|| _|| _d S )N)launcherr   r   debug)selfr   r   r   r!   r!   r"   __init__  s    
zPrepareForLaunch.__init__c                 G   s   | j rBttjd}tjd}tjjd|tj|||d n6| j	t
jt
jt
jt
jfv rxt|tjd< t|tjd< tdtjd< | j|  d S )	NZ
WORLD_SIZEZACCELERATE_DEBUG_RDV_FILEZgloo)Zrankstore
world_sizeZ
LOCAL_RANKZRANKr   ZFORK_LAUNCHED)r   r`   r=   r>   r   torchZdistributedZinit_process_groupZ	FileStorer   r   Z	MULTI_GPUZ	MULTI_NPUZ	MULTI_XPUZ	MULTI_CPUr@   r   )r   r   r   r   Zrdv_filer!   r!   r"   __call__  s&    zPrepareForLaunch.__call__N)r   F)__name__
__module____qualname____doc__r   r   r!   r!   r!   r"   r     s   
r   )F))rz   r=   r8   astr   typingr   r   r   r   r   Zcommands.config.config_argsr	   Zcommands.config.config_utilsr
   utilsr   r   r   r   r   Zutils.constantsr   Zutils.otherr   r   Zdataclassesr   r   r#   	Namespacer@   rR   rd   rj   boolrm   r   r   rg   r   r!   r!   r!   r"   <module>   s4   &7e&p +o
