a
    dk                     @   s  d dl Z d dlmZ d dlmZmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZ d dlmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 d dl5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z;m<Z< d dl=m>Z> d dl?m@Z@ e	eeef ZAe	eAeBf ZCG dd dZDdS )    N)Counter)AnycastDictListOptionalUnion)get_args)ACCELERATOR_REGISTRY)Accelerator)CUDAAccelerator)MPSAccelerator)TPUAccelerator)CheckpointIODeepSpeedPrecisionMixedPrecision	PrecisionTPUBf16PrecisionTPUPrecision)ClusterEnvironmentKubeflowEnvironmentLightningEnvironmentLSFEnvironmentSLURMEnvironmentTorchElasticEnvironment)DoublePrecision)FSDPPrecision)_PRECISION_INPUT_PRECISION_INPUT_INT_PRECISION_INPUT_STR)DeepSpeedStrategyParallelStrategySingleDeviceStrategySingleTPUStrategyStrategySTRATEGY_REGISTRYXLAStrategy)_DDP_FORK_ALIASES)_FSDP_ALIASESFSDPStrategy)rank_zero_inforank_zero_warn)_determine_root_gpu_device)_IS_INTERACTIVEc                   @   s  e Zd ZdZd(eeeef  eeeef  eee	e
 ee
f  e
eeeee	e f  ddddZeeeef  eeeef  eeeee	e f  ddd	d
Zeee	e
 ee
f  e
ddddZedddZeedddZddddZddddZedddZeeef dddZddddZddddZeddd Zddd!d"Zddd#d$Zeeeeed%d&d'ZdS ))
_Connectora  The Connector parses several Fabric arguments and instantiates the Strategy including its owned components.

        A. accelerator flag could be:
            1. accelerator class
            2. accelerator str
            3. accelerator auto

        B. strategy flag could be:
            1. strategy class
            2. strategy str registered with STRATEGY_REGISTRY
            3. strategy str in _strategy_type enum which listed in each strategy as
               backend (registed these too, and _strategy_type could be deprecated)

        C. plugins flag could be:
            1. List of str, which could contain:
                i. precision str (Not supported in the old accelerator_connector version)
                ii. checkpoint_io str (Not supported in the old accelerator_connector version)
                iii. cluster_environment str (Not supported in the old accelerator_connector version)
            2. List of class, which could contains:
                i. precision class (should be removed, and precision flag should allow user pass classes)
                ii. checkpoint_io class
                iii. cluster_environment class


    priorities which to take when:
        A. Class > str
        B. Strategy > Accelerator/precision/plugins
    N       )acceleratorstrategydevices	num_nodes	precisionpluginsreturnc                 C   s0  | j d|d d}| j d|d d}| j d|d d}| j d|dd}| j d|dd}t | _t | _d | _d | _d	| _	d | _
d | _g | _d | _| j||||d
 | j||d | jdks| jd u r|  | _n| jdkr|  | _|   |  | _| jdv r
|  | _|   |   |  | _|   d S )Nr1   )defaultr2   r3   r4   r/   r5   r0   32)r2   r1   r5   r6   )r3   r4   autogpuNr:   )_argument_from_envr%   Zavailable_strategies_registered_strategiesr
   Zavailable_accelerators_registered_accelerators_strategy_flag_accelerator_flag_precision_input_precision_instance_cluster_environment_flag_parallel_devicescheckpoint_io!_check_config_and_set_final_flags(_check_device_config_and_set_final_flags_choose_auto_accelerator_choose_gpu_accelerator_backend*_set_parallel_devices_and_init_accelerator$_choose_and_init_cluster_environmentcluster_environment_choose_strategy_check_strategy_and_fallback_init_strategy_check_and_init_precisionr5   _lazy_init_strategy)selfr1   r2   r3   r4   r5   r6    rT   c/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/lightning_fabric/connector.py__init__^   sB    






z_Connector.__init__)r2   r1   r5   r6   r7   c                 C   s  |durt |ts|gn|}t |tr.| }|dur<|| _|dvrh|| jvrht |tshtd|d|dur|| jvr|dvrt |t	std|dd
| j d	t |tod
|v }t |tod|v }t |tod|v }t |tp|p|p|}t o|dv pt |t}	|	r0|r0td| d|| _tttt }
||
vrhtdt| d|
 ttt|| _|r@t }|D ]}t |tr|| _|tj  d7  < n\t |tr|| _|tj  d7  < n6t |tr|| _|tj  d7  < ntd| dqdd | D }|r@tdd
| d| jrt | jtr| jjr|| jrrtdn
| jj| _| jj r| jrtdn
| jj | _| jj!r| jrtdn
| jj!| _t"| jddr| jrtdnt"| jd| _t#| jdr| jj$r| jj$d j%d kr\| jrV| jd!vrVtd"| jj&j d#| j d$d | _| jj$d j%d%kr| jr| jd&vrtd'| jj&j d#| j d$d%| _| jj$| _'dS )(a  This method checks:

        1. strategy: whether the strategy name is valid, and sets the internal flags if it is.
        2. accelerator: if the value of the accelerator argument is a type of accelerator (instance or string),
            set self._accelerator_flag accordingly.
        3. precision: The final value of the precision flag may be determined either by the precision argument or
            by a plugin instance.
        4. plugins: The list of plugins may contain a Precision plugin, CheckpointIO, ClusterEnvironment and others.
            Additionally, other flags such as `precision` can populate the list with the
            corresponding plugin instances.
        Nr<   z1You selected an invalid strategy name: `strategy=z`. It must be either a string or an instance of `lightning.fabric.strategies.Strategy`. Example choices: ddp, ddp_spawn, deepspeed, dp, ... Find a complete list of options in our documentation at https://lightning.ai)r:   r;   z7You selected an invalid accelerator name: `accelerator=z`. Available names are: z, .ddpdpZ	deepspeed)mpsr:   r;   NzYou set `strategy=z` but strategies from the DDP family are not supported on the MPS accelerator. Either explicitly set `accelerator='cpu'` or change the strategy.z
Precision z' is invalid. Allowed precision values: r/   zFound invalid type for plugin z>. Expected one of: Precision, CheckpointIO, ClusterEnviroment.c                 S   s   g | ]\}}|d kr|qS )r/   rT   ).0kvrT   rT   rU   
<listcomp>       z@_Connector._check_config_and_set_final_flags.<locals>.<listcomp>zReceived multiple values for z> flags in `plugins`. Expected one value for each type at most.zLaccelerator set through both strategy class and accelerator flag, choose onezAprecision set through both strategy class and plugins, choose onezEcheckpoint_io set through both strategy class and plugins, choose onerM   zKcluster_environment set through both strategy class and plugins, choose oneparallel_devicesr   cpu)r:   ra   z!CPU parallel_devices set through z class, but accelerator set to z, please choose one device typecuda)r:   rb   r;   z!GPU parallel_devices set through )(
isinstanceliststrlowerr@   r>   r$   
ValueErrorr?   r   joinr!   r   is_availablerA   r	   r   r   reprr   rB   r   r   rC   __name__r   rF   r   rD   	TypeErroritemsZ_acceleratorZ
_precisionZ_checkpoint_iogetattrhasattrr`   type	__class__rE   )rS   r2   r1   r5   r6   Z
is_ddp_strZ	is_dp_strZis_deepspeed_strZis_parallel_strategyZis_mps_acceleratorZsupported_precisionZplugins_flags_typesZpluginZduplicated_plugin_keyrT   rT   rU   rG      s    

















z,_Connector._check_config_and_set_final_flags)r3   r4   r7   c                 C   s   |d urt |nd| _|| _| jg ddfv r`t| jtrB| jjjn| j}td| jd| d| jdkr| jd u rtd| d	d S )
Nr/   r   0z`Fabric(devices=z$)` value is not a valid input using z accelerator.r:   zYou passed `devices=z_` but haven't specified `accelerator=('auto'|'tpu'|'gpu'|'cpu'|'mps')` for the devices mapping.)	int_num_nodes_flag_devices_flagrc   rA   r   rq   __qualname__rg   )rS   r3   r4   Zaccelerator_namerT   rT   rU   rH   ,  s     


z3_Connector._check_device_config_and_set_final_flags)r7   c                 C   s2   | j dkr.t rdS t r"dS t r.dS dS )zTChoose the accelerator type (str) based on availability when ``accelerator='auto'``.r:   tpurZ   rb   ra   )rA   r   ri   r   r   rS   rT   rT   rU   rI   C  s    
z#_Connector._choose_auto_acceleratorc                   C   s$   t  rdS t rdS tdd S )NrZ   rb   zNo supported gpu backend found!)r   ri   r   RuntimeErrorrT   rT   rT   rU   rJ   N  s
    z*_Connector._choose_gpu_accelerator_backendc                 C   s   t | jtr| j| _n| jd us$J t| j| _| jj}| sjdd | jD }t	d|j
 d| d|   || j| _| js|| j| _d S )Nc                 S   s    g | ]}t | d   r|qS )r1   )r
   ri   )r[   Zacc_strrT   rT   rU   r^   `  s   zI_Connector._set_parallel_devices_and_init_accelerator.<locals>.<listcomp>`z` can not run on your system since the accelerator is not available. The following accelerator(s) is available and can be passed into `accelerator` argument of `Fabric`: rW   )rc   rA   r   r1   r
   getrq   ri   r?   ry   rv    _set_devices_flag_if_auto_passedZparse_devicesru   rE   Zget_parallel_devices)rS   Zaccelerator_clsZavailable_acceleratorrT   rT   rU   rK   W  s$    

z5_Connector._set_parallel_devices_and_init_acceleratorc                 C   s$   | j dks| j d u r | j | _ d S r<   )ru   r1   Zauto_device_countrx   rT   rT   rU   r|   r  s    z+_Connector._set_devices_flag_if_auto_passedc                 C   s<   t | jtr| jS ttttfD ]}| r|   S qt S )N)	rc   rD   r   r   r   r   r   detectr   )rS   Zenv_typerT   rT   rU   rL   v  s    z/_Connector._choose_and_init_cluster_environmentc                 C   s   | j dkr2| jr"t| jdkr"dS t| jd dS | jdkr@dS t| jdkrt| j ttfstt| j tr| j dv rt	| j}nd}t
|dS t| jdkrtrd	S dS )
Nrw   r/   Z	tpu_spawnr   )devicerX   )rb   r;   rZ   ra   Zddp_fork)rA   rE   lenr#   rt   rc   r   r   re   r,   r"   r-   rS   r~   rT   rT   rU   rN     s$    



z_Connector._choose_strategyc                 C   s   t | jtrdn| j}|dkrBt s>t s>t s>t rBd}|dkrf| jdkrft	|d d}|t
v rdtj vrtd| d	|tv st | jtr| jd
vrtd|r|| _dS )zChecks edge cases when the strategy selection was a string input, and we need to fall back to a
        different choice depending on other parameters or the environment. Z	ddp_spawnrX   rY   ra   z: is not supported on CPUs, hence setting `strategy='ddp'`.forkzYou selected `Fabric(strategy='zn')` but process forking is not supported on this platform. We recommed `Fabric(strategy='ddp_spawn')` instead.)rb   r;   zYou selected the FSDP strategy but FSDP is only available on GPU. Set `Fabric(accelerator='gpu', ...)` to continue or select a different strategy.N)rc   r@   r$   r   r}   r   r   r   rA   r+   r'   torchmultiprocessingZget_all_start_methodsrg   r(   r)   )rS   Zstrategy_flagrT   rT   rU   rO     s:    

z'_Connector._check_strategy_and_fallbackc                 C   s<   t | jttfsJ t | jtr0t| j| _n| j| _dS )zNInstantiate the Strategy given depending on the setting of ``_strategy_flag``.N)rc   r@   re   r$   r%   r{   r2   rx   rT   rT   rU   rP     s    z_Connector._init_strategyc                 C   s  |    t| jtr| jS t| jtrX| jdkr6t S | jdv rX| jdkrRtd t	 S t| j
trnt| jS | jdkr~t S | jdkrt S | jdkr| jdkrtd d| _| jdv rt| jdkrd	nd
 | jdkrdnd}t| j
tr t| j|dS t| j|dS tdd S )Nr9   )16bf16r   zzYou passed `Fabric(accelerator='tpu', precision=16)` but AMP is not supported with TPUs. Using `precision='bf16'` instead.64ra   z~You passed `Fabric(accelerator='cpu', precision=16)` but native AMP is not supported on CPU. Using `precision='bf16'` instead.r   z,Using 16-bit Automatic Mixed Precision (AMP)z.Using bfloat16 Automatic Mixed Precision (AMP)rb   )r5   r~   zNo precision set)_validate_precision_choicerc   rC   r   r1   r   rB   r   r+   r   r2   r    r   r   rA   r*   r)   r   r   ry   r   rT   rT   rU   rQ     sD    





z$_Connector._check_and_init_precisionc                 C   sJ   t | jtrF| jdkrtd| jrFt | jttfsFtd| j ddS )zCValidate the combination of choices for precision, and accelerator.r   z`Fabric(accelerator='tpu', precision=64)` is not implemented. Please, open an issue in `https://github.com/Lightning-AI/lightning/issues` requesting this feature.zKThe `TPUAccelerator` can only be used with a `TPUPrecision` plugin, found: rW   N)	rc   r1   r   rB   NotImplementedErrorrC   r   r   rg   rx   rT   rT   rU   r     s    
z%_Connector._validate_precision_choicec                 C   s  | j | j_ | jr| j| j_| jr*| j| j_t| jdrV| jjdu rL| j| j_| jj| _t| jdr| jjrv| jj| _n
| j| j_t| jdr| j| j_	t| jdr| j
  | j  tr| jjr| jjjstd| jdt| j trt| jttfstd| jjj d	dS )
zFLazily set missing attributes on the previously instantiated strategy.rM   Nr`   r4   set_world_ranksz`Fabric(strategy=a  )` is not compatible with an interactive environment. Run your code as a script, or choose one of the compatible strategies: `Fabric(strategy=None|'dp'|'ddp_notebook')`. In case you are spawning processes yourself, make sure to include the Fabric creation inside the worker function.zYThe `TPUAccelerator` can only be used with a `SingleTPUStrategy` or `XLAStrategy`, found rW   )r1   r2   r5   rF   ro   rM   r`   rE   rt   Z
_num_nodesr   Z_configure_launcherr-   launcherZis_interactive_compatiblery   r@   rc   r   r#   r&   rg   rq   rk   rx   rT   rT   rU   rR     s<    










z_Connector._lazy_init_strategy)namecurrentr8   r7   c              
   C   sv   t jd|   }|d u r |S |d urf|t|krft|t|krftd|  d|d|  d| d	|d u rr|S |S )NZLT_zYour code has `Fabric(=z+, ...)` but it conflicts with the value `--zZ` set through the CLI.  Remove it either from the CLI or from the Lightning Fabric object.)osenvironr{   upperre   rg   )r   r   r8   Z	env_valuerT   rT   rU   r=   '  s    $z_Connector._argument_from_env)NNNr/   r0   N) rk   
__module__rv   __doc__r   r   re   r   r$   r   rs   r   _PLUGIN_INPUTrV   rG   rH   rI   staticmethodrJ   rK   r|   r   rL   rN   rO   rP   r   rQ   r   rR   r   r=   rT   rT   rT   rU   r.   @   sP         D  	,)r.   )Er   collectionsr   typingr   r   r   r   r   r   r   Ztyping_extensionsr	   Zlightning_fabric.acceleratorsr
   Z)lightning_fabric.accelerators.acceleratorr   Z"lightning_fabric.accelerators.cudar   Z!lightning_fabric.accelerators.mpsr   Z!lightning_fabric.accelerators.tpur   Zlightning_fabric.pluginsr   r   r   r   r   r   Z%lightning_fabric.plugins.environmentsr   r   r   r   r   r   Z)lightning_fabric.plugins.precision.doubler   Z'lightning_fabric.plugins.precision.fsdpr   Z,lightning_fabric.plugins.precision.precisionr   r   r   Zlightning_fabric.strategiesr    r!   r"   r#   r$   r%   r&   Zlightning_fabric.strategies.ddpr'   Z lightning_fabric.strategies.fsdpr(   r)   Zlightning_fabric.utilitiesr*   r+   Z(lightning_fabric.utilities.device_parserr,   Z"lightning_fabric.utilities.importsr-   Z_PLUGINre   r   r.   rT   rT   rT   rU   <module>   s.      $	