a
    d                     @   s   d dl mZ d dlmZmZmZmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlZd dlmZ d dlmZ d d	lmZmZ d d
lmZmZ d dlmZ d dlmZ d dlmZ d dl m!Z! erd dl"m#Z# d dl$m%Z% ne& Z%Z#G dd deZ'dS )    )contextmanager)AnyDict	GeneratorListTuple)Tensor)Module)	OptimizerN)_optimizers_to_device)LightningOptimizer)_LightningModuleWrapperBase$_LightningPrecisionModuleWrapperBase)_FAIRSCALE_AVAILABLE_reinit_optimizers_with_oss)DDPStrategy)	TrainerFn)MisconfigurationException)rank_zero_deprecation)ShardedDataParallel)OSSc                       s   e Zd ZU dZdZdZeed< eedd fddZ	d	dd
 fddZ
dddddZddddZddddZeee eeee f dddZee ed dddZeddddZeedd d!Zddd"d#Zeedd$d%d&Z  ZS )'DDPShardedStrategyz>Optimizer and gradient sharded training provided by FairScale.Zddp_shardedi   _REDUCE_BUFFER_SIZE_DEFAULTN)argskwargsreturnc                    s   t d t j|i | d S )Na  PyTorch Lightning's sharded implementation using FairScale has been deprecated in v1.9.0 and will be removed in v2.0.0. You can try using the `Trainer(strategy='fsdp_native')` instead. The difference is that native FSDP uses PyTorch's implementation and the current strategy uses FairScale's implementation (which was upstreamed to PyTorch). After removal, `strategy='fsdp'` will use the native version by default.)r   super__init__)selfr   r   	__class__ m/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/strategies/sharded.pyr   ,   s    zDDPShardedStrategy.__init__zpl.LightningModule)modelr   c                    s   t stdt |S )Nzi`DDPShardedStrategy` requires `fairscale` to be installed. Install it by running `pip install fairscale`.)r   r   r   connect)r   r#   r   r!   r"   r$   6   s
    zDDPShardedStrategy.connectz
pl.Trainer)trainerr   c                 C   s   |  | j| _|  r|   | jd us,J | j| |   |jj}|t	j
krv| jrv| jd usfJ | j| j| _|   |t	j
kr|   d S N)	broadcastZ"_rank_0_will_call_children_scriptsZ_should_run_deadlock_detectionZ&_share_information_to_prevent_deadlockZacceleratorsetupZmodel_to_devicestatefnr   FITTINGZ_layer_syncr#   applyZsetup_precision_pluginconfigure_ddp)r   r%   Z
trainer_fnr!   r!   r"   r(   >   s    

zDDPShardedStrategy.setup)r   c                 C   sj   |    | jd usJ | | jj t| jtjtfs:J | j	t
| j| jd\| _| _t| j| j d S )N)r#   
optimizers)_set_ddp_kwargslightning_moduleZsetup_optimizersr%   
isinstancer#   plZLightningModuler   _setup_model_and_optimizersr   r.   r   Zroot_devicer   r!   r!   r"   r-   V   s    z DDPShardedStrategy.configure_ddpc                 C   s(   d| j vr$| jdkr| jnd| j d< d S )NZreduce_buffer_size   r   )_ddp_kwargs	num_nodesr   r4   r!   r!   r"   r/   a   s    
z"DDPShardedStrategy._set_ddp_kwargs)r#   r.   r   c                 C   s(   |  |}t|fd|i| j}||fS )a  Wraps the model and optimizers with fairscale components.

        Return:
            The model wrapped into a :class:`~fairscale.nn.data_parallel.ShardedDataParallel` module
            and a list of optimizer wrapped in :class:~`fairscale.optim.OSS`.
        Zsharded_optimizer)_wrap_optimizersr   r6   )r   r#   r.   r!   r!   r"   r3   f   s    
z.DDPShardedStrategy._setup_model_and_optimizersr   )r.   r   c                 C   sL   | j d usJ | jd ur.| j jjjtjkr.|S dd |D }t|| j| j	S )Nc                 S   s    g | ]}t |tr|jn|qS r!   )r1   r   Z
_optimizer).0or!   r!   r"   
<listcomp>u       z7DDPShardedStrategy._wrap_optimizers.<locals>.<listcomp>)
r0   r#   r%   r)   r*   r   r+   r   Zprecision_pluginr7   )r   r.   r!   r!   r"   r8   q   s
    z#DDPShardedStrategy._wrap_optimizers)closure_lossr   c                 C   s   d S r&   r!   )r   r=   r!   r!   r"   pre_backwardx   s    zDDPShardedStrategy.pre_backwardc                 c   sH   t | jtr>| j  dV  W d   qD1 s20    Y  ndV  dS )zBlocks syncing gradients behaviour on backwards pass.

        This is useful for skipping sync when accumulating gradients, reducing communication overhead
        Returns: context manager with sync behaviour off
        N)r1   r#   r   Zno_syncr4   r!   r!   r"   block_backward_sync{   s    &z&DDPShardedStrategy.block_backward_syncc                 C   s   d S r&   r!   r4   r!   r!   r"   post_training_step   s    z%DDPShardedStrategy.post_training_step)strategy_registryr   c                 C   s.   |j d| ddd |j | j| | jj d d S )NZ(ddp_sharded_find_unused_parameters_falsez;DDP Sharded Strategy with `find_unused_parameters` as FalseF)descriptionZfind_unused_parameters)rB   )registerstrategy_namer    __name__)clsrA   r!   r!   r"   register_strategies   s    z&DDPShardedStrategy.register_strategies)rE   
__module____qualname____doc__rD   r   int__annotations__r   r   r$   r(   r-   r/   r	   r   r
   r   r3   r8   r   r>   r   r   r?   r@   classmethodr   rG   __classcell__r!   r!   r   r"   r   &   s    

"r   )(
contextlibr   typingr   r   r   r   r   Ztorchr   Ztorch.nnr	   Ztorch.optimr
   Zpytorch_lightningr2   Z$lightning_fabric.utilities.optimizerr   Z pytorch_lightning.core.optimizerr   Z pytorch_lightning.overrides.baser   r   Z%pytorch_lightning.overrides.fairscaler   r   Z pytorch_lightning.strategies.ddpr   Z pytorch_lightning.trainer.statesr   Z&pytorch_lightning.utilities.exceptionsr   Z%pytorch_lightning.utilities.rank_zeror   Z&fairscale.nn.data_parallel.sharded_ddpr   Zfairscale.optimr   objectr   r!   r!   r!   r"   <module>   s$   