a
    d9                     @   s4  d dl Z d dlZd dlmZmZmZmZmZ d dlZd dl	m
Z
 d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# erd dl$m%Z%m&Z& d dl'm(Z( ndZ(e)e*Z+G dd deZ,G dd deZ-e
e.dddZ/dS )    N)AnyDict	GeneratorListOptional)	Optimizer)CheckpointIOClusterEnvironment)_optimizers_to_device)_LightningModuleWrapperBase)_FAIRSCALE_AVAILABLE)PrecisionPlugin)DDPStrategy)	TrainerFn)MisconfigurationException)is_overridden)rank_zero_deprecation)STEP_OUTPUT)default_auto_wrap_policyenable_wrap)FullyShardedDataParallelc                   @   s&   e Zd Zeeeeef dddZdS )%_DDPFullyShardedStrategyModuleWrapperargskwargsreturnc                 O   s   | j j|i |S N)Z_forward_module
state_dictselfr   r    r    s/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/strategies/fully_sharded.pyr   ,   s    z0_DDPFullyShardedStrategyModuleWrapper.state_dictN)__name__
__module____qualname__r   r   strr   r    r    r    r!   r   +   s   r   c                       s  e Zd ZdZd2ed eeeee ee eej eeeee	ej
  ee ee ee ee d fd	d
ZeedddZeeef dddZddd fddZdd fddZdddddZddd fddZejjedddZejedddZ ddd d!Z!ddd"d#Z"eee#d$d%d&Z$eeee# d$d'd(Z%eeee# d$d)d*Z&eee#d$d+d,Z'ddd-d.Z(e)edd/d0d1Z*  Z+S )3DDPFullyShardedStrategyZddp_fully_shardedNFT    zpl.accelerators.Accelerator)acceleratorcpu_offloadflatten_parametersreshard_after_forwardmove_grads_to_cpufp32_reduce_scattercompute_dtypebucket_cap_mbmin_num_paramsstate_dict_to_cpuparallel_devicescluster_environmentcheckpoint_ioprecision_pluginprocess_group_backendc                    sn   t d t j||||||d || _|| _|| _|| _|| _|| _|| _	|	| _
|
r^tdnd| _d| _dS )ao  Plugin for Fully Sharded Data Parallel provided by FairScale.

        .. warning:: ``DDPFullyShardedStrategy`` is in beta and subject to change.

        Full Sharded Training shards the entire model across all available GPUs, allowing you to scale model
        size, whilst using efficient communication to reduce overhead. In practice, this means we can remain
        at parity with PyTorch DDP, whilst scaling our model sizes dramatically. The technique is similar
        to ZeRO-Stage 3 but has been built for upstreaming to PyTorch.

        For more information
        `check out FairScale's docs <https://fairscale.readthedocs.io/en/latest/api/nn/fsdp.html>`__.

        Defaults have been set and options have been exposed, but may require configuration
        based on your level of memory/speed efficiency. We suggest having a look at
        `this PR for more information <https://github.com/facebookresearch/fairscale/pull/413>`__.

        Many of the helpful doc strings below came from the original
        `FairScale documentation <https://fairscale.readthedocs.io/en/latest/api/nn/fsdp.html>`__.

        Arguments:
            cpu_offload: Offload FP32 params to CPU. Only usable in precision=16 mode.
                (Default: False).
            move_grads_to_cpu: Moves gradient shards to CPU after reduction.
                Only disable if using CPU based optimizers
                (Default to ``cpu_offload``).
            flatten_parameters: Flattens parameter into single contiguous tensor for speed efficiency
                (Default: True).
            reshard_after_forward: Reshard parameters after the forward pass, which saves memory but slows
                down training. This is only relevant when resharding individual layers.
                (Default: True).
            fp32_reduce_scatter: Reduce-Scatter gradients in FP32. Only relevant in mixed precision
                (Default: None).
            compute_dtype: dtype for full parameters for computation. Default to torch.float32,
                unless using mixed precision, in which case defaults to torch.float16.
                (Default: None).
            bucket_cap_mb: bucket parameters so that gradient reduction
                can potentially overlap with backward computation.
                bucket_cap_mb controls the bucket size in MegaBytes (MB).
                Buckets are sub-divided based on world_size,
                so the max shard size is roughly bucket_cap_mb / world_size.
                Values <= 0 disable bucketing.
                (Default: 25).
            min_num_params: Number of parameters to wrap when using FairScale ``auto_wrap``.
                (Default: 1e8)
            state_dict_to_cpu: Whether to return parameters (returned by :func:`state_dict`) on CPU device.
                If ``False``, this will default to ``compute_device``.
                (Default: True).
        a  PyTorch Lightning's sharded implementation using FairScale has been deprecated in v1.9.0 and will be removed in v2.0.0. You can try using the `Trainer(strategy='fsdp_native')` instead. The difference is that native FSDP uses PyTorch's implementation and the current strategy uses FairScale's implementation (which was upstreamed to PyTorch). After removal, `strategy='fsdp'` will use the native version by default.)r)   r3   r4   r5   r6   r7   cpuN)r   super__init__r*   r-   r+   r,   r.   r/   r0   r1   torchdevicestate_dict_device_process_group)r   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   	__class__r    r!   r:   7   s*    Bz DDPFullyShardedStrategy.__init__)r   c                 C   s   | j d u rtj | _ | j S r   )r>   r;   ZdistributedZ	new_groupr   r    r    r!   process_group   s    
z%DDPFullyShardedStrategy.process_groupc                 C   s   | j dusJ | j  S )zReturns model state.N)modelr   rA   r    r    r!   lightning_module_state_dict   s    z3DDPFullyShardedStrategy.lightning_module_state_dictzpl.LightningModule)rC   r   c                    sV   t d| jsFtt| dkrF| jdus.J td| jjjj dt	 
| dS )zTCalled by the accelerator to connect the accelerator and the model with this plugin.configure_sharded_modelr   Nz/Using the same instance of model with `trainer.zm()` is not supported with Fairscale FSDP auto-wrap. Please reinitialize your `LightningModule` and pass that.)r   lightning_modulelenlist
parametersr   trainerstatefnr9   connectr   rC   r?   r    r!   rM      s     zDDPFullyShardedStrategy.connectc                    s"   | j jdkstdt   d S )NZcudazJYou selected strategy to be `ddp_fully_sharded`, but GPU is not available.)root_devicetyper   r9   setup_distributedrA   r?   r    r!   rQ      s
    z)DDPFullyShardedStrategy.setup_distributedz
pl.Trainer)rJ   r   c                 C   s   | j s
J | j | |jjtjkrD| jrD| js4J | j| j| _| 	  t
| jtjs^J t| j| _| jd usxJ td| js| | j| _| | jj t| j| j |   |   d S )NrE   )r)   setuprK   rL   r   ZFITTINGZ_layer_syncrC   applyconfigure_ddp
isinstanceplZLightningModuler   rF   r   _setup_modelsetup_optimizersrJ   r
   
optimizersrO   ZbarrierZsetup_precision_plugin)r   rJ   r    r    r!   rR      s     

zDDPFullyShardedStrategy.setupc              
      sp   d}zt  | W n6 tyJ } zdt|vr2 d}W Y d }~n
d }~0 0 |sdtdd | jD rltdd S )NFz%optimizer got an empty parameter listTc                 s   s   | ]}t | V  qd S r   )_optimizer_has_flat_params).0	optimizerr    r    r!   	<genexpr>       z;DDPFullyShardedStrategy.setup_optimizers.<locals>.<genexpr>zThe optimizer does not seem to reference any FSDP parameters. HINT: Make sure to create the optimizer after setting up the model by referencing `self.trainer.model.parameters()` in the `configure_optimizers()` hook.)r9   rX   
ValueErrorr%   anyrY   )r   rJ   Zinvalid_params_errorer?   r    r!   rX      s    z(DDPFullyShardedStrategy.setup_optimizersc                 C   sN   t d| jj d t|| j| j| j| j| j	j
dk| j| j| j| j| jdS )zWraps the model into a
        :class:`~fairscale.nn.data_parallel.fully_sharded_data_parallel.FullyShardedDataParallel` module.z2setting up `Fairscale FSDP` model with device id: .16)modulerB   r*   r-   r+   mixed_precisionr,   r.   r/   r0   r=   )logdetailrO   indexr   rB   r*   r-   r+   r6   	precisionr,   r.   r/   r0   r=   rN   r    r    r!   rW      s    
z$DDPFullyShardedStrategy._setup_modelc                 #   s   t  jj d  jj}tttd fdd}tt| j	 j
 j j|dk j j j j jd d V  W d    n1 s0    Y  t  jj d d S )Nz : entered model_sharded_context.r   c                     s   t | i |d jiS )Nr1   )r   r1   )r   r   rA   r    r!   wrap_policy   s    zBDDPFullyShardedStrategy.model_sharded_context.<locals>.wrap_policyrc   )Zwrapper_clsZauto_wrap_policyrB   r*   r-   r+   re   r,   r.   r/   r0   r=   z : exiting model_sharded_context.)rf   rg   r@   r"   r6   ri   r   r   r   rB   r*   r-   r+   r,   r.   r/   r0   r=   )r   ri   rj   r    rA   r!   model_sharded_context   s&    $z-DDPFullyShardedStrategy.model_sharded_contextc                 C   s.   t | jj d| j d | js*|   d S )Nz%: configuring FSDP... (cpu_offload: [z]))rf   rg   r@   r"   r*   model_to_devicerA   r    r    r!   rT     s    z%DDPFullyShardedStrategy.configure_ddpc                 C   s8   t | jj d| j d | js&J | j| j d S )Nz: moving model to device [z]...)rf   rg   r@   r"   rO   rF   torA   r    r    r!   rl     s    
z'DDPFullyShardedStrategy.model_to_devicer   c                 O   s   | j d usJ | j |i |S r   rC   r   r    r    r!   training_step  s    z%DDPFullyShardedStrategy.training_stepc                 O   s   | j d usJ | j |i |S r   rn   r   r    r    r!   validation_step  s    z'DDPFullyShardedStrategy.validation_stepc                 O   s   | j d usJ | j |i |S r   rn   r   r    r    r!   	test_step  s    z!DDPFullyShardedStrategy.test_stepc                 O   s   | j d usJ | j |i |S r   rn   r   r    r    r!   predict_step"  s    z$DDPFullyShardedStrategy.predict_stepc                 C   s   d S r   r    rA   r    r    r!   post_training_step&  s    z*DDPFullyShardedStrategy.post_training_step)strategy_registryr   c                 C   s,   |j d| dd |j | j| | jj d d S )NZfsdpz>Fully sharded training with checkpointing the full state dict.)description)registerstrategy_namer@   r"   )clsrt   r    r    r!   register_strategies)  s    z+DDPFullyShardedStrategy.register_strategies)NFTTNNNr'   r(   TNNNNN),r"   r#   r$   rw   r   boolr;   Zdtypeintr   r<   r	   r   r   r%   r:   propertyr   rB   r   rD   rM   rQ   rR   rX   nnModuler   rW   
contextlibcontextmanagerr   rk   rT   rl   r   ro   rp   rq   rr   rs   classmethodry   __classcell__r    r    r?   r!   r&   3   sh                  \	r&   )r\   r   c                    s,   ddl m  t fdd| jd d D S )Nr   FlatParameterc                 3   s   | ]}t | V  qd S r   )rU   )r[   paramr   r    r!   r]   9  r^   z-_optimizer_has_flat_params.<locals>.<genexpr>params)Z(fairscale.nn.misc.flatten_params_wrapperr   r`   Zparam_groups)r\   r    r   r!   rZ   6  s    rZ   )0r   loggingtypingr   r   r   r   r   r;   Ztorch.optimr   Zpytorch_lightningrV   Zlightning_fabric.pluginsr   r	   Z$lightning_fabric.utilities.optimizerr
   Z pytorch_lightning.overrides.baser   Z%pytorch_lightning.overrides.fairscaler   Z#pytorch_lightning.plugins.precisionr   Z pytorch_lightning.strategies.ddpr   Z pytorch_lightning.trainer.statesr   Z&pytorch_lightning.utilities.exceptionsr   Z)pytorch_lightning.utilities.model_helpersr   Z%pytorch_lightning.utilities.rank_zeror   Z!pytorch_lightning.utilities.typesr   Zfairscale.nnr   r   Zfairscale.nn.data_parallelr   	getLoggerr"   rf   r   r&   rz   rZ   r    r    r    r!   <module>   s4   
  