a
    d+                     @   s   d dl mZmZmZ d dlmZ d dlmZ d dlm	Z	 d dl
Zd dlmZ d dlmZ d dlmZmZmZ d dlmZ e oed	Zerd d
lmZ neZG dd deZejddddZee	 eeed dddZdS )    )ListOptionalUnionN)package_available)	Optimizer)	Precision)_IS_WINDOWS)_LightningModuleWrapperBase$_LightningPrecisionModuleWrapperBaseunwrap_lightning_module)rank_zero_deprecationZ	fairscale)OSSc                       s@   e Zd Zdeedef  eedef  dd fddZ  ZS )LightningShardedDataParallelNpl.LightningModule)forward_module	pl_modulereturnc                    s*   t d | || t j|p |d d S )Na  PyTorch Lightning's sharded implementation using FairScale has been deprecated in v1.9.0 and will be removed in v2.0.0. You can try using the `Trainer(strategy='fsdp_native')` instead. The difference is that native FSDP uses PyTorch's implementation and the current strategy uses FairScale's implementation (which was upstreamed to PyTorch). After removal, `strategy='fsdp'` will use the native version by default.)r   )r   Z_validate_init_argumentssuper__init__)selfr   r   	__class__ n/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/overrides/fairscale.pyr   '   s
    z%LightningShardedDataParallel.__init__)NN)__name__
__module____qualname__r   r   r
   r   __classcell__r   r   r   r   r   &   s     r   r   )wrapped_modelr   c                 C   s4   ddl m} td | }t||r(|j}t|ddS )Nr   )ShardedDataParallelzThe function `unwrap_lightning_module_sharded` is deprecated in v1.8.0 and will be removed in v2.0.0. Access the `LightningModule` directly through the strategy attribute `Strategy.lightning_module`.T)Z_suppress_warning)Z&fairscale.nn.data_parallel.sharded_ddpr   r   
isinstancemoduler   )r   r   modelr   r   r   unwrap_lightning_module_sharded7   s    
r#   r   )
optimizers	precision	num_nodesr   c                 C   sb   t | D ]T\}}t|tst|}tf |j|d|j}|jdk}|oN|dk|_|| |< ~q| S )N)paramsZoptimZ16   )	enumerater    r   typeZparam_groupsdefaultsr%   Zbroadcast_fp16)r$   r%   r&   xZ	optimizerZoptim_classZzero_optimizerZis_fp16r   r   r   _reinit_optimizers_with_ossE   s    

r-   )typingr   r   r   Ztorch.nnnnZ lightning_utilities.core.importsr   Ztorch.optimr   Zpytorch_lightningplZlightning_fabric.pluginsr   Z"lightning_fabric.utilities.importsr   Z pytorch_lightning.overrides.baser	   r
   r   Z%pytorch_lightning.utilities.rank_zeror   Z_FAIRSCALE_AVAILABLEZfairscale.optimr   objectr   Moduler#   intr-   r   r   r   r   <module>   s   