a
    d                     @   s   d dl mZmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlmZ d dlmZ e ZG d	d
 d
eZdS )    )AnyCallablecastOptionalUnion)Tensor)	Optimizer)LiteralN)	Steppable)PrecisionPlugin)WarningCachec                   @   s   e Zd ZdZded ddddZedee ee	 e
e
dd	d
dZeee	ef ddddZeee	ef ddddZede	eg e
f e
e
dddZdddddZdS )ColossalAIPrecisionPluginzPrecision plugin for ColossalAI integration.

    Args:
        precision: Half precision (16).

    Raises:
        ValueError:
            If precison is not 16.
       16r   N)	precisionreturnc                 C   s0   |dvrt d|dttd t|| _d S )Nr   z*`Trainer(strategy='colossalai', precision=z5)` is not supported. Consider setting `precision=16`.r   )
ValueErrorr   r	   strr   )selfr    r   w/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/colossalai.py__init__'   s
    
z"ColossalAIPrecisionPlugin.__init__zpl.LightningModule)tensormodel	optimizeroptimizer_idxargskwargsr   c                 O   s   |d usJ | | d S N)backward)r   r   r   r   r   r   r   r   r   r   r    /   s    	z"ColossalAIPrecisionPlugin.backward)r   clip_valr   c                 C   s   | d | d S r   )Zclip_grad_normr   r   r!   r   r   r   clip_grad_by_norm;   s    z+ColossalAIPrecisionPlugin.clip_grad_by_normc                 C   s   t dd S )Nz5`clip_grad_by_value` is not supported by `ColossalAI`)NotImplementedErrorr"   r   r   r   clip_grad_by_value>   s    z,ColossalAIPrecisionPlugin.clip_grad_by_value)r   r   r   closurer   r   c                 K   sF   | }|  ||| |d u }t|tjr:|jr:|r:td|  d S )NzaSkipping backward by returning `None` from your `training_step` is not supported by `ColossalAI`.)Z_after_closure
isinstanceplZLightningModuleZautomatic_optimizationr   step)r   r   r   r   r&   r   Zclosure_resultZskipped_backwardr   r   r   optimizer_stepA   s    z(ColossalAIPrecisionPlugin.optimizer_stepz
pl.Trainer)trainerr   c                 C   s&   |j dkrd S td|j d d S )Nz!You set `Trainer(track_grad_norm=zI)' but this is not supported for ColossalAI. The setting will be ignored.)Ztrack_grad_normwarning_cachewarn)r   r+   r   r   r   _track_grad_normR   s
    
z*ColossalAIPrecisionPlugin._track_grad_norm)r   )__name__
__module____qualname____doc__r	   r   r   r   r
   intr   r    r   r   floatr#   r%   r   r*   r/   r   r   r   r   r      s(   


r   )typingr   r   r   r   r   Ztorchr   Ztorch.optimr   Ztyping_extensionsr	   Zpytorch_lightningr(   Z lightning_fabric.utilities.typesr
   Z4pytorch_lightning.plugins.precision.precision_pluginr   Z%pytorch_lightning.utilities.rank_zeror   r-   r   r   r   r   r   <module>   s   