a
    d                      @   s6  d dl Z d dlmZmZmZmZmZmZmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlm Z  d dl!m"Z" e
dee#ef dZ$G dd dZ%G dd de Z&G dd dZ'eee#ef ee#ef dddZ(dS )    N)
AnyCallableDict	GeneratorIteratorMappingOptionaloverloadTypeVarUnion)apply_to_collection)nn)Tensor)_IncompatibleKeys)	Optimizer)
DataLoader)	Precision)_convert_fp_tensor)Strategy)move_data_to_device)_DeviceDtypeModuleMixin)OptimizableT_destination)boundc                   @   sl   e Zd ZeeddddZeedddZee	e
f ddd	Zdee ed
ddZeddddZdS )_FabricOptimizerN)	optimizerstrategyreturnc                 C   sD   dd |j  D | _ td|jj | j|jfi | _|| _|| _dS )a  FabricOptimizer is a thin wrapper around the :class:`~torch.optim.Optimizer` that delegates the
        optimizer step calls to the strategy plugin.

        The underlying wrapped optimizer object can be accessed via the property :attr:`optimizer`.

        Args:
            optimizer: The optimizer to wrap
            strategy: Reference to the strategy for handling the optimizer step
        c                 S   s   i | ]\}}|d vr||qS ))
state_dictstep	zero_grad__del__ ).0kvr"   r"   b/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/lightning_fabric/wrappers.py
<dictcomp>0   s   z-_FabricOptimizer.__init__.<locals>.<dictcomp>ZFabricN)__dict__itemstype	__class____name__
_optimizer	_strategy)selfr   r   r"   r"   r&   __init__$   s    z_FabricOptimizer.__init__r   c                 C   s   | j S N)r-   r/   r"   r"   r&   r   7   s    z_FabricOptimizer.optimizerc                 C   s   | j | jS r2   )r.   Zget_optimizer_stater   r3   r"   r"   r&   r   ;   s    z_FabricOptimizer.state_dict)closurer   c                 C   sT   |d urt |dni }t| jdr:t| jjtr:| jj}n| j}| jj|fi |S )N)r4   model)dicthasattrr.   
isinstancer5   r   r   Zoptimizer_step)r/   r4   kwargsr   r"   r"   r&   r   >   s    
z_FabricOptimizer.step)r9   r   c                 K   s"   t | j|}| jjf i | d S r2   )#_process_optimizer_zero_grad_kwargsr   r    )r/   r9   r"   r"   r&   r    J   s    z_FabricOptimizer.zero_grad)N)r,   
__module____qualname__r   r   r0   propertyr   r   strr   r   r   r   r   r   r    r"   r"   r"   r&   r   #   s   r   c                       s   e Zd Zdejeeej dd fddZeejdddZ	e
e
e
dd	d
ZedddeeeedddZedddeeeee
f dddZdee eeeeee
f  dddZdeee
f eedddZe
e
d fddZ  ZS )_FabricModuleN)forward_module	precisionoriginal_moduler   c                    s$   t    || _|p|| _|| _dS )a  The FabricModule is a thin wrapper around the :class:`torch.nn.Module` and handles precision / autocast
        automatically for the forward pass.

        The underlying wrapped module can be accessed via the property :attr:`module`.

        Args:
            forward_module: The module to wrap the ``forward`` method on.
            precision: Reference to the precision plugin for handling precision context
            original_module: The original, unmodified module as passed into the
                :meth:`lightning_fabric.fabric.Fabric.setup` method. This is needed when attribute lookup
                on this wrapper should pass through to the original module.
        N)superr0   _forward_module_original_module
_precision)r/   r@   rA   rB   r+   r"   r&   r0   P   s    

z_FabricModule.__init__r1   c                 C   s   | j p
| jS r2   )rE   rD   r3   r"   r"   r&   moduled   s    z_FabricModule.module)argsr9   r   c                 O   sl   t ||g| jjtd\}}| j   | j|i |}W d   n1 sJ0    Y  t |ttt d}|S )zqCasts all inputs to the right precision and handles autocast for operations in the module forward
        method.)functiondtypeN)rJ   rK   Zdst_type)	r   rF   Zconvert_inputr   Zforward_contextrD   r   torchZget_default_dtype)r/   rI   r9   outputr"   r"   r&   forwardh   s    .z_FabricModule.forward.)prefix	keep_vars)destinationrO   rP   r   c                C   s   d S r2   r"   r/   rQ   rO   rP   r"   r"   r&   r   u   s    z_FabricModule.state_dict)rO   rP   r   c                C   s   d S r2   r"   )r/   rO   rP   r"   r"   r&   r   y   s     Fc                 C   s   | j j|||dS )N)rQ   rO   rP   )rE   r   rR   r"   r"   r&   r   }   s
    T)r   strictr   c                 C   s   | j j||dS )N)r   rT   )rE   load_state_dict)r/   r   rT   r"   r"   r&   rU      s    z_FabricModule.load_state_dict)itemr   c                    s<   zt  |W S  ty6   t  d}t|| Y S 0 d S )NrE   )rC   __getattr__AttributeErrorgetattr)r/   rV   rB   rG   r"   r&   rW      s
    z_FabricModule.__getattr__)N)NrS   F)T)r,   r;   r<   r   Moduler   r   r0   r=   rH   r   rN   r	   r   r>   boolr   r   r   r   rU   rW   __classcell__r"   r"   rG   r&   r?   O   s&    $ 
	r?   c                   @   sp   e Zd Zdeeej ddddZeeej dddZe	ddd	Z
eee eeddf f dd
dZdS )_FabricDataLoaderN)
dataloaderdevicer   c                 C   s$   | j |j  || _|| _d| _dS )a  The FabricDataLoader is a wrapper for the :class:`~torch.utils.data.DataLoader`. It moves the data to
        the device automatically if the device is specified.

        Args:
            dataloader: The dataloader to wrap
            device: The device to which the data should be moved. By default the device is `None` and no data
                transfers will be made (identical behavior as :class:`~torch.utils.data.DataLoader`).
        r   N)r(   update_dataloader_device_num_iter_calls)r/   r^   r_   r"   r"   r&   r0      s    	z_FabricDataLoader.__init__r1   c                 C   s   | j S r2   )rb   r3   r"   r"   r&   r_      s    z_FabricDataLoader.devicec                 C   s
   t | jS r2   )lenra   r3   r"   r"   r&   __len__   s    z_FabricDataLoader.__len__c                 c   sj   t | jjdr| jj| j |  jd7  _t| j}| jd u rN|E d H  d S |D ]}t|| jV  qRd S )N	set_epoch   )r7   ra   Zsamplerrf   rc   iterrb   r   )r/   iteratorrV   r"   r"   r&   __iter__   s    


z_FabricDataLoader.__iter__)N)r,   r;   r<   r   r   rL   r_   r0   r=   intre   r   r   r   r   rj   r"   r"   r"   r&   r]      s
   r]   )r   r9   r   c                 C   s,   d|v r(dt | jjv r(|d|d< |S )NZset_to_noneZset_grads_to_None)inspect	signaturer    
parameterspop)r   r9   r"   r"   r&   r:      s    r:   ))rl   typingr   r   r   r   r   r   r   r	   r
   r   rL   Z#lightning_utilities.core.apply_funcr   r   r   Ztorch.nn.modules.moduler   Ztorch.optimr   Ztorch.utils.datar   Zlightning_fabric.pluginsr   Z(lightning_fabric.plugins.precision.utilsr   Zlightning_fabric.strategiesr   Zlightning_fabric.utilitiesr   Z-lightning_fabric.utilities.device_dtype_mixinr   Z lightning_fabric.utilities.typesr   r>   r   r   r?   r]   r:   r"   r"   r"   r&   <module>   s&   0,E'