a
    de                     @   s  d dl Z d dlmZmZmZmZmZmZmZm	Z	 d dl
Z
d dl
mZ d dlmZ d dlmZmZmZ d dlZd dlmZ d dlmZmZ G dd	 d	eZe	eeeeef e	ee e jf d
ddZeeddddZG dd deZ G dd de Z!G dd dZ"dS )    N)AnycastIterableIteratorListOptionalSizedUnion)Tensor)DistributedDataParallel)BatchSamplerDistributedSamplerSampler)_DatasetSamplerWrapper)_LightningModuleWrapperBase$_LightningPrecisionModuleWrapperBasec                       s@   e Zd Zdeedef  eedef  dd fddZ  ZS )LightningDistributedModuleNzpl.LightningModule)forward_module	pl_modulereturnc                    s"   |  || t j|p|d d S )N)r   )Z_validate_init_argumentssuper__init__)selfr   r   	__class__ p/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/overrides/distributed.pyr      s    z#LightningDistributedModule.__init__)NN)__name__
__module____qualname__r   r	   r   r   __classcell__r   r   r   r   r      s     r   )objr   c                 C   sP   t | tr| gS t | ttfr.tjtt|  S t | trLtjtt| 	  S g S )z?Recursively find all tensors contained in the specified object.)

isinstancer
   listtuple	itertoolschainmap_find_tensorsdictvalues)r!   r   r   r   r(   %   s    

r(   )modeloutputr   c                 C   sZ   t  rP| jrPd| _| jr&tt|ng }tt jj	j
| j}|  || nd| _d S )NTF)torchZis_grad_enabledZrequire_backward_grad_syncZrequire_forward_param_syncZfind_unused_parametersr#   r(   r   Z_CZ_distributed_c10dZReducerreducerZ_rebuild_bucketsprepare_for_backward)r+   r,   argsr.   r   r   r   r/   5   s    r/   c                       s@   e Zd ZdZeedd fddZeee  dddZ	  Z
S )	UnrepeatedDistributedSampleraF  A fork of the PyTorch DistributedSampler that doesn't repeat data, instead allowing the number of batches
    per process to be off-by-one from each other. This makes this sampler usable for predictions (it's
    deterministic and doesn't require shuffling). It is potentially unsafe to use this sampler for training,
    because during training the DistributedDataParallel syncs buffers on each forward pass, so it could freeze if
    one of the processes runs one fewer batch. During prediction, buffers are only synced on the first batch, so
    this is safe to use as long as each process runs at least one batch. We verify this in an assert.

    Taken from https://github.com/jpuigcerver/PyLaia/blob/v1.0.0/laia/data/unpadded_distributed_sampler.py
    and https://github.com/pytorch/pytorch/issues/25162#issuecomment-634146002
    N)r0   kwargsr   c                    sj   t  j|i | t| jts&tdtt| jt| j| j	| _
t| j| _| j
dksf| jdksfJ d S )N6The given dataset must implement the `__len__` method.   r   )r   r   r"   datasetr   	TypeErrorlenrangeranknum_replicasnum_samples
total_size)r   r0   r2   r   r   r   r   R   s    z%UnrepeatedDistributedSampler.__init__r   c                 C   s   t | jtstd| jrHt }|| j tj	t
| j|d }nttt
| j}t
|| jkslJ || j| j| j }t
|| jksJ t|S )Nr3   )	generator)r"   r5   r   r6   shuffler-   	GeneratorZmanual_seedepochZrandpermr7   tolistr#   r8   r<   r9   r:   r;   iter)r   gindicesr   r   r   __iter__\   s    z%UnrepeatedDistributedSampler.__iter__)r   r   r   __doc__r   r   r   r   intrF   r    r   r   r   r   r1   F   s   
r1   c                       sF   e Zd ZdZeeef eedd fddZe	d fddZ
  ZS )	#UnrepeatedDistributedSamplerWrapperz_Equivalent class to ``DistributedSamplerWrapper`` but for the ``UnrepeatedDistributedSampler``.N)samplerr0   r2   r   c                    s"   t  jt|g|R i | d S N)r   r   r   )r   rJ   r0   r2   r   r   r   r   s   s    z,UnrepeatedDistributedSamplerWrapper.__init__r=   c                    s"    j    fddt  D S )Nc                 3   s   | ]} j | V  qd S rK   )r5   ).0indexr   r   r   	<genexpr>x       z?UnrepeatedDistributedSamplerWrapper.__iter__.<locals>.<genexpr>)r5   resetr   rF   rN   r   rN   r   rF   v   s    
z,UnrepeatedDistributedSamplerWrapper.__iter__)r   r   r   rG   r	   r   r   r   r   r   rF   r    r   r   r   r   rI   p   s    rI   c                   @   s   e Zd ZdZeddddZeee  dddZ	edd	d
Z
eedddZeedddZeeeef dddZdS )IndexBatchSamplerWrapperz\This class is used to wrap a :class:`torch.utils.data.BatchSampler` and capture its indices.N)rJ   r   c                 C   s   g | _ || _d S rK   )seen_batch_indices_sampler)r   rJ   r   r   r   r   ~   s    z!IndexBatchSamplerWrapper.__init__r=   c                 c   s(   g | _ | jD ]}| j | |V  qd S rK   )rS   rT   append)r   batchr   r   r   rF      s    
z!IndexBatchSamplerWrapper.__iter__c                 C   s
   t | jS rK   )r7   rT   rN   r   r   r   __len__   s    z IndexBatchSamplerWrapper.__len__c                 C   s   | j jS rK   )rT   	drop_lastrN   r   r   r   rX      s    z"IndexBatchSamplerWrapper.drop_lastc                 C   s   | j jS rK   )rT   
batch_sizerN   r   r   r   rY      s    z#IndexBatchSamplerWrapper.batch_sizec                 C   s   | j jS rK   )rT   rJ   rN   r   r   r   rJ      s    z IndexBatchSamplerWrapper.sampler)r   r   r   rG   r   r   r   r   rH   rF   rW   propertyboolrX   rY   r	   r   r   rJ   r   r   r   r   rR   {   s   rR   )#r%   typingr   r   r   r   r   r   r   r	   r-   r
   Ztorch.nn.parallelr   Ztorch.utils.datar   r   r   Zpytorch_lightningplZ&lightning_fabric.utilities.distributedr   Z pytorch_lightning.overrides.baser   r   r   r#   r$   r)   r&   r(   r/   r1   rI   rR   r   r   r   r   <module>   s    (*