a
    dY>                     @   s  d dl Z d dlZd dlmZmZmZ d dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZmZm Z m!Z! d dl"m#Z# d dl$m%Z%m&Z&m'Z' d dl(m)Z) e *e+Z,G dd ded Z-dee dddZ.dS )    N)AnyOptionalType)CUDAAccelerator)Loop)TrainingEpochLoop)_OUTPUTS_TYPE)_is_max_limit_reached_set_sampler_epoch)_ResultCollection)Progress)CombinedLoaderTensorRunningAccum)MisconfigurationException)AbstractDataFetcherDataFetcherDataLoaderIterDataFetcherInterBatchParallelDataFetcher)is_overridden)rank_zero_debugrank_zero_inforank_zero_warn)is_param_in_hook_signaturec                       s  e Zd ZdZd<ee ee dd fddZeeddd	Zeedd
dZ	eedddZ
eee dddZejee ddddZeedddZejeddddZeedddZejjeddddZeedddZeedddZejeddddZeeddd Zeedd!d"Zeedd#d$Zeedd%d&Zedd'd(d)Zddd*d+Zddd,d-Zddd.d/Zddd0d1Zddd2d3Z ddd4d5Z!ddd6d7Z"edd8d9Z#edd:d;Z$  Z%S )=FitLoopzThis Loop iterates over the epochs to run the training.

    Args:
        min_epochs: The minimum number of epochs
        max_epochs: The maximum number of epochs, can be set -1 to turn this limit off
    r   N)
min_epochs
max_epochsreturnc                    s^   t    t|tr,|dk r,td| d|| _|| _t | _t	 | _
d| _g | _d | _d S )NzA`max_epochs` must be a non-negative integer or -1. You passed in .T)super__init__
isinstanceintr   r   r   r   
epoch_loopr   epoch_progress_is_fresh_start_epoch_outputs_data_fetcher)selfr   r   	__class__ i/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.pyr    1   s    

zFitLoop.__init__)r   c                 C   s   | j jS )z/Returns the current batch index (across epochs))r#   total_batch_idxr(   r+   r+   r,   r-   F   s    zFitLoop.total_batch_idxc                 C   s   | j jS )z3Returns the current batch index (within this epoch))r#   	batch_idxr.   r+   r+   r,   r/   K   s    zFitLoop.batch_idxc                 C   s
   | j jjS )zQReturns the index of the current batch split (within the current batch) for bptt.)r#   
batch_loop	split_idxr.   r+   r+   r,   r1   P   s    zFitLoop.split_idxc                 C   s   | j jS )z+Returns the minimum number of steps to run.r#   	min_stepsr.   r+   r+   r,   r3   U   s    zFitLoop.min_steps)valuer   c                 C   s   || j _dS )z9Sets the minimum number of steps (forwards to epoch_loop)Nr2   r(   r4   r+   r+   r,   r3   [   s    c                 C   s   | j jS )z+Returns the maximum number of steps to run.)r#   	max_stepsr.   r+   r+   r,   r6   a   s    zFitLoop.max_stepsc                 C   s$   |dk rt d| d|| j_dS )z9Sets the maximum number of steps (forwards to epoch_loop)r   zQ`max_steps` must be a non-negative integer or -1 (infinite steps). You passed in r   N)r   r#   r6   r5   r+   r+   r,   r6   f   s
    
c                 C   s
   | j jjS )zReturns the running loss.)r#   r0   running_lossr.   r+   r+   r,   r7   p   s    zFitLoop.running_loss)
restartingr   c                    sL    j jj j jjf}t fdd|D }|r2|p8  }tj | d S )Nc                 3   s   | ]}| j jjkV  qd S N)r$   current	processed).0vr.   r+   r,   	<genexpr>y       z%FitLoop.restarting.<locals>.<genexpr>)	r$   r:   readystartedany_iteration_based_trainingr   r8   fset)r(   r8   valuesZepoch_unfinishedr+   r.   r,   r8   u   s    zFitLoop.restartingc                 C   s0   | j jtdk}tdddk}|s(|r,dS dS )NinfPL_INTER_BATCH_PARALLELISM01   r   )trainernum_training_batchesfloatosgetenv)r(   Z
is_unsizedZinter_batch_parallelismr+   r+   r,   prefetch_batches}   s    zFitLoop.prefetch_batchesc                 C   s   | j jjjS )MDetermines whether the loop will skip backward during automatic optimization.r#   r0   Zoptimizer_loop_skip_backwardr.   r+   r+   r,   rS      s    zFitLoop._skip_backwardc                 C   s   || j jj_dS )rQ   NrR   r5   r+   r+   r,   rS      s    c                 C   s.   | j jr| jjS | j jr"| jjjS tdd S )NzD`FitLoop._results` property isn't defined. Accessed outside of scope)rK   Ztrainingr#   _resultsZ
validatingval_loopRuntimeErrorr.   r+   r+   r,   rT      s
    
zFitLoop._resultsc                 C   s:   | j r| jjj| j knd}| jr.| jj| jknd}|o8|S )NT)r   r$   r:   r;   r3   r#   global_step)r(   Zmet_min_epochsZmet_min_stepsr+   r+   r,   _can_stop_early   s    zFitLoop._can_stop_earlyc                 C   s   | j jdkrtd dS t| jj| j}|rBtd| jd dS t| jt	sRJ t| j
jj| j}|r| j
jj| j
j_td| jd dS | j jr| jrtd dS dS )	z!Evaluates when to leave the loop.r   z+`Trainer.fit` stopped: No training batches.Tz"`Trainer.fit` stopped: `max_steps=z
` reached.z#`Trainer.fit` stopped: `max_epochs=z5`Trainer.fit` stopped: `trainer.should_stop` was set.F)rK   rL   r   r	   r#   rW   r6   r!   r   r"   r$   r:   r;   	completedZshould_stoprX   r   )r(   Z
stop_stepsZstop_epochsr+   r+   r,   done   s"    zFitLoop.donec                 C   s   | j p| jjdkS )zXWhether we should skip the training and immediately return from the call to :meth:`run`.r   )rZ   rK   Zlimit_train_batchesr.   r+   r+   r,   skip   s    zFitLoop.skip)r#   r   c                 C   s
   || _ dS )z0Connects a training epoch loop to this fit loop.N)r#   )r(   r#   r+   r+   r,   connect   s    zFitLoop.connectc                 C   s   | j r| j  dS )z'Resets the internal state of this loop.N)r8   r$   Zreset_on_restartr.   r+   r+   r,   reset   s    zFitLoop.resetc                 C   s   |   s| jjj| jj_| j| jj | j	 r>| jj
  t| j}|| jd| _d| _| jj| jjjd | jd | jd | jd dS )z"Calls the ``on_train_start`` hook.)rP   T)deviceZon_train_startN)rC   r$   r:   r;   rY   rK   reset_train_dataloaderlightning_moduler#   Z_should_check_val_epochrU   Z_reload_evaluation_dataloaders_select_data_fetcherrP   r'   r%   rT   tor^   _call_callback_hooks_call_lightning_module_hook_call_strategy_hook)r(   Zdata_fetcher_clsr+   r+   r,   on_run_start   s    

zFitLoop.on_run_startc                 C   s   | j j}| js8| j jjr8t| jj d | j 	| d| _g | _
| j jdurvt| j jtsbJ t| j j| jjj | j j| j | j j | jjjj| j jd | j  | j j  | j d | j d | j  dS )zPPrepares the dataloader for training and calls the hook ``on_train_epoch_start``z: resetting train dataloaderFN)Zwindow_lengthon_train_epoch_start)rK   r`   r%   Z_data_connectorZ_should_reload_train_dllogdetailr*   __name__r_   r&   train_dataloaderr!   r   r
   r$   r:   r;   Zaccumulation_schedulerrg   r#   r0   Zaccumulated_lossr]   Zaccumulate_grad_batchesZincrement_ready_logger_connectorZon_epoch_startrc   rd   Zincrement_started)r(   modelr+   r+   r,   on_advance_start   s     
zFitLoop.on_advance_startc                    s   t  jj d  jjdus$J  jj}ttd fdd} jdusNJ  jj||d  jj	
d   j j _W d   n1 s0    Y  dS )zRuns one whole epoch.z: advancing loopN)batchr   c                    s(    j jj| dd}  j jd| dd} | S )Nr   )Zdataloader_idxbatch_to_device)rK   r`   Z_on_before_batch_transferre   )ro   r.   r+   r,   rp     s    z(FitLoop.advance.<locals>.batch_to_device)rp   Zrun_training_epoch)rh   ri   r*   rj   rK   rk   r   r'   setupZprofilerZprofiler#   runr&   )r(   Z
dataloaderrp   r+   r.   r,   advance   s    zFitLoop.advancec                 C   s   | j j  | j j}td|r^| jr^| jj| j|t| j j	d}| j 
d|}|d ur^tdg | _| j  | j d | j 
d | j j  | j r| jjd| j d | j jd8  _| j j  | j jd7  _| j  | j   d S )NZtraining_epoch_end)r`   Znum_optimizerszi`training_epoch_end` expects a return of None. HINT: remove the return statement in `training_epoch_end`.Zon_train_epoch_endepoch)Zupdate_plateau_schedulersrJ   )rK   rl   Zepoch_end_reachedr`   r   r&   r#   Z#_prepare_outputs_training_epoch_endlenZ
optimizersrd   r   r$   Zincrement_processedrc   Zon_epoch_endZ_num_ready_batches_reachedZupdate_lr_schedulersr8   Z_batches_that_steppedZupdate_train_epoch_metricsZincrement_completedZ_exit_gracefully_on_signal)r(   rm   Zepoch_end_outputsr+   r+   r,   on_advance_end  s2    



zFitLoop.on_advance_endc                 C   s<   t | jj d | jd | jd | jd dS )z Calls the ``on_train_end`` hook.z: train run endedZon_train_endN)rh   ri   r*   rj   rK   rc   rd   re   r.   r+   r+   r,   
on_run_end>  s    zFitLoop.on_run_endc                 C   s(   | j d ur| j   d | _ | j  d S r9   )r'   teardownr#   r.   r+   r+   r,   rx   G  s    

zFitLoop.teardownc                 C   s
   | j  S )z,Whether the gradients should be accumulated.)r#   _should_accumulater.   r+   r+   r,   ry   M  s    zFitLoop._should_accumulatec                 C   s   | j jdkS )Nr   )rK   r6   r.   r+   r+   r,   rC   Q  s    z!FitLoop._iteration_based_training)r   N)&rj   
__module____qualname____doc__r   r"   r    propertyr-   r/   r1   r3   setterr6   r   r7   r   r8   boolrP   rS   r   rT   rX   rZ   r[   r   r\   r]   rf   rn   rs   rv   rw   rx   ry   rC   __classcell__r+   r+   r)   r,   r   )   sd   	  	 1	r   z
pl.Trainer)rK   r   c                 C   sR   t | jd}t|dddr&td tS tdddkrNt| jt	sJt
d	tS tS )
NZtraining_stepZdataloader_iterT)explicitzFound `dataloader_iter` argument in the `training_step`. Note that the support for this signature is experimental and the behavior is subject to change.rG   rH   rI   zAInter batch parallelism is available only when using Nvidia GPUs.)getattrr`   r   r   r   rN   rO   r!   Zacceleratorr   r   r   r   )rK   Ztraining_step_fxr+   r+   r,   ra   U  s    ra   )/loggingrN   typingr   r   r   Zpytorch_lightningplZpytorch_lightning.acceleratorsr   Zpytorch_lightning.loopsr   Zpytorch_lightning.loops.epochr   Z1pytorch_lightning.loops.epoch.training_epoch_loopr   Z_EPOCH_OUTPUTS_TYPEZ!pytorch_lightning.loops.utilitiesr	   r
   Z<pytorch_lightning.trainer.connectors.logger_connector.resultr   Z"pytorch_lightning.trainer.progressr   Z$pytorch_lightning.trainer.supportersr   r   Z&pytorch_lightning.utilities.exceptionsr   Z$pytorch_lightning.utilities.fetchingr   r   r   r   Z)pytorch_lightning.utilities.model_helpersr   Z%pytorch_lightning.utilities.rank_zeror   r   r   Z+pytorch_lightning.utilities.signature_utilsr   	getLoggerrj   rh   r   ra   r+   r+   r+   r,   <module>   s*   
  .