a
    d%                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZ d dlmZm Z m!Z! d	d
 Z"dd Z#dd Z$dd Z%e&dkre'e(e)ej*ej*Z+e%e+ dS )    N)path)build_dataloaderbuild_dataset)EnlargedSampler)CPUPrefetcherCUDAPrefetcher)build_model)AvgTimerMessageLoggercheck_resumeget_env_infoget_root_loggerget_time_strinit_tb_loggerinit_wandb_loggermake_exp_dirsmkdir_and_renamescandir)copy_opt_filedict2strparse_optionsc                 C   s   | d  dd urV| d d  dd urVd| d vrV| d  ddu sNJ dt|  d }| d  drd| d vrtt| d	 d
| d d}|S )NloggerZwandbprojectdebugnameuse_tb_loggerTz+should turn on tensorboard when using wandb	root_path	tb_logger)Zlog_dir)getr   r   ospjoin)optr    r"   V/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/basicsr/train.pyinit_tb_loggers   s    "
r$   c                 C   s|  d g  }}| d   D ]T\}}|dkr|dd}t|}t|| d | d |}t||| d | d || d	 d
}tt|| |d | d   }	t| d d }
t|
|	 }|	dt| d| d|d  d| d  d|	 d| d|
 d q|
dd dkr\t|}t||| d | d d | d	 d
}|	d|d  dt|  || qtd| dq|||||
fS )Ndatasetstraindataset_enlarge_ratio   Z
world_sizeranknum_gpudistZmanual_seed)r*   r+   ZsamplerseedZbatch_size_per_gpuZ
total_iterz.Training statistics:
	Number of train images: z
	Dataset enlarge ratio: z
	Batch size per gpu: z
	World size (gpu number): z!
	Require iter number per epoch: z
	Total epochs: z	; iters: ._r   valz Number of val images/folders in r   z: zDataset phase z is not recognized.)itemsr   r   r   r   mathceillenintinfosplitappend
ValueError)r!   r   train_loaderval_loadersphaseZdataset_optr'   Z	train_settrain_samplerZnum_iter_per_epochtotal_iterstotal_epochsZval_set
val_loaderr"   r"   r#   create_train_val_dataloader   sV    
r@   c                    s   d }| d r|t d| d d}t |rtt|dddd}t|dkrd	d
 |D }t |t|dd}|| d d< n| d dr| d d }|d u rd }n.tj	
  tj| fddd}t| |d  |S )NZauto_resumeZexperimentsr   Ztraining_statesstateF)suffix	recursive	full_pathr   c                 S   s   g | ]}t |d d qS ).stater   )floatr6   ).0vr"   r"   r#   
<listcomp>K       z%load_resume_state.<locals>.<listcomp>z.0frE   r   resume_statec                    s
   |   S )N)cuda)ZstoragelocZ	device_idr"   r#   <lambda>V   rJ   z#load_resume_state.<locals>.<lambda>)Zmap_locationiter)r   r    isdirlistr   r3   maxr   torchrL   Zcurrent_deviceloadr   )r!   Zresume_state_pathZ
state_pathZstatesrK   r"   rN   r#   load_resume_stateD   s"    

rV   c              	   C   sH  t | dd\}}| |d< dtjj_t|}|d u rzt| |d drzd|d vrz|d d	krztt	
|d d
|d  t|j|d d  t	
|d d d|d  dt  d}tdtj|d}|t  |t| t|}t||}|\}}	}
}}t|}|rJ|| |d|d  d|d  d |d }|d }nd	}d	}t|||}|d d d}|d u s|dkrt|}nX|dkrt||}|d| d |d d ddurtd ntd!| d"|d#| d|  t t  }}t }t||d$ D ]}|	| |   |! }|d ur |"  |d$7 }||krhq |j#||d d%d&d' |$| |%| |"  |d$kr|&  ||d d(  d	kr||d)}|'d*|( i |'|) |) d+ |'|*  || ||d d,  d	kr<|d- |+|| |d.d ur||d. d/  d	krt,|
d$krz|-d0 |
D ]}|.||||d. d1  q~|/  |/  |! }q@q t0t1j2t3t | d2}|d3|  |d4 |j+d&d&d5 |d.d ur6|
D ]}|.||||d. d1  q|rD|4  d S )6NT)Zis_trainr   r   r   r   r   r)   r   r   r   Zexperiments_rootlogZtrain_r.   z.logZbasicsr)Zlogger_nameZ	log_levellog_filezResuming training from epoch: epochz, iter: rP   r-   r%   r&   prefetch_modecpurL   zUse z prefetch dataloaderZ
pin_memoryz.Please set pin_memory=True for CUDAPrefetcher.zWrong prefetch_mode z*. Supported ones are: None, 'cuda', 'cpu'.zStart training from epoch: r(   warmup_iter)r\   Z
print_freq)rY   rP   Zlrs)timeZ	data_timeZsave_checkpoint_freqz"Saving models and training states.r/   Zval_freqz=Multiple validation datasets are *only* supported by SRModel.Zsave_img)secondsz End of training. Time consumed: zSave the latest model.)rY   current_iter)5r   rT   backendsZcudnnZ	benchmarkrV   r   r   r   r   r    r   r!   r   r   loggingINFOr5   r   r   r$   r@   r   Zresume_trainingr
   r   r   r8   r	   r^   rangeZ	set_epochresetnextrecordZupdate_learning_rateZ	feed_dataZoptimize_parametersZreset_start_timeupdateZget_current_learning_rateZget_avg_timeZget_current_logsaver3   warningZ
validationstartstrdatetime	timedeltar4   close)r   r!   argsrK   rX   r   r   resultr9   r<   r:   r>   r=   modelZstart_epochr`   Z
msg_loggerrZ   Z
prefetcherZ
data_timerZ
iter_timer
start_timerY   Z
train_dataZlog_varsr?   Zconsumed_timer"   r"   r#   train_pipeline[   s    
&(

 












&

rt   __main__),rm   rb   r1   r^   rT   osr   r   Zbasicsr.datar   r   Zbasicsr.data.data_samplerr   Z basicsr.data.prefetch_dataloaderr   r   Zbasicsr.modelsr   Zbasicsr.utilsr	   r
   r   r   r   r   r   r   r   r   r   Zbasicsr.utils.optionsr   r   r   r$   r@   rV   rt   __name__abspathr    __file__pardirr   r"   r"   r"   r#   <module>   s$   4'z