a
    þdO  ã                   @   sÈ  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlZ	d dl
Z
d dl
mZ d dlmZ zd dlZW n eyŽ   dZY n0 zd dlm  mZ W n ey¼   dZY n0 zd dlm
Z W n eyä   dZY n0 d dlmZmZmZmZ d dlmZ d dlmZmZmZ d dl m!Z! d d	l"m#Z# d d
l$m%Z%m&Z&m'Z' d dl(m)Z)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/ dZ0ddd„Z1dd„ Z2e3e4dœdd„Z5dd„ Z6dd„ Z7e8dkrÄe6ej9dd… ƒ dS )é    N)Údatetime)Úoptim)Ú
GradScaler)Úcreate_model_and_transformsÚtrace_modelÚget_tokenizerÚcreate_loss)Úget_data)Ú	is_masterÚinit_distributed_deviceÚbroadcast_object)Úsetup_logging)Ú
parse_args)Ú	cosine_lrÚconst_lrÚconst_lr_cooldown)Útrain_one_epochÚevaluate)Úpt_loadÚcheck_existsÚstart_sync_processÚremote_synczepoch_latest.pté*   c                 C   s0   t  | | ¡ tj | | ¡ t | | ¡ d S ©N)ÚtorchZmanual_seedÚnpÚrandomÚseed)r   Úrank© r   úV/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/training/main.pyÚrandom_seed+   s    r!   c                 C   s   dd„ t  d|  ¡ ¡D ƒS )z9See http://www.codinghorror.com/blog/archives/001018.htmlc                 S   s    g | ]}|  ¡ rt|ƒn|‘qS r   )ÚisdigitÚint)Ú.0Úsr   r   r    Ú
<listcomp>3   ó    znatural_key.<locals>.<listcomp>z(\d+))ÚreÚsplitÚlower)Zstring_r   r   r    Únatural_key1   s    r+   )ÚpathÚremotec                    s   |rbt jdddˆ d gt jt jd}t|ƒ |jdkr:d S ‡ fdd„|j ¡  d	¡d d
… D ƒ}ntjˆ d dd}|rŒt	|t
d}|d
 S d S )NZawsÚs3Zlsú/)ÚstdoutÚstderré   c                    s$   g | ]}t j ˆ | d ¡d ¡‘qS )ú éÿÿÿÿ)Úosr,   Újoinr)   )r$   Úx©r,   r   r    r&   =   r'   z)get_latest_checkpoint.<locals>.<listcomp>Ú
r4   z**/*.ptT)Ú	recursive)Úkey)Ú
subprocessÚrunÚPIPEÚprintÚ
returncoder0   Údecoder)   ÚglobÚsortedr+   )r,   r-   ÚresultÚcheckpointsr   r8   r    Úget_latest_checkpoint6   s     
(rF   c           -         sJ  t | ƒ} tj ¡ r2dtjjj_dtjj_dtjj_	t
| ƒ}| jd u r®| j dd¡}t ¡  d¡}| jrpt| |ƒ}d |d|› d| j› d| j› d	| j› d
| j› g¡| _| jdk}tj | j| j¡}d | _t| | jdr8tj|dd | jrd| j › nd}tj ||¡| _tj !| j¡r8|s8t"dƒ dS | j#rFt$j%nt$j&| _'t(| j| j'ƒ d| j)v ppd| j)v | _*d| j)v pˆd| j)v | _+tj |d¡| _,t| ƒrì| j+r¼tj |d¡nd| _-| j-| j,fD ]}|rÎtj|dd qÎnd| _-|rØd }| j,}	| j.d urNtj | j.| jd¡}	| j/r6t"dƒ dS | j0dkrNt"dƒ dS t| ƒrÀ| j/r‚tj |	t1¡}tj !|¡s”d }nt2|	| j.d ud}|r®t$ 3d|› d¡ nt$ 3d|	› d¡ | jrÒt| |ƒ}|| _| j4rèt4| ƒ d }
t| ƒr‚| j.d ur‚t.tj | j| j¡tj | j.| j¡| j0ƒ}|r>t$ 3d¡ nt$ 3d¡ dS t5| j6tj | j| j¡tj | j.| j¡| j0ƒ}
|
 7¡  | jd kr˜t$ 8d!¡ | j9rÎt$ 3d"| j:› d#| j › d$| j;› d%| j<› d	¡ nJ| jrt$ 3d&| j:› d#| j › d$| j;› d%| j<› d	¡ nt$ 3d'| j:› d¡ d }| j=d uo0| j>d u| _?| j?r`| j@d(ksLJ ‚d)| j A¡ vs`J ‚tB| jCtDtEfƒrŽtF| jCƒd(krŽ| jCd* | _CtG| jHd*ƒ tI| j| jJ| j|| jK| jL| jM| jN| jC| jO| jP| jQ| jRdd+\}}}| j?rtI| j=| j>|| jdd,\}}}| jSd ur^t"d-ƒ d*d lT}d*d.lUmV} t"d/| jS› ƒ tW|jXjY| jSƒ}|||ƒ | Z|¡}tG| jH| j ƒ | j[r„t\|| j|d0}| j]rž|j^| j_| j`d1 | jar¸|jb| jc| jdd2 | jerÈ| f¡  t| ƒr€t$ 3d3¡ t$ 3tg|ƒ› ¡ t$ 3d4¡ tj | j| jd5¡}th|d6ƒZ}titj| ƒƒD ]<}tW| |ƒ}t$ 3d7|› d8|› ¡ | k|› d8|› d9¡ q"W d   ƒ n1 sv0    Y  | jrú| j9sú| jlr¦tjXjm n|¡}i }| jorºd|d:< tjXjpjq|fd;|gi|¤Ž}| j?rútjXjpjq|fd;|gi|¤Ž}d }d }| jrs| jsd<krê| j[r&J d=ƒ‚d>d?„ ‰‡fd@d?„‰tE| t¡ ƒ}‡fdAdB„|D ƒ}‡fdCdB„|D ƒ}tujv|dDdEœ|| jwdEœg| j| jx| jyf| jzdF}| j9rÔt{j||| t¡ dG}t{j}| ~¡ d*dH t{j|d*dH | jdIkræt€ƒ nd }d*}| jd urÜt| jdJdK}dL|v r¸|dL }|dM } | jsXt‚tƒ|  „¡ ƒƒd*  …dN¡rXdOdP„ |  „¡ D ƒ} | †| ¡ |d urz| †|dQ ¡ |d urœdR|v rœ| †|dR ¡ t$ 3dS| j› dT|› dU¡ n$| †|¡ t$ 3dV| j› dT|› dU¡ t‡| ||f|tˆ| jƒdW‰ tFˆ ƒ	sJ dXƒ‚d }!dYˆ v 	rê|d u	rêˆ dY j‰jŠ| j@ | j‹ }"| jŒdZk	rXt|| j| jŽ|"ƒ}!n’| jŒd[k	rxt|| j| jŽ|"ƒ}!nr| jŒd\k	rÎ| jd u	s˜J d]ƒ‚ˆ dY j‰jŠ| j@ | j }#t‘|| j| jŽ|"|#| j’| j“ƒ}!nt$ ”d^| jŒ› d_¡ t•d(ƒ | j
o| j A¡ d`k
ot| ƒ| _–d }$| j–
r>| j+
r>t+d u
s2J daƒ‚t+ —| j-¡}$| j*
ròt| ƒ
ròt*d u
sbJ dbƒ‚t$ #dc¡ ˆ dY j‰j˜| _™| jšd u
r”ˆ dd j‰j˜| _›t*jœ| j| j| j| jžg | jdk
rºdend tj| ƒdf | j#
rÞt*jŸ|ddg t*  |¡ t$ #dh¡ | j¡rt$ 3di¡ t ¢|¡}dYˆ vrL| jSd ur8d*djlUm£}% |%|ƒ t¤|ˆ || |$ƒ d S t¥| ƒ}&t¦|| j‹ƒD ]f}'t| ƒr€t$ 3dk|'› ¡ t§|ˆ |&|'|||!|| |$dl
 |'d( }(t¨‡ fdmdn„doD ƒƒrÌt¤|ˆ |(| |$ƒ | j–r`|(| j| ~¡ | ~¡ dpœ})|d ur| ~¡ |)dR< |(| j‹ks*| j©d*krJ|(| j© d*krJt  |)tj | j,dq|(› dr¡¡ | jªr†tj | j,dq|(d( › dr¡}*tj !|*¡r†t «|*¡ | j/r`tj | j,ds¡}+tj | j,t1¡},t  |)|+¡ t |+|,¡ q`| j*rät| ƒrät* ¬¡  |
d urFt$ 3dt¡ |
 ­¡  t.tj | j| j¡tj | j.| j¡| j0ƒ}|r<t$ 3du¡ n
t$ 3dv¡ d S )wNTFr/   ú-z%Y_%m_%d-%H_%M_%SZmodel_Zlr_Zb_Zj_Zp_Zlatest)Úlocal)Úexist_okzout-zout.logzLError. Experiment already exists. Use --name {} to specify a new experiment.r4   ÚwandbÚallÚtensorboardrE   Ú zFError. Cannot use save-most-recent with remote_sync and resume latest.r.   z<Error. Sync protocol not supported when using resume latest.)r-   z"Found latest resume checkpoint at Ú.z%No latest resume checkpoint found in zremote sync successful.z#Error: remote sync failed. Exiting.Zfp16z‡It is recommended to use AMP mixed-precision instead of FP16. FP16 support needs further verification and tuning, especially for train.zARunning in horovod mode with multiple processes / nodes. Device: z.Process (global: z, local z	), total z=Running in distributed mode with multiple processes. Device: z&Running with a single process. Device r2   Zcocar   )Ú	precisionÚdeviceZjitÚforce_quick_geluÚforce_custom_textÚforce_patch_dropoutÚforce_image_sizeÚpretrained_imageÚ
image_meanÚ	image_stdÚaug_cfgÚoutput_dict)rP   rO   rY   z¶=> using a layer from bitsandbytes.
   this is an experimental feature which requires two extra pip installs
   pip install bitsandbytes triton   please make sure to use triton 2.0.0)Úreplace_linearz => replacing linear layers with )Ú
batch_sizerP   )Zunlocked_groupsZfreeze_bn_stats)Zunlocked_layersZfreeze_layer_normzModel:zParams:z
params.txtÚwz  z: r9   Zstatic_graphZ
device_idsZ	syntheticzCannot train with traced modelc                 S   s*   |j dk p(d| v p(d| v p(d| v p(d| v S )Né   ZbnÚlnZbiasZlogit_scale)Úndim©ÚnÚpr   r   r    Ú<lambda>/  r'   zmain.<locals>.<lambda>c                    s   ˆ | |ƒ S r   r   r`   ©Úexcluder   r    rc   0  r'   c                    s$   g | ]\}}ˆ ||ƒr|j r|‘qS r   ©Zrequires_grad©r$   ra   rb   rd   r   r    r&   3  r'   zmain.<locals>.<listcomp>c                    s$   g | ]\}}ˆ ||ƒr|j r|‘qS r   rf   rg   )Úincluder   r    r&   4  r'   g        )ÚparamsZweight_decay)ÚlrZbetasÚeps)Únamed_parameters)Z	root_rankÚampÚcpu)Zmap_locationÚepochÚ
state_dictÚmodulec                 S   s"   i | ]\}}|t d ƒd… |“qS )zmodule.N)Úlen)r$   ÚkÚvr   r   r    Ú
<dictcomp>O  r'   zmain.<locals>.<dictcomp>Ú	optimizerÚscalerz=> resuming checkpoint 'z	' (epoch ú)z=> loaded checkpoint ')ro   Ú	tokenizerz5At least one train or eval dataset must be specified.ZtrainZcosineÚconstzconst-cooldownzBPlease specify the number of cooldown epochs for this lr schedule.zUnknown scheduler, z7. Available options are: cosine, const, const-cooldown.ÚnonezPlease install tensorboard.zPlease install wandb.zStarting wandb.ÚvalÚauto)ÚprojectÚnameÚidZnotesÚtagsÚresumeÚconfig)ÚlogzFinished loading wandb.zCompiling model...)Ú$convert_int8_model_to_inference_modezStart epoch )Z	tb_writerc                 3   s   | ]}|ˆ v V  qd S r   r   )r$   rt   )Údatar   r    Ú	<genexpr>¥  r'   zmain.<locals>.<genexpr>)r|   zimagenet-valzimagenet-v2)ro   r   rp   rv   Zepoch_z.ptztmp.ptzFinal remote sync.zFinal remote sync successful.zFinal remote sync failed.)®r   r   ZcudaZis_availableÚbackendsÚmatmulZ
allow_tf32ZcudnnZ	benchmarkZdeterministicr   r   ÚmodelÚreplacer   ÚnowÚstrftimeZdistributedr   r6   rj   r[   ÚworkersrO   r‚   r5   r,   ÚlogsZlog_pathr
   Z	log_localÚmakedirsr   Úexistsr?   ÚdebugÚloggingÚDEBUGÚINFOZ	log_levelr   Z	report_torJ   rL   Úcheckpoint_pathZtensorboard_pathr   Zsave_most_recentZremote_sync_protocolÚLATEST_CHECKPOINT_NAMErF   ÚinfoÚcopy_codebaser   Zremote_sync_frequencyÚstartÚwarningZhorovodrP   Z
local_rankZ
world_sizeZdistill_modelZdistill_pretrainedZdistillZ
accum_freqr*   Ú
isinstancerT   ÚtupleÚlistrr   r!   r   r   Z
pretrainedZtorchscriptrQ   rR   rS   rU   rV   rW   rX   Zuse_bnb_linearZbitsandbytesZopen_clip.utilsrZ   ÚgetattrÚnnZtriton_based_modulesÚtoÚtracer   Z
lock_imageZlock_image_towerZlock_image_unlocked_groupsZlock_image_freeze_bn_statsZ	lock_textZlock_text_towerZlock_text_unlocked_layersZlock_text_freeze_layer_normZgrad_checkpointingZset_grad_checkpointingÚstrÚopenrC   ÚvarsÚwriteZuse_bn_syncZSyncBatchNormZconvert_sync_batchnormZddp_static_graphÚparallelZDistributedDataParallelZ
train_dataZdataset_typerl   r   ZAdamWÚwdZbeta1Zbeta2rk   ÚhvdZDistributedOptimizerZbroadcast_parametersrp   Zbroadcast_optimizer_stater   r   ÚnextÚiterÚitemsÚ
startswithZload_state_dictr	   r   Z
dataloaderZnum_batchesZepochsZlr_schedulerr   Zwarmupr   Zepochs_cooldownr   Zlr_cooldown_powerZlr_cooldown_endÚerrorÚexitZ	save_logsZSummaryWriterZnum_samplesZtrain_szZval_dataZval_szÚinitZwandb_project_nameZwandb_notesZwatchÚsaveZtorchcompileÚcompiler…   r   r   Úranger   ÚanyZsave_frequencyZdelete_previous_checkpointÚremoveÚfinishÚ	terminate)-ÚargsrP   Zmodel_name_safeZdate_strZresume_latestZlog_base_pathZlog_filenameÚdirnameZresume_fromr–   Zremote_sync_processrD   Z
dist_modelrŠ   Zpreprocess_trainZpreprocess_valÚ_ZbnbrZ   Zlinear_replacement_clsZparams_fileÚfr   r|   Zddp_argsrv   rw   rl   Zgain_or_bias_paramsZrest_paramsZstart_epochÚ
checkpointÚsdZ	schedulerZtotal_stepsZcooldown_stepsÚwriterr…   Zlossro   Zcompleted_epochZcheckpoint_dictZprevious_checkpointZtmp_save_pathZlatest_save_pathr   )r†   re   rh   r    ÚmainF   sˆ   








ú	
ÿ


ý
üÿ
ÿÿÿÿ
ÿÿÿÿ"òû

þþ



:
þ
ù	
$


ÿ
þÿ"
ù	





ü
ÿÿþ


ýr¿   c                 C   sš   ddl m}m} tj | j| jd¡}tj |¡rDt	d|› dƒ dS t	d|› ƒ tj 
t¡}tdƒD ]}tj |¡}qf||||d	d
dƒd t	dƒ dS )Nr   )ÚcopytreeÚignore_patternsÚcodez$Error. Experiment already exists at z). Use --name to specify a new experiment.r4   zCopying codebase to é   r„   r   rJ   )ÚignorezDone copying code.r2   )ÚshutilrÀ   rÁ   r5   r,   r6   r   r   r‘   r?   ÚrealpathÚ__file__r³   r¹   )r¸   rÀ   rÁ   Znew_code_pathZcurrent_code_pathrº   r   r   r    r™   Ø  s    
ÿr™   Ú__main__r2   )r   r   ):rB   r“   r5   r(   r<   Úsysr   r   Únumpyr   r   r   Ztorch.cuda.ampr   rJ   ÚImportErrorZtorch.utils.tensorboardÚutilsrL   Zhorovod.torchr©   Z	open_clipr   r   r   r   Ztraining.datar	   Ztraining.distributedr
   r   r   Ztraining.loggerr   Ztraining.paramsr   Ztraining.schedulerr   r   r   Ztraining.trainr   r   Ztraining.file_utilsr   r   r   r   r—   r!   r+   r£   ÚboolrF   r¿   r™   Ú__name__Úargvr   r   r   r    Ú<module>   sT   



   
