a
    þd×7  ã                   @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlm	  m
Z d dlmZ zd dlZW n eyx   dZY n0 d dlmZmZmZ ddlmZ ddlmZ ddlmZ G dd	„ d	eƒZd
d„ Zdd„ Zdd„ Zddd„Zddd„Z dd„ Z!dd„ Z"dS )é    N)ÚDistributedDataParallel)Úget_input_dtypeÚCLIPÚCustomTextCLIPé   )Ú	is_master)Úzero_shot_eval)Úget_autocastc                   @   s*   e Zd ZdZdd„ Zdd„ Zd
dd„Zd	S )ÚAverageMeterz1Computes and stores the average and current valuec                 C   s   |   ¡  d S ©N)Úreset©Úself© r   úW/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/training/train.pyÚ__init__   s    zAverageMeter.__init__c                 C   s   d| _ d| _d| _d| _d S )Nr   )ÚvalÚavgÚsumÚcountr   r   r   r   r      s    zAverageMeter.resetr   c                 C   s8   || _ |  j|| 7  _|  j|7  _| j| j | _d S r   )r   r   r   r   )r   r   Únr   r   r   Úupdate#   s    zAverageMeter.updateN)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r   r   r
      s   r
   c                 C   s   | d | d | d dœS )Nr   r   é   ©Úimage_featuresÚtext_featuresÚlogit_scaler   )Ú	model_outr   r   r   Úpostprocess_clip_output)   s    ýr"   c                 C   s   t | dƒr| jS | S d S )NÚmodule)Úhasattrr#   )Úmodelr   r   r   Úunwrap_model0   s    
r&   c                 C   s$   |d ur|  | ¡ ¡  n|  ¡  d S r   )ÚscaleÚbackward)Ú
total_lossÚscalerr   r   r   r(   7   s    r(   c
           2      C   s  t  |j¡}
t|jƒ}t|jƒ}|  ¡  |jr6| ¡  |d  |¡ |d j	}|j
|j }t t |jd d¡¡}|jdkrŒg g i   }}}i }tƒ }tƒ }t ¡ }t|ƒD ]R\}}||j }|| | }|jsÚ||ƒ |\}}|j|
|dd}|j|
dd}| t ¡ | ¡ | ¡  |jdkrì|ƒ   | ||ƒ}|d }|jr”t  ¡  |||ƒ}W d   ƒ n1 sr0    Y  | dd	„ | ¡ D ƒ¡ |f i |¤d
di¤Ž} t|  ¡ ƒ}!|!| d< W d   ƒ n1 sÔ0    Y  t|!|ƒ n¶t  ¡ ˜ |ƒ \ | ||ƒ}| d¡ | ¡ D ].\}"}#|"|v r<||"  |#¡ n
|#g||"< qW d   ƒ n1 s`0    Y  | |¡ | |¡ W d   ƒ n1 s”0    Y  |d |j dkr´q¬| ¡  t|jƒD ]Ú}$||$ }||$ }|ƒ ¢ | ||ƒ}| d¡}i }%| ¡ D ]B\}"}#||" }&t  |&d |$… ||" g |&|$d d …  ¡|%|"< q|f i |%¤|ddœ¤Ž} ~%t|  ¡ ƒ}!|!| d< W d   ƒ n1 sŠ0    Y  t|!|ƒ qÆ|d urf|j r"| !¡  | "|¡ |j#d urìt j$j%j&|  '¡ |j#dd | (¡  | )|¡ W d   ƒ n1 s0    Y  n:|j#d urR| "|¡ t j$j%j&|  '¡ |j#dd | )|¡ | ¡  n.|j#d urŒt j$j%j&|  '¡ |j#dd | )¡  |jdkr°g g i   }}}t  ¡ ( t*| ƒj+ ,dt d¡¡ W d   ƒ n1 sè0    Y  | t ¡ | ¡ t ¡ }|d }'t-|ƒr¬||j. dks4|'|kr¬t/|ƒ}(|'|( |j |j0 })|j}*d|' | }+|  ¡ D ]2\}"}#|"|vr†tƒ ||"< ||"  |# 1¡ |(¡ qj| 1¡ },d 2dd„ | ¡ D ƒ¡}-|j|j3 |j0 |j4 }.|j|j3 |j4 }/t5 6d|› d|)d|› ›d|*› d|+d›d|j7d›d|j7d›d|.d›d |/d›d!|j8d d" d#›d$|,d›d|- ¡ |j4|j4|.|/|,|j8d d" d%œ}0|0 d&d	„ | ¡ D ƒ¡ |0 ¡ D ]V\}1}#d'|1 }1|	d urÀ|	 9|1|#|¡ |j:r˜t:d usÚJ d(ƒ‚t: |1|#d)|i¡ q˜| ;¡  | ;¡  q¬d S )*NÚtrainr   é
   T©ÚdeviceZdtypeÚnon_blocking©r.   r/   r    c                 S   s   i | ]\}}d |› |“qS )Zdist_r   ©Ú.0ÚkÚvr   r   r   Ú
<dictcomp>i   ó    z#train_one_epoch.<locals>.<dictcomp>Úoutput_dictÚlossr   )r    r7   g       @)Z	norm_typeéd   g      Y@ú c                 S   s2   g | ]*\}}|  ¡ › d |jd›d|jd›d‘qS )ú: z#.5gú (ú))Ú
capitalizer   r   )r2   Z	loss_nameZloss_mr   r   r   Ú
<listcomp>Ä   s   ÿz#train_one_epoch.<locals>.<listcomp>zTrain Epoch: ú [ú>ú/r<   z.0fz%)] Data (t): z.3fz Batch (t): z, z#gz/s, z/s/gpu LR: ÚlrZ5fz Logit Scale: )Z	data_timeZ
batch_timeÚsamples_per_secondÚsamples_per_second_per_gpur'   rC   c                 S   s   i | ]\}}||j “qS r   )r   )r2   Únamer   r   r   r   r5   Ü   r6   ztrain/úPlease install wandb.Ústep)<Útorchr.   r	   Ú	precisionr   r+   ZdistillÚevalZ	set_epochÚ
dataloaderZnum_batchesZ
accum_freqÚmathÚceilÚlogÚnum_samplesr
   ÚtimeÚ	enumerateZskip_schedulerÚtor   Z	zero_gradÚno_gradÚitemsr   Úvaluesr(   ÚpopÚappendÚrangeÚcatZhorovodZsynchronizeZunscale_Zgrad_clip_normÚnnÚutilsZclip_grad_norm_Ú
parametersZskip_synchronizerH   r&   r    Zclamp_r   Zlog_every_n_stepsÚlenZ
world_sizeÚitemÚjoinÚ
batch_sizer   ÚloggingÚinfor   Zparam_groupsÚ
add_scalarÚwandbr   )2r%   Údatar8   ÚepochZ	optimizerr*   Z	schedulerZ
dist_modelÚargsÚ	tb_writerr.   ÚautocastÚinput_dtyperL   Znum_batches_per_epochZsample_digitsZaccum_imagesZaccum_textsZaccum_featuresZlosses_mZbatch_time_mZdata_time_mÚendÚiÚbatchZi_accumrH   ÚimagesÚtextsr!   r    Zdist_model_outZlossesr)   Úkeyr   ÚjÚinputsZaccumulatedZbatch_countra   rP   Zsamples_per_epochZpercent_completeZlogit_scale_scalarZloss_logrD   rE   Zlog_datarF   r   r   r   Útrain_one_epoch>   s   






*(



.
*

4(


,



8 

þÿ$ÿþþþýüüÿ
ú
rt   c           $      C   s¾  i }t |ƒs|S t |j¡}|  ¡  t| |||ƒ}| |¡ t|jƒ}t|jƒ}	d|v r´|j	r´||j	 dks|||j
kr´|d j}
d}|
j}d}d}g g  }}t ¡ ö t|
ƒD ]r\}}|\}}|j||	dd}|j|dd}|ƒ ® | ||ƒ}|d }|d }|d	 }| | ¡ ¡ | | ¡ ¡ | ¡ }|| | ¡  }| ¡ }|jd }tj||d
 ¡ }t ||¡t ||¡ d }t|ƒ}W d   ƒ n1 s¢0    Y  ||| 7 }||7 }t |ƒr¶|d dkr¶t d|› d|› d|› d|| d›d	¡ |d ur¶||| 7 }t d|| d›d¡ q¶tt |¡t |¡| ¡ d}|| } | i |¥|  ¡ ||dœ¥¡ |d ur”|| }| d| ¡ i¡ W d   ƒ n1 sª0    Y  |s¾|S t d|› dd dd„ |  ¡ D ƒ¡ ¡ |j!rt|  ¡ D ](\}!}"|d urø| "d|!› |"|¡ qøt#t$j% |j&d¡dƒ*}#|# 't( )|¡¡ |# 'd¡ W d   ƒ n1 sj0    Y  |j*rºt*d usŽJ dƒ‚|  ¡ D ]"\}!}"t* +d|!› |"d|i¡ q–|S ) Nr   r   g        Tr-   r0   r   r   r    )r.   r   r9   zEval Epoch: r@   z / z]	Clip Loss: z.6fú	zGenerative Loss: r   )Zclip_val_lossrg   rP   Zval_generative_lossr:   c                 S   s&   g | ]\}}|› d t |dƒd›‘qS )r;   é   z.4f)Úroundr1   r   r   r   r?   =  r6   zevaluate.<locals>.<listcomp>zval/zresults.jsonlza+Ú
rG   rg   ),r   rI   r.   rK   r   r   r	   rJ   r   Zval_frequencyZepochsrL   rP   rT   rR   rS   rX   ÚcpuÚmeanÚtÚshapeÚarangeÚlongÚFÚcross_entropyÚmaybe_compute_generative_lossrb   rc   Úget_clip_metricsrZ   r_   r`   rU   Z	save_logsrd   ÚopenÚosÚpathZcheckpoint_pathÚwriteÚjsonÚdumpsre   rO   )$r%   rf   rg   rh   ri   Úmetricsr.   Zzero_shot_metricsrj   rk   rL   rP   Zsamples_per_valZcumulative_lossZcumulative_gen_lossZall_image_featuresZall_text_featuresrm   rn   ro   rp   r!   r   r   r    Úlogits_per_imageÚlogits_per_textra   Úlabelsr)   Zgen_lossZval_metricsr8   rF   r   Úfr   r   r   Úevaluateì   s¦    


,





ÿý(ÿÿÿýÿ
2
ÿÿ
*rŽ   c                 C   sì   i }||  |  ¡   ¡  ¡ }|  ¡  ¡  ¡ }||dœ}t t|ƒ¡ dd¡}| ¡ D ]’\}}	tj|	dd}
t 	|
|k¡d }| ¡  ¡  
¡ }| ¡ d ||› d< t t |¡¡d ||› d< dD ] }t ||k ¡||› d	|› < qÄqT|S )
N)Zimage_to_textZtext_to_imageéÿÿÿÿr   T)Z
descendingZ
_mean_rankZ_median_rank)r   é   r,   z_R@)r{   Údetachry   rI   r}   r^   ÚviewrU   ZargsortÚwhereÚnumpyrz   ÚnpÚfloorZmedian)r   r   r    r‰   rŠ   r‹   ÚlogitsZground_truthrF   ZlogitZrankingÚpredsr3   r   r   r   r‚   Q  s    
 r‚   c                 C   s:   d| v r6d| v r6| d }| d }t  | ddd¡|¡S d S )Nr—   rŒ   r   r   r   )r   r€   Zpermute)r!   Ztoken_logitsZtoken_labelsr   r   r   r   e  s    r   )N)N)#r‡   rb   rM   r„   rQ   r”   r•   rI   Ztorch.nn.functionalr[   Z
functionalr   Ztorch.nn.parallel.distributedr   re   ÚImportErrorZ	open_clipr   r   r   Zdistributedr   Z	zero_shotr   rJ   r	   Úobjectr
   r"   r&   r(   rt   rŽ   r‚   r   r   r   r   r   Ú<module>   s2   

 /
e