a
    Aþd€‹  ã                   @   sš  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlmZmZ d dlmZ d dlmZmZmZmZmZmZmZ d dlmZmZ d dlmZ d dl	mZ d d	lmZmZm Z m!Z!m"Z"m#Z# d d
l$m%Z% d dl&m'Z'm(Z( dd„ e )e	j*ej+¡D ƒZ,G dd„ de	j-j.ƒZ/dd„ Z0G dd„ dƒZ1dd„ Z2dd„ Z3d+dd„Z4d,dd„Z5d-dd„Z6d.dd„Z7d d!„ Z8d"d#„ Z&d/d%d&„Z9d'd(„ Z:d)d*„ Z;dS )0é    N)Úclosing)Ú	rearrangeÚrepeat)Údefault)ÚdevicesÚ	sd_modelsÚsharedÚsd_samplersÚhashesÚsd_hijack_checkpointÚerrors)Útextual_inversionÚlogging)ÚLearnRateScheduler)Úeinsum)Únormal_Úxavier_normal_Úxavier_uniform_Úkaiming_normal_Úkaiming_uniform_Úzeros_)Údeque)ÚstdevÚmeanc                 C   s   i | ]\}}|d kr||“qS )Ú	Optimizer© )Ú.0Z
optim_nameÚcls_objr   r   úJ/var/www/html/stable-diffusion-webui/modules/hypernetworks/hypernetwork.pyÚ
<dictcomp>   ó    r   c                       s†   e Zd Zejjejjejjejjejj	ejj
ejjdœZe dd„ e ejjj¡D ƒ¡ d‡ fdd„	Zd	d
„ Zdd„ Zdd„ Z‡  ZS )ÚHypernetworkModule)ÚlinearÚreluÚ	leakyreluÚeluÚswishÚtanhÚsigmoidc                 C   s.   i | ]&\}}t  |¡r|jd kr| ¡ |“qS )ztorch.nn.modules.activation)ÚinspectÚisclassÚ
__module__Úlower)r   Úcls_namer   r   r   r   r   #   r    zHypernetworkModule.<dictcomp>NÚNormalFc	                    sÐ  t ƒ  ¡  d| _|d us J dƒ‚|d dks4J dƒ‚|d dksHJ dƒ‚g }	tt|ƒd ƒD ]}
|	 tj t	|||
  ƒt	|||
d   ƒ¡¡ |dksä|d u sä|
t|ƒd	 kr¸|s¸n,|| j
v rÖ|	 | j
| ƒ ¡ ntd
|› ƒ‚|r|	 tj t	|||
d   ƒ¡¡ |d ur\||
d  dkr\d||
d    k rDdk sNn J dƒ‚|	 tjj||
d  d¡ q\tjj|	Ž | _|d urœ|  |¡ |  |¡ n$| jD ]}t|ƒtjjksÌt|ƒtjjkr¢|jj|jj }}|dksút|ƒtjjkrt|ddd t|ddd n¤|dkr4t|ƒ t|ƒ nˆ|dkrPt|ƒ t|ƒ nl|dkr~t|d|krldndd t|ƒ n>|dkr¬t|d|kršdndd t|ƒ ntd|› dƒ‚q¢|  tj¡ d S )Nç      ð?z layer_structure must not be Noner   é   z-Multiplier Sequence should start with size 1!éÿÿÿÿz+Multiplier Sequence should end with size 1!r"   é   z6hypernetwork uses an unsupported activation function: z9Dropout probability should be 0 or float between 0 and 1!)Úpr.   g        g{®Gáz„?)r   ÚstdZXavierUniformZXavierNormalZKaimingUniformr$   Ú
leaky_relur#   )ÚnonlinearityZKaimingNormalzKey z" is not defined as initialization!) ÚsuperÚ__init__Ú
multiplierÚrangeÚlenÚappendÚtorchÚnnÚLinearÚintÚactivation_dictÚRuntimeErrorÚ	LayerNormÚDropoutÚ
Sequentialr"   Úfix_old_state_dictÚload_state_dictÚtypeÚweightÚdataÚbiasr   r   r   r   r   r   ÚKeyErrorÚtor   Údevice)ÚselfÚdimÚ
state_dictÚlayer_structureÚactivation_funcÚweight_initÚadd_layer_normÚactivate_outputÚdropout_structureZlinearsÚiÚlayerÚwÚb©Ú	__class__r   r   r8   %   sT    
0$
"*

$







zHypernetworkModule.__init__c                 C   sH   dddddœ}|  ¡ D ],\}}| |d ¡}|d u r4q||= |||< qd S )Nzlinear.0.biaszlinear.0.weightzlinear.1.biaszlinear.1.weight)zlinear1.biaszlinear1.weightzlinear2.biaszlinear2.weight)ÚitemsÚget)rO   rQ   ÚchangesÚfrrM   Úxr   r   r   rF   d   s    üz%HypernetworkModule.fix_old_state_dictc                 C   s   ||   |¡| js| jnd  S )Nr0   )r"   Útrainingr9   )rO   rb   r   r   r   Úforwardt   s    zHypernetworkModule.forwardc                 C   sD   g }| j D ]4}t|ƒtjjks.t|ƒtjjkr
||j|jg7 }q
|S ©N)r"   rH   r=   r>   r?   rC   rI   rK   )rO   rR   rY   r   r   r   Ú
trainablesw   s
    
 zHypernetworkModule.trainables)NNNr.   FFN)Ú__name__r+   Ú__qualname__r=   r>   ÚIdentityÚReLUÚ	LeakyReLUÚELUÚ	HardswishÚTanhÚSigmoidrA   Úupdater)   Ú
getmembersÚmodulesÚ
activationr8   rF   rd   rf   Ú__classcell__r   r   r\   r   r!      s   ù	   ÿ?r!   c                 C   sh   | d u rg d¢} |s"dgt | ƒ S dg}| dgt | ƒd  ¡ |rP| d¡ n
| d¡ | d¡ |S )N©r0   r2   r0   r   g333333Ó?é   )r;   Úextendr<   )rR   Úuse_dropoutÚlast_layer_dropoutZdropout_valuesr   r   r   Úparse_dropout_structure€   s    

rz   c                   @   s`   e Zd ZdZdZddd„Zdd„ Zddd	„Zd
d„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ ZdS )ÚHypernetworkNFc	                 K   sü   d | _ || _i | _d| _d | _d | _|| _|| _|| _|| _	|| _
|| _|	 dd¡| _|	 dd ¡| _| jd u r‚t| j| j
| jƒ| _d | _d | _d | _|pšg D ]R}
t|
d | j| j| j| j	| j| jdt|
d | j| j| j| j	| j| jdf| j|
< qœ|  ¡  d S )Nr   ry   TrW   )rW   )ÚfilenameÚnameÚlayersÚstepÚsd_checkpointÚsd_checkpoint_namerR   rS   rT   rU   rx   rV   r_   ry   rW   rz   Úoptimizer_nameÚoptimizer_state_dictÚoptional_infor!   Úeval)rO   r}   Úenable_sizesrR   rS   rT   rU   rx   rV   ÚkwargsÚsizer   r   r   r8   “   s8    
ÿÿýzHypernetwork.__init__c                 C   s.   g }| j  ¡ D ]}|D ]}|| ¡ 7 }qq|S re   )r~   ÚvaluesÚ
parameters)rO   Úresr~   rY   r   r   r   Úweights±   s
    zHypernetwork.weightsTc                 C   s>   | j  ¡ D ].}|D ]$}|j|d | ¡ D ]
}||_q*qq
d S )N)Úmode)r~   r‰   ÚtrainrŠ   Úrequires_grad)rO   r   r~   rY   Úparamr   r   r   rŽ   ¸   s
    zHypernetwork.trainc                 C   s(   | j  ¡ D ]}|D ]}| |¡ qq
| S re   )r~   r‰   rM   )rO   rN   r~   rY   r   r   r   rM   ¿   s    zHypernetwork.toc                 C   s$   | j  ¡ D ]}|D ]
}||_qq
| S re   )r~   r‰   r9   )rO   r9   r~   rY   r   r   r   Úset_multiplierÆ   s    
zHypernetwork.set_multiplierc                 C   s:   | j  ¡ D ]*}|D ] }| ¡  | ¡ D ]
}d|_q&qq
d S )NF)r~   r‰   r…   rŠ   r   )rO   r~   rY   r   r   r   r   r…   Í   s
    zHypernetwork.evalc                 C   s8  i }i }| j  ¡ D ]$\}}|d  ¡ |d  ¡ f||< q| j|d< | j|d< | j|d< | j|d< | j|d< | j|d< | j	|d	< | j
|d
< | j|d< | j|d< | j|d< | jd ur¾| jd dkn| j|d< | jrÔ| jnd |d< | jd urð| j|d< t ||¡ tjjr4| jr4|  ¡ |d< | j|d< t ||d ¡ d S )Nr   r0   r   r}   rR   rS   Úis_layer_normÚweight_initializationr€   r   rV   rx   rW   éþÿÿÿry   r„   r‚   Úhashrƒ   ú.optim)r~   r^   rQ   r   r}   rR   rS   rU   rT   r€   r   rV   rx   rW   ry   r„   r‚   r=   Úsaver   ÚoptsÚsave_optimizer_staterƒ   Ú	shorthash)rO   r|   rQ   Úoptimizer_saved_dictÚkÚvr   r   r   r—   Ô   s0    










"


zHypernetwork.savec                 C   sì  || _ | jd u r*tj tj |¡¡d | _tj|dd}| dg d¢¡| _	| dd ¡| _
| dd ¡| _| dd	¡| _| d
d¡| _| dd ¡| _| jd ur¨t| jƒr¨dn
| dd¡| _| dd¡| _| dd¡| _| jd u rðt| j	| j| jƒ| _tjjrˆ| j
d urtd| j
› dƒ td| j	› ƒ td| j› ƒ td| j› ƒ td| j› ƒ td| j› ƒ td| j› ƒ td| j› ƒ tj | j d ¡r°tj| j d ddni }|  ¡ | dd ¡krÚ| dd ¡| _nd | _| jr| dd¡| _tjjr2tdƒ td | j› ƒ nd| _tjjr2td!ƒ | ¡ D ]j\}}t|ƒtkr:t||d | j	| j| j| j| j| jƒt||d" | j	| j| j| j| j| jƒf| j |< q:| d#| j¡| _| d$d¡| _!| d%d ¡| _"| d&d ¡| _#|  $¡  d S )'Nr   Úcpu)Úmap_locationrR   ru   r„   rS   r“   r.   r’   FrW   Trx   rV   ry   z	  INFO:
 Ú
z  Layer structure: z  Activation function: z  Weight initialization: z  Layer norm: z  Dropout usage: z  Activate last layer: z  Dropout structure: r–   r•   rƒ   r‚   ÚAdamWz)Loaded existing optimizer from checkpointzOptimizer name is z'No saved optimizer exists in checkpointr0   r}   r   r€   r   )%r|   r}   ÚosÚpathÚsplitextÚbasenamer=   Úloadr_   rR   r„   rS   rT   rU   rW   Úanyrx   rV   ry   rz   r   r˜   Úprint_hypernet_extraÚprintÚexistsrš   rƒ   r‚   r^   rH   r@   r!   r~   r   r€   r   r…   )rO   r|   rQ   r›   rˆ   Úsdr   r   r   r¦   ò   sf    
&

,

ÿÿýzHypernetwork.loadc                 C   s*   t  | jd| j› ¡}|r&|dd… S d S )Nz	hypernet/r   é
   )r
   Úsha256r|   r}   )rO   r­   r   r   r   rš   1  s    zHypernetwork.shorthash)NNNNNFFF)T)rg   r+   rh   r|   r}   r8   rŒ   rŽ   rM   r‘   r…   r—   r¦   rš   r   r   r   r   r{      s   

?r{   c                 C   sX   i }t tjtj | d¡ddtjdD ],}tj tj 	|¡¡d }|dkr&|||< q&|S )Nz**/*.ptT)Ú	recursive)Úkeyr   ÚNone)
ÚsortedÚglobÚiglobr¢   r£   ÚjoinÚstrr,   r¤   r¥   )r£   r‹   r|   r}   r   r   r   Úlist_hypernetworks7  s    &
r¶   c                 C   s^   t j | d ¡}|d u rd S ztƒ }| |¡ |W S  tyX   tjd|› dd Y d S 0 d S )NzError loading hypernetwork T©Úexc_info)r   Úhypernetworksr_   r{   r¦   Ú	Exceptionr   Úreport)r}   r£   Úhypernetworkr   r   r   Úload_hypernetworkA  s    
r½   c                 C   sŒ   i }t jD ]}|j| v r
|||j< q
t j ¡  t| ƒD ]P\}}| |d ¡}|d u rZt|ƒ}|d u rdq6| |rt|| nd¡ t j |¡ q6d S )Nr/   )	r   Úloaded_hypernetworksr}   ÚclearÚ	enumerater_   r½   r‘   r<   )ÚnamesZmultipliersZalready_loadedr¼   rX   r}   r   r   r   Úload_hypernetworksP  s    


rÂ   c                 C   s„   | d ur| j ni  |jd d ¡}|d u r0||fS |d urL|d |_|d |_t |d t |¡ƒ¡}t |d t |¡ƒ¡}||fS )Nr2   r   r0   )r~   r_   ÚshapeZhyper_kZhyper_vr   Úcond_cast_unetÚcond_cast_float)r¼   Ú	context_kÚ	context_vrY   Zhypernetwork_layersr   r   r   Úapply_single_hypernetworke  s     

rÈ   c                 C   s,   |}|}| D ]}t ||||ƒ\}}q||fS re   )rÈ   )r¹   ÚcontextrY   rÆ   rÇ   r¼   r   r   r   Úapply_hypernetworkst  s
    rÊ   c                    sÞ   | j ‰ |  |¡}t||ƒ}ttj|| ƒ\}}|  |¡}|  |¡}	‡ fdd„|||	fD ƒ\}}}	td||ƒ| j	 }
|d ur®t
|dƒ}t |
j¡j }t|dˆ d}|
 | |¡ |
jdd}td	||	ƒ}t
|d
ˆ d}|  |¡S )Nc                 3   s   | ]}t |d ˆ dV  qdS )zb n (h d) -> (b h) n d©ÚhN)r   )r   ÚtrË   r   r   Ú	<genexpr>‡  r    z3attention_CrossAttention_forward.<locals>.<genexpr>zb i d, b j d -> b i jzb ... -> b (...)zb j -> (b h) () jrË   r1   )rP   zb i j, b j d -> b i dz(b h) n d -> b n (h d))ÚheadsÚto_qr   rÊ   r   r¾   Úto_kÚto_vr   Úscaler   r=   ÚfinfoÚdtypeÚmaxr   Úmasked_fill_ÚsoftmaxÚto_out)rO   rb   rÉ   Úmaskr‡   ÚqrÆ   rÇ   rœ   r   ÚsimÚmax_neg_valueÚattnÚoutr   rË   r   Ú attention_CrossAttention_forward}  s"    




rà   c                 C   s˜   t | ƒdkrt | ¡S tdd„ | D ƒƒ}tt | ƒƒD ]X}| | jd |kr4| | dd … }| || | jd  dg¡}t | | |g¡| |< q4t | ¡S )Nr0   c                 S   s   g | ]}|j d  ‘qS )r   )rÃ   ©r   rb   r   r   r   Ú
<listcomp>ž  r    zstack_conds.<locals>.<listcomp>r   r1   )r;   r=   ÚstackrÖ   r:   rÃ   r   Úvstack)ÚcondsÚtoken_countrX   Zlast_vectorZlast_vector_repeatedr   r   r   Ústack_conds™  s    
rç   c                 C   s¤   t | ƒdk rd}nt| ƒ}dt| ƒd›d d|t | ƒd  d›d }| d	d … }t |ƒdk rfd}nt|ƒ}d
t|ƒd›d d|t |ƒd  d›d }||fS )Nr2   r   zloss:z.3fõ   Â±ú(g      à?ú)iàÿÿÿzrecent 32 loss:)r;   r   r   )rJ   r4   Ztotal_informationZrecent_dataZrecent_informationr   r   r   Ú
statistics¨  s    ..rë   Fc	              
   C   sî   d  dd„ | D ƒ¡} | s J dƒ‚tj  tjj| › d¡}	|sXtj |	¡rXJ d|	› dƒ‚t|ƒtkrxdd	„ | 	d
¡D ƒ}|r¢|r¢t|ƒtkr¢dd	„ | 	d
¡D ƒ}ndgt
|ƒ }tjjj| dd	„ |D ƒ||||||d}
|
 |	¡ t ¡  d S )NÚ c                 s   s"   | ]}|  ¡ s|d v r|V  qdS )z._- N)Úisalnumrá   r   r   r   rÎ   ¹  r    z&create_hypernetwork.<locals>.<genexpr>zName cannot be empty!ú.ptzfile z already existsc                 S   s   g | ]}t | ¡ ƒ‘qS r   ©ÚfloatÚstriprá   r   r   r   râ   Á  r    z'create_hypernetwork.<locals>.<listcomp>ú,c                 S   s   g | ]}t | ¡ ƒ‘qS r   rï   rá   r   r   r   râ   Ä  r    r   c                 S   s   g | ]}t |ƒ‘qS r   )r@   rá   r   r   r   râ   Ê  r    )r}   r†   rR   rS   rT   rU   rx   rW   )r´   r¢   r£   r   Úcmd_optsÚhypernetwork_dirrª   rH   rµ   Úsplitr;   rr   r¹   r¼   r{   r—   Úreload_hypernetworks)r}   r†   Zoverwrite_oldrR   rS   rT   rU   rx   rW   ÚfnZhypernetr   r   r   Úcreate_hypernetwork·  s,    ø

rø   c           S         s‚
  ddl m}m} |pd}|pd}tj |d ¡}tj||||||||
|||dd |j}tj	 |d ¡} t
ƒ ‰ ˆ  | ¡ ˆ gt_dtj_dtj_|
tj_| dd¡d }tj tjj|› d	¡}!tj |tj ¡  d
¡|¡}tjj}"|dkrtj |d¡}#tj|#dd nd }#|dkr4tj |d¡}$tj|$dd nd }$t ¡ }%ˆ jpJd}&|&|
krfdtj_ˆ |!fS t||
|&ƒ}'|dkr†t j!j"j#n|dkršt j!j"j$nd }(|(r´t||
|&dd})tjj%rÈt &|¡}*dt' (|¡› dtj_tjj)}+t jj*j+|||tjj,|tj-tj-j.t/j0|d||||||	|d},tjj1rnt2f |%j3|%j4t5|,ƒdœ‡ fdd„dD ƒ¤Ž}-t6 7|i |-¥t8ƒ ¥¡ |,j9}t jj*j:|,||,j;|+d}.tj<}/|"r¾dt_<tj-j. =t/j>¡ tj-j? =t/j>¡ ˆ  @¡ }0ˆ  A¡  ˆ jBtCv rötCˆ jB |0|'jDd}1ˆ jB}2n(tEdˆ jB› dƒ t jFjG|0|'jDd}1d}2ˆ jHrnz|1 Iˆ jH¡ W n6 tJyl }3 ztEd ƒ tE|3ƒ W Y d }3~3n
d }3~30 0 t jKjL M¡ }4|,j;}|,jN}t5|,ƒ| | }5t5|,ƒ| t5|,ƒ| |  }6d}7d}8tOt5|,ƒd! d"}9d}:d#};d#}<d#}=tPjP|
|& d$}>züz´tQ R¡  tS|
|& | ƒD ]”}?|'jTr  	q¤tjjUr0 	q¤tV|.ƒD ]d\}@}A|@|6krR q|' W|1ˆ j¡ |'jTrn qtjjUr~ q|(r|) ˆ j¡ t/ X¡ ä |AjYj=t/j0|+d%}B|rÄ|AjZj=t/j0|+d%}C|dksÔ|rtj-j. =t/j0¡ tj- .|Aj[¡j=t/j0|+d%}Dtj-j. =t/j>¡ nt\|Aj]ƒj=t/j0|+d%}D|rHtj- ^|B|D|C¡d | }E~Cntj- _|B|D¡d | }E~B~D|8|E `¡ 7 }8W d   ƒ n1 s„0    Y  |4 a|E¡ b¡  |@d | dkr²q8|9 c|8¡ |(rÎ|(|0|)jDƒ |4 |1¡ |4 d¡  ˆ  jd7  _|> d¡  |1jedd& |8}7d}8ˆ jd }Fˆ j|5 }Gˆ j|5 }Hd'|G› d(|Hd › d)|5› d*|7d+›}I|> f|I¡ |#d ur¼|F| dkr¼|› d,|F› }Jtj |#|J› d	¡};|2ˆ _Btjjgr¨|1 h¡ ˆ _Htiˆ |%||;ƒ d ˆ _Htjj%rˆ jt5|,ƒ }Gˆ j|Gt5|,ƒ  d }Htj|9ƒt5|9ƒ }Ktjk|*|Kˆ j|H|'jD|Gd- t l|d.ˆ j|5|7d+›|'jDd/œ¡ |$d u	rP|F| dk	rP|› d,|F› }=tj |$|=¡}<ˆ  m¡  t  n¡ }Ld }Mt jK o¡ r’t jK p¡ }Mtj-j. =t/j0¡ tj-j? =t/j0¡ |jqtj-ddd0}Nd|N_r|r
||N_s||N_t||N_utvjw| jx|N_y||N_z||N_{||N_|||N_}n|Aj[d |N_sd1|N_u||N_|||N_}|Njs}Ot~|Nƒ8 | |N¡}Pt5|Pjƒdkr\|Pjd nd }QW d   ƒ n1 sv0    Y  |"r¦tj-j. =t/j>¡ tj-j? =t/j>¡ t  €|L¡ t jK o¡ rÈt jK |M¡ ˆ  A¡  |Qd u	rPtj ‚|Q¡ tjj%	rtjjƒ	rt „|*d2|G› |Qˆ j¡ |j…|Q|$d3|Nj{|Njstjj†|Pj‡d |N|=dd4
\}<}R|<d5|O› 7 }<ˆ jtj_ˆd6|7d+›d7|F› d8t' (|Aj[d ¡› d9t' (|;¡› d:t' (|<¡› d;tj_q8qW n" t‰	yÈ   tŠj‹d<dd= Y n0 W d|>_Œ|> ¡  ˆ  m¡  tQ Ž¡  n d|>_Œ|> ¡  ˆ  m¡  tQ Ž¡  0 tj tjj|› d	¡}!|2ˆ _Btjjg
r>|1 h¡ ˆ _Htiˆ |%||!ƒ ~1d ˆ _Htj-j. =t/j0¡ tj-j? =t/j0¡ |/t_<ˆ |!fS )>Nr   )ÚimagesÚ
processingr¼   )r}   ztrain-hypernetworkz%Initializing hypernetwork training...ré   r0   rî   z%Y-%m-%dr¹   T)Úexist_okrù   z9Model has already been trained beyond specified max stepsÚvalueÚnormF)ÚverbosezPreparing dataset from z...)Ú	data_rootÚwidthÚheightÚrepeatsZplaceholder_tokenÚmodelZ
cond_modelrN   Útemplate_fileZinclude_condÚ
batch_sizeÚgradient_stepÚshuffle_tagsÚtag_drop_outÚlatent_sampling_methodÚvarsizeÚ
use_weight)Ú
model_nameZ
model_hashZnum_of_dataset_imagesc                    s   i | ]}|t ˆ |ƒ“qS r   )Úgetattr)r   Úfield©r¼   r   r   r     r    z&train_hypernetwork.<locals>.<dictcomp>)rR   rS   rT   rU   rx   )r	  r  Ú
pin_memory)ÚparamsÚlrzOptimizer type z is not defined!r¡   z#Cannot resume from saved optimizer!rv   )Úmaxlenz<none>)Útotal)Únon_blocking)Úset_to_nonezTraining hypernetwork [Epoch z: ú/z]loss: z.7fÚ-)ÚlossÚglobal_stepr   Ú
learn_rateÚ	epoch_numzhypernetwork_loss.csv)r  r  )Úsd_modelÚdo_not_save_gridÚdo_not_save_samplesé   zValidation at epoch rì   )r3   Úforced_filenameÚsave_to_dirsz
, prompt: z
<p>
Loss: z<br/>
Step: z<br/>
Last prompt: z<br/>
Last saved hypernetwork: z<br/>
Last saved image: z<br/>
</p>
z"Exception in training hypernetworkr·   )rr   rù   rú   r   Ztextual_inversion_templatesr_   Zvalidate_train_inputsr£   r   r¹   r{   r¦   r¾   ÚstateÚjobÚtextinfoÚ	job_countÚrsplitr¢   r´   ró   rô   ÚdatetimeÚnowÚstrftimer˜   Úunload_models_when_trainingÚmakedirsr   Úselect_checkpointr   r   r=   r>   ÚutilsÚclip_grad_value_Úclip_grad_norm_Útraining_enable_tensorboardZtensorboard_setupÚhtmlÚescaper  ÚdatasetZPersonalizedBaseÚ training_image_repeats_per_epochr  Úcond_stage_modelr   rN   Úsave_training_settings_to_txtÚdictr  rš   r;   r   Zsave_settings_to_fileÚlocalsr	  ZPersonalizedDataLoaderr  Úparallel_processing_allowedrM   rž   Úfirst_stage_modelrŒ   rŽ   r‚   Úoptimizer_dictr  r©   Úoptimr¡   rƒ   rG   rB   ÚcudaÚampÚ
GradScalerr  r   Útqdmr   Úaddr:   ÚfinishedÚinterruptedrÀ   ÚapplyÚautocastZlatent_samplerI   Z	cond_textrç   ÚcondÚweighted_forwardrd   ÚitemrÓ   Úbackwardr<   rp   Ú	zero_gradÚset_descriptionr™   rQ   Úsave_hypernetworkÚsumZtensorboard_addZ
write_lossr…   Úget_rng_stateÚis_availableÚget_rng_state_allÚ StableDiffusionProcessingTxt2ImgÚdisable_extra_networksÚpromptÚnegative_promptÚstepsr	   Úsamplersr}   Úsampler_nameÚ	cfg_scaleÚseedr   r  r   Úprocess_imagesÚset_rng_stateÚset_rng_state_allÚassign_current_imageÚ training_tensorboard_save_imagesZtensorboard_add_imageÚ
save_imageÚsamples_formatÚ	infotextsÚjob_norº   r   r»   ÚleaveÚcloseÚremove)SZid_taskÚhypernetwork_namer  r  r  rÿ   Zlog_directoryZtraining_widthZtraining_heightr
  rV  Zclip_grad_modeZclip_grad_valuer  r  r	  r  Zcreate_image_everyZsave_hypernetwork_everyZtemplate_filenameZpreview_from_txt2imgZpreview_promptZpreview_negative_promptZpreview_stepsZpreview_sampler_indexZpreview_cfg_scaleZpreview_seedZpreview_widthZpreview_heightrù   rú   r  r£   r|   Úunloadrô   Ú
images_dirÚ
checkpointZinitial_stepÚ	schedulerÚ	clip_gradZclip_grad_schedZtensorboard_writerr  ÚdsZsaved_paramsÚdlZold_parallel_processing_allowedrŒ   Ú	optimizerr‚   ÚeÚscalerÚsteps_per_epochZmax_steps_per_epochZ	loss_stepZ
_loss_stepZloss_loggingZsteps_without_gradZlast_saved_fileZlast_saved_imager!  ÚpbarÚ_ÚjÚbatchrb   rZ   Úcr  Z
steps_doner  Z
epoch_stepÚdescriptionZhypernetwork_name_everyZ	mean_lossÚ	rng_stateZcuda_rng_stater3   Zpreview_textÚ	processedÚimageZlast_text_infor   r  r   Útrain_hypernetwork×  sÈ   "



,

<
ÿþ



,




"



þ
ý

>


þ0

ýüûú	
ý


r|  c                 C   sx   | j }t| dƒr| jnd }t| dƒr*| jnd }z$|j| _|j| _|| _ |  |¡ W n    || _|| _|| _ ‚ Y n0 d S )Nr€   r   )r}   Úhasattrr€   r   rš   r  r—   )r¼   rj  rg  r|   Zold_hypernetwork_nameZold_sd_checkpointZold_sd_checkpoint_namer   r   r   rM    s    rM  )N)N)N)NN)NNNFFN)<r(  r²   r2  r¢   r)   Ú
contextlibr   Z!modules.textual_inversion.datasetrr   r=   rA  Úeinopsr   r   Úldm.utilr   r   r   r   r	   r
   r   r   Zmodules.textual_inversionr   r   Z(modules.textual_inversion.learn_scheduler   r   Ztorch.nn.initr   r   r   r   r   r   Úcollectionsr   rë   r   r   rq   r=  r*   r<  r>   ÚModuler!   rz   r{   r¶   r½   rÂ   rÈ   rÊ   rà   rç   rø   r|  rM  r   r   r   r   Ú<module>   sF   $ g )



	

   ,