a
    Ad(                    @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	m
Z
 d dlZd dlZd dlmZmZ d dlZd dlZd dlmZ d dlmZ d dlZd dlmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z' d d	lm(Z( d d
l)m*Z*m+Z+m,Z, d dl-m.Z.m/Z/m0Z0 d dl-m1Z1 d dl2m3Z3 d dl4Zd dl5m6Z6 d dl7Zd dl8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZAmBZB d dlCmDZDmEZE dZFdZGdd ZHdd ZIdd ZJdd ZKdd ZLe	ddG dd dZMG d d! d!ZNd<d#d$ZOG d%d& d&ePZQd=d'd(ZRd)d* ZSd+d, ZTd-d. ZUd>d/d0ZVdd!d1d2d3ZWdd!d1d4d5ZXd6d7 ZYe	ddG d8d9 d9eMZZe	ddG d:d; d;eMZ[dS )?    )annotationsN)	dataclassfield)ImageImageOps)exposure)Any)devicesprompt_parsermaskingsd_samplerslowvramgeneration_parameters_copypasteextra_networkssd_vae_approxscriptssd_samplers_commonsd_uneterrorsrng)slerp)model_hijack)images_tensor_to_samplesdecode_first_stageapproximation_indexes)optscmd_optsstate)AddMiDaS)LatentDepth2ImageDiffusion)repeat	rearrange)blendLayers	BlendType      c                 C  s&   t d tt|  tj}|S )NzCalibrating color correction.)logginginfocv2cvtColornpasarraycopyCOLOR_RGB2LAB)imageZcorrection_target r/   :/var/www/html/stable-diffusion-webui/modules/processing.pysetup_color_correction+   s    
r1   c              
   C  sX   t d tttjtt	|tj
| ddtjd}t||tj}|dS )NzApplying color correction.   )Zchannel_axisuint8RGB)r&   r'   r   	fromarrayr(   r)   r   Zmatch_histogramsr*   r+   r-   ZCOLOR_LAB2RGBastyper"   r#   Z
LUMINOSITYconvert)
correctionZoriginal_imager.   r/   r/   r0   apply_color_correction1   s    
	r9   c           
      C  s   |d u s|t |kr| S || }|d url|\}}}}td|j|jf}	td| ||} |	| ||f |	} | d} | 	| | d} | S )NRGBA   r4   )
lenr   newwidthheightimagesresize_imagepaster7   alpha_composite)
r.   Z	paste_locindexZoverlaysoverlayxywhZ
base_imager/   r/   r0   apply_overlayA   s    


rJ   c                 C  sF   | j dkr8|  d dkr8|  d ddd } n
| d} | S )Nr:   )   rL   Lc                 S  s   | dkrdS dS )N   rL   r   r/   )rF   r/   r/   r0   <lambda>V       z$create_binary_mask.<locals>.<lambda>)mode
getextremasplitr7   point)r.   r/   r/   r0   create_binary_maskT   s    
rU   c                 C  s   | j jdv rbtj|jd d|||jdd }t|tt	j
}tjjj|ddd}||j}|S | j jd	kr|j|jd d
| jjj |j|jdS |j|jd ddd|j|jdS d S )N>   hybridconcatr      device      ?)r   r   r   r   r;   r         ?)valuecrossattn-admr2   )dtyperZ      r;   )modelconditioning_keytorchonesshaperZ   r   r   getr   sd_vae_encode_methodnn
functionalpadtor_   	new_zerosnoise_augmentor
time_embeddim)sd_modelrF   r>   r?   image_conditioningr/   r/   r0   txt2img_image_conditioning[   s     &rr   F)reprc                   @  sv  e Zd ZU dZded< dZded< dZded< dZded< dZded	< dZ	ded
< dZ
ded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded < d!Zded"< d!Zded#< dZded$< dZded%< d&Zded'< d&Zded(< dZd)ed*< dZd+ed,< dZded-< d&Zded.< dZ ded/< dZ!ded0< dZ"ded1< dZ#ded2< dZ$ded3< dZ%ded4< dZ&ded5< dZ'd)ed6< dZ(ded7< dZ)ded8< dZ*ded9< dZ+ded:< dZ,dZ-d&Z.ded;< e/dd&d<Z0d=ed>< e/dd&d<Z1d+ed?< e/d&d&d<Z2ded@< ddgZ3ddgZ4dZ5dAedB< e/dd&d<Z6dCedD< e/d&d&d<Z7dedE< e/dd&d<Z8dFedG< e/d&d&d<Z9dedH< e/dd&d<Z:dIedJ< e/dd&d<Z;dIedK< e/dd&d<Z<dLedM< e/dd&d<Z=dedN< e/dd&d<Z>d+edO< e/dd&d<Z?d+edP< e/dd&d<Z@d+edQ< e/dd&d<ZAd+edR< e/dd&d<ZBd+edS< e/dd&d<ZCdedT< e/dd&d<ZDdedU< e/dd&d<ZEdedV< e/dd&d<ZFd+edW< e/dd&d<ZGd+edX< e/dd&d<ZHd+edY< e/dd&d<ZId+edZ< e/dd&d<ZJdAed[< e/dd&d<ZKded\< e/dd&d<ZLded]< e/dd&d<ZMded^< e/dd&d<ZNded_< e/dd&d<ZOded`< e/d&d&d<ZPdeda< dbdc ZQeRddde ZejSdfde ZeRdgdh ZTeTjSdidh ZTeRdjdk ZUeUjSdldk ZUdmdn ZVdodp ZWddqdrZXdsdt ZYdudv ZZdwdx Z[ddydzZ\dd{d|Z]d}d~ Z^dd Z_dd Z`dddZadd ZbdddZcdddZddd Zedd Zfdd ZgddddZhdS )StableDiffusionProcessingNobjectrp   stroutpath_samplesoutpath_grids promptprompt_for_displaynegative_promptz	list[str]stylesrK   intseedsubseedr   floatsubseed_strengthseed_resize_from_hseed_resize_from_wTboolseed_enable_extrassampler_namer;   
batch_sizen_iter2   stepsg      @	cfg_scalei   r>   r?   restore_facestilingFdo_not_save_samplesdo_not_save_gridzdict[str, Any]extra_generation_paramslistoverlay_imagesetado_not_reload_embeddingsdenoising_strengthddim_discretizes_min_unconds_churns_tmaxs_tmins_noiseoverride_settings$override_settings_restore_afterwardssampler_indexrefiner_checkpointrefiner_switch_atdisable_extra_networksdefaultinitzscripts.ScriptRunnerscripts_valuescript_args_valuescripts_setup_completedictcommentsz!sd_samplers_common.Sampler | Nonesampler is_using_inpainting_conditioningtuple | Nonepaste_to
is_hr_passtuplecuczrng.ImageRNG | Noner   step_multipliercolor_correctionsall_promptsall_negative_prompts	all_seedsall_subseeds	iterationmain_promptmain_negative_promptpromptsnegative_promptsseedssubseedsextra_network_datausersd_model_namesd_model_hashsd_vae_namesd_vae_hashis_apic                 C  s  | j d urtdtjd i | _| jd u r.g | _d | _| jd urD| jntj| _| j	d ur\| j	ntj	| _	| j
d urt| j
ntj
| _
| jd ur| jntjptd| _| jd ur| jntj| _| jpi | _| jpi | _| jpi | _d | _| jsd| _d| _d| _d| _tj| _tj| _d S )Nz[sampler_index argument for StableDiffusionProcessing does not do anything; use sampler_name)fileinfrK   r   )r   printsysstderrr   r}    sampler_noise_scheduler_overrider   r   r   r   r   r   r   r   r   script_argsrefiner_checkpoint_infor   r   r   r   r   rt   	cached_uccached_cselfr/   r/   r0   __post_init__   s,    

 z'StableDiffusionProcessing.__post_init__c                 C  s   t jS N)sharedrp   r   r/   r/   r0   rp      s    z"StableDiffusionProcessing.sd_modelc                 C  s   d S r   r/   r   r]   r/   r/   r0   rp      s    c                 C  s   | j S r   )r   r   r/   r/   r0   r      s    z!StableDiffusionProcessing.scriptsc                 C  s$   || _ | j r | jr | js |   d S r   )r   r   r   setup_scriptsr   r/   r/   r0   r      s    c                 C  s   | j S r   )r   r   r/   r/   r0   r      s    z%StableDiffusionProcessing.script_argsc                 C  s$   || _ | jr | j r | js |   d S r   )r   r   r   r   r   r/   r/   r0   r     s    c                 C  s   d| _ | jj| | j d d S )NT)is_ui)r   r   setup_scripsr   r   r/   r/   r0   r   
  s    z'StableDiffusionProcessing.setup_scriptsc                 C  s   d| j |< d S Nr;   )r   )r   textr/   r/   r0   comment  s    z!StableDiffusionProcessing.commentc                 C  s,   | j jjdv | _t| j ||p | j|p(| jS )N>   rV   rW   )rp   ra   rb   r   rr   r>   r?   )r   rF   r>   r?   r/   r/   r0   rr     s    z4StableDiffusionProcessing.txt2img_image_conditioningc           	      C  s   t dd}|dt|d di}t|d d jtjd}t|d	| jd
}t	|d d t
tj}tjjj| j||jdd  ddd}t|\}}d||  ||  d }|S )NZ
dpt_hybrid)
model_typejpgr   zc h w -> h w cmidas_in)N.rY   z1 ... -> n ...)nr[   r2   bicubicF)sizerQ   align_corners       @r\   )r   r!   rc   
from_numpyrk   r   rZ   r    r   r   r   rf   r   rg   rh   ri   interpolaterp   depth_modelre   aminmax)	r   source_imagetransformertransformedr   conditioning_imageconditioning	depth_min	depth_maxr/   r/   r0   depth2img_image_conditioning  s    

z6StableDiffusionProcessing.depth2img_image_conditioningc                 C  s   t |d d ttj}|S )Nr[   )r   r   rf   r   rg   )r   r   r   r/   r/   r0   edit_image_conditioning*  s    z1StableDiffusionProcessing.edit_image_conditioningc                 C  sd   | j |}| j jd ur`d}| j j|tt|g|jd|jd dd\}}t	||fd}|S )Nr   z1 -> b)b)noise_levelr;   )
rp   embedderrm   r    rc   tensorrk   rZ   re   cat)r   r   c_admr   noise_level_embr/   r/   r0   unclip_image_conditioning/  s    4z3StableDiffusionProcessing.unclip_image_conditioningc                 C  s  d| _ |d urXt|r|}qvt|d}|tjd }t|d }t	|}n|j
ddg|jdd  R  }|j|j|jd}t||d|  t| d	tjj}| j| j|}tjjj||jdd  d
}||jd ddd}tj||gdd}|tj| jj}|S )NTrM        o@)NNr;   )rZ   r_   r\   inpainting_mask_weight)r   r   rK   )ro   )r   rc   	is_tensorr*   arrayr7   r6   float32r   roundnew_onesre   rk   rZ   r_   lerpgetattrr   r   r   rp   get_first_stage_encodingencode_first_stagerh   ri   r   expandr   type)r   r   latent_image
image_maskZconditioning_maskr   rq   r/   r/   r0   inpainting_image_conditioning7  s*    

z7StableDiffusionProcessing.inpainting_image_conditioningc                 C  s~   t |}t| jtr | |S | jjdkr6| |S | jj	dv rR| j
|||dS | jj	dkrh| |S ||jd dddS )Nedit>   rV   rW   )r	  r^   r   r`   r;   )r	   cond_cast_float
isinstancerp   r   r   cond_stage_keyr   r   rb   r
  r   rl   re   )r   r   r  r	  r/   r/   r0   img2img_image_conditioning\  s    



z4StableDiffusionProcessing.img2img_image_conditioningc                 C  s   d S r   r/   r   r   r   r   r/   r/   r0   r   p  s    zStableDiffusionProcessing.initc                 C  s
   t  d S r   )NotImplementedError)r   r   unconditional_conditioningr   r   r   r   r/   r/   r0   samples  s    z StableDiffusionProcessing.samplec                 C  s0   d | _ d | _d | _tjs,d d gt_d d gt_d S r   )r   r   r   r   persistent_cond_cachert   r   r   r   r/   r/   r0   closev  s    
zStableDiffusionProcessing.closec                 C  s(   |r| j ptj p| jptjS | jp&tjS r   )token_merging_ratio_hrr   token_merging_ratior   for_hrr/   r/   r0   get_token_merging_ratio~  s    z1StableDiffusionProcessing.get_token_merging_ratioc                   s   t  jtr j _n8t  jtr8 jgt j  _n j j  jg  _t  jtrd j _n jgt j  _t jt jkrt	dt j dt j d fdd jD  _ fdd jD  _ jd  _
 jd  _d S )Nz(Received a different number of prompts (z) and negative prompts ()c                   s   g | ]}t j| jqS r/   r   prompt_stylesapply_styles_to_promptr}   .0rF   r   r/   r0   
<listcomp>  rP   z;StableDiffusionProcessing.setup_prompts.<locals>.<listcomp>c                   s   g | ]}t j| jqS r/   r   r  apply_negative_styles_to_promptr}   r  r   r/   r0   r!    rP   r   )r  rz   r   r   r|   r<   r   r   r   RuntimeErrorr   r   r   r/   r   r0   setup_prompts  s    

"z'StableDiffusionProcessing.setup_promptsc                 C  s(   ||||t jtjj|t jt j| j| jfS )z<Returns parameters that invalidate the cond cache if changed)	r   CLIP_stop_at_last_layersr   rp   sd_checkpoint_infosdxl_crop_leftsdxl_crop_topr>   r?   )r   required_promptsr   r   hires_stepsuse_old_schedulingr/   r/   r0   cached_params  s    z'StableDiffusionProcessing.cached_paramsc                 C  s   t jjr:t|||d}t|||d}||kr:d| jd< | ||||t jj}	|D ](}
|
d durT|	|
d krT|
d   S qT|d }
t * |t j	|||t jj|
d< W d   n1 s0    Y  |	|
d< |
d S )a  
        Returns the result of calling function(shared.sd_model, required_prompts, steps)
        using a cache to store the result if the same arguments have been used before.

        cache is an array containing two elements. The first element is a tuple
        representing the previously used arguments, or None if no arguments
        have been used before. The second element is where the previously
        computed result is stored.

        caches is a list with items described above.
        FTzOld prompt editing timelinesr   Nr;   )
r   r   r,  r
   Z)get_learned_conditioning_prompt_schedulesr   r-  r	   autocastrp   )r   functionr*  r   cachesr   r+  Zold_schedulesZnew_schedulesr-  cacher/   r/   r0   get_conds_with_caching  s    

8z0StableDiffusionProcessing.get_conds_with_cachingc                 C  s   t j| j| j| jd}t j| j| j| jdd}t| j}|rJ|	| j
n| j
}|| j
 | _|| _| t j||| jg| j| _| t j||| jg| j| _d S N)r>   r?   T)r>   r?   Zis_negative_prompt)r
   SdConditioningr   r>   r?   r   r   find_sampler_configr   total_stepsr   r   firstpass_stepsr2  get_learned_conditioningr   r   r   "get_multicond_learned_conditioningr   r   )r   r   r   sampler_configr6  r/   r/   r0   setup_conds  s    z%StableDiffusionProcessing.setup_condsc                 C  s   | j | jfS r   )r   r   r   r/   r/   r0   	get_conds  s    z#StableDiffusionProcessing.get_condsc                 C  s   t | j\| _| _d S r   )r   parse_promptsr   r   r   r/   r/   r0   parse_extra_network_prompts  s    z5StableDiffusionProcessing.parse_extra_network_prompts)returnc                 C  s$   t jo"| j o"t jp"tj o"tj S )z;Returns whether generated images need to be written to disk)r   samples_saver   save_incomplete_imagesr   interruptedskippedr   r/   r/   r0   save_samples  s    z&StableDiffusionProcessing.save_samples)NN)N)N)F)NF)N)i__name__
__module____qualname__rp   __annotations__rw   rx   rz   r{   r|   r}   r   r   r   r   r   r   r   r   r   r   r   r>   r?   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   propertysetterr   r   r   r   rr   r   r   r   r
  r  r   r  r  r  r%  r-  r2  r;  r<  r>  rD  r/   r/   r/   r0   rt   s   s   








%



!rt   c                
   @  s<   e Zd ZdddddZd	d
 ZddddZdddZdS )	ProcessedrK   ry   Nr   rt   pc                 C  sv  || _ |j| _|j| _|| _|| _|j| _|| _ddd |jD | _|j	| _	|j
| _
|j| _|j| _t|dd | _|j| _|j| _|j| _|jrtjnd | _|j| _|j| _|j| _|j| _|j| _|j| _t|dd | _|j| _|
| _|j| _tj| _tj | _!|j"| _"|j#| _#|j$| _$|j%| _%|j&| _&|j'| _'|j(| _(|j)| _)|j*| _*|j+| _+t,| jt-sh| jn| jd | _t,| jt-s| jn| jd | _| jd urt.t,| jt-s| jn| jd nd| _| jd urt.t,| jt-s| jn| jd nd| _|j/| _/|p|j0p| jg| _0|p0|j1p0| jg| _1|pH|j2pH| jg| _2|	p`|j3p`| jg| _3|pn|g| _4d S )Nry   c                 s  s   | ]}| d V  qdS )
Nr/   )r   r   r/   r/   r0   	<genexpr>  rP   z%Processed.__init__.<locals>.<genexpr>image_cfg_scaler   r   rK   )5r@   rz   r|   r   r   r   r'   joinr   r>   r?   r   r   r  rP  r   r   r   r   face_restoration_modelr   r   r   r   r   r   r   r   index_of_first_imager}   r   job_timestampr&  	clip_skipr  r  r   r   r   r   r   r   r   r   r  r   r~   r   r   r   r   r   	infotexts)r   rM  images_listr   r'   r   r   r   r   r   rS  rV  r   r/   r/   r0   __init__  sb      44zProcessed.__init__c                  C  s   | j d | j | jd | j| j| j| j| j| j| j| j| j	| j
| j| j| j| j| j| j| j| j| j| j| j| j| j| j| j| j| j| jd}t|S )Nr   )rz   r   r|   r   r   r   r   r   r   r>   r?   r   r   r   r   r   rR  r   r   r   r   r   r   r   r   rS  rV  r}   rT  rU  r   )r   r   r   r   r   r   r   r>   r?   r   r   r   r   r   rR  r   r   r   r   r   r   r   r   rS  rV  r}   rT  rU  r   jsondumps)r   objr/   r/   r0   js  sB    "zProcessed.jsc              	   C  s(   t || j| j| jg || j || j dS )N)r   position_in_batchr   )create_infotextr   r   r   r   )r   rM  rD   r/   r/   r0   infotext>  s    zProcessed.infotextFc                 C  s   |r
| j S | jS r   )r  r  r  r/   r/   r0   r  A  s    z!Processed.get_token_merging_ratio)
rK   ry   NNNNNr   Nry   )F)rE  rF  rG  rX  r\  r_  r  r/   r/   r/   r0   rK    s   5%rK          c                 C  s   t j| |||||d}| S )Nr   r   r   r   )r   ImageRNGnext)re   r   r   r   r   r   rM  gr/   r/   r0   create_random_tensorsE  s    re  c                   @  s   e Zd ZdZdS )DecodedSamplesTN)rE  rF  rG  already_decodedr/   r/   r/   r0   rf  J  s   rf  c                 C  s   t  }t|jd D ]}t| |||d  d }|rzt|d W n tjy } zjtjtj	kslt
jjsp|td tj	t_| jtj |tj}t| |||d  d }W Y d }~n
d }~0 0 |d ur||}|| q|S )Nr   r;   vaea  A tensor with all NaNs was produced in VAE.
Web UI will now convert VAE into 32-bit float and retry.
To disable this behavior, disable the 'Automatically revert VAE to 32-bit floats' setting.
To always start with 32-bit VAE, use --no-half-vae commandline flag.)rf  rangere   r   r	   test_for_nansNansException	dtype_vaerc   r   r   r   auto_vae_precisionr   print_error_explanationfirst_stage_modelrk   append)ra   batchtarget_devicecheck_for_nanssamplesir  er/   r/   r0   decode_latent_batchN  s(    0
rw  c                 C  s^   | dks| d u rd} n.t | trDzt| } W n tyB   d} Y n0 | dkrZttdS | S )Nry   rK   l    )r  rv   r~   	Exceptionrandom	randrange)r   r/   r/   r0   get_fixed_seedp  s    

r{  c                 C  s   t | j| _t | j| _d S r   )r{  r   r   rL  r/   r/   r0   fix_seed  s    r|  c                  C  s    dd l } |  }|dkrd }|S )Nr   z<none>)launchgit_tag)r}  resr/   r/   r0   program_version  s
    r  c
                 C  s  |d u r||| j   }|	d u r$| j}	t| dtj}
t| dd}|  }| jdd}tjdk}|rjt| }| j	| j
| jt| dd |r| jd n|| | jrtjnd | j d| j tjr| jnd tjr| jnd tjr| jnd tjr| jnd | jdkrd n|r| jd n|| | jdkrd n| j| jdks:| jdkr>d n| j d| j t| d	d | jrpt| d
tjjnd |
dkrd n|
|rtjnd |dkrd n||r|dkrd n|t| dd tjdkrtjnd | jdkrd n| j| j rdnd d| j!tj"rt# nd tj$r| j%nd d}d&dd |' D }|rL| j(n|| }|	| rxd|rl| j)n|	|  nd}| | d| * S )NrU  	enable_hrFTr  r   rP  rF   r   r   r;   init_img_hashGPUTrue)StepsSamplerz	CFG scalezImage CFG scaleZSeedzFace restorationSizez
Model hashModelzVAE hashVAEzVariation seedzVariation seed strengthzSeed resize fromzDenoising strengthzConditional mask weightz	Clip skipENSDzToken merging ratiozToken merging ratio hrzInit image hashRNGZNGMSTiling)VersionZUserz, c                 S  s8   g | ]0\}}|d ur||kr |n| dt | qS )Nz: )r   quote)r   kvr/   r/   r0   r!    rP   z#create_infotext.<locals>.<listcomp>z
Negative prompt: ry   rN  )+r   r   r  r   r&  r  eta_noise_seed_deltar   Z%is_sampler_using_eta_noise_seed_deltar   r   r   r   r   rR  r>   r?   add_model_hash_to_infor   add_model_name_to_infor   r   r   r   r   r   r   r   r   r   randn_sourcer   r   r   add_version_to_infotextr  add_user_name_to_infor   rQ  itemsr   r   strip)rM  r   r   r   r   r   r]  use_main_promptrD   r   rU  r  r  r  Z	uses_ensdZgeneration_paramsZgeneration_params_textprompt_textZnegative_prompt_textr/   r/   r0   r^    sZ    


$,

(r^  )rM  r?  c                 C  sN  | j d ur| j |  dd | j D }ztj| jdd u rZ| jdd  t  | j	 D ]:\}}t
j||ddd |dkrt  |dkrdt  qdt| j|   t| }W t| jd | jr|	 D ]$\}}tt
|| |dkrt  qnJt| jd | jrH|	 D ](\}}tt
|| |dkrt  q0 |S )	Nc                 S  s   i | ]}|t j| qS r/   )r   data)r   r  r/   r/   r0   
<dictcomp>  rP   z"process_images.<locals>.<dictcomp>sd_model_checkpointTF)r   run_callbackssd_vaer   )r   before_processr   keys	sd_modelsZcheckpoint_aliasesrf   popreload_model_weightsr  r   setr  reload_vae_weightsapply_token_mergingrp   r  process_images_innerr   setattr)rM  Zstored_optsr  r  r  r/   r/   r0   process_images  s8    



r  c                   sd
  t  jtr t jdks.J n jdus.J t  t jt j j	du r\t
j _	 jdu rnt
j _ jdvrt j _ jdu rtd j tjjj _tjj _t  _t  _tjj  j tjj!   "  t tr _#n  fddt$t j%D  _#t tr6 _&nfddt$t j%D  _&t'j()t*j+rv j,svtj-.   j/dur j/0  g }g }t12 H  j3  t4 J  5 j% j# j& tj
j6rt
j7dkrt89  t:;  W d   n1 s
0    Y  t<j=d	kr( j>t<_=t$ j>D ]}| _?t<j@rLd
t<_@t<jArZ qtB   j%| jC |d  jC   _D jE| jC |d  jC   _F j#| jC |d  jC   _G j&| jC |d  jC   _HtIjJtK jLtM  jNtM f jG jH jO jP jQd _I j/dur> j/jR | jD jG jHd t jDdkrT q S   jTst4  tUV  jW W d   n1 s0    Y   j/durĈ j/jX | jD jG jHd |dkr&tYt'j(Zt[j\dddd,}t] g }|^|_ d W d   n1 s0    Y   `  tjaD ]} b| q4 jcdtjc  j>dkrzd|d  d j> tj<_etjfrtg nt4 2  jh ji jj jG jH jO jDd}W d   n1 s0    Y  tk|dd
r|}n,t
jldkrt
jl jcd< tm j|tjndd}t1o|p }t1jq|d d ddd}~trstjrTtrt  t   j/dur j/ju ||d  j%| jC |d  jC   _D jE| jC |d  jC   _Ft/vt|}	 j/jw |	|d |	jx}d4 fdd 	}
 y }tz|D ]\}}| _{d!t|}|n ~ dd" }|t|j} j	r|rt
jrtxjt| jd# jG|  jD| t
j|
| d$d%	 t  tj	|}t  t|} j/durt/|} j/ | |j} jdurL|t jk rL|r<t
jr<t| j| j}txj| jd# jG|  jD| t
j|
| d&d%	 t j| |}t| j| j}|rtxj| jd# jG|  jD| t
j|
| d' |
|}|| t
jr||jd(< || |rt d)r jrtt
jt
jt
jt
jgr jd*}t|d+d,td,|jtxd" j|jN|jLd-d+}t
jrztxj| jd# jG|  jD| t
j|
| d.d%	 t
jrtxj| jd# jG|  jD| t
j|
| d/d%	 t
jr|| t
jr|| q~t  t<  q2d _d}t|d"k 	ot
j}t
j	st
j	r j	s|	stx| jC}t
j	rz|
dd0}|d| t
j	rj||jd(< |d| d}t
j	rtxj| jd1 j#d  j%d t
j|
dd0t
j  dd2
 W d   n1 	s0    Y  W d   n1 	s0    Y   jT
s jW
rtU  jW t  t] | j#d |d  j&d ||d3} j/du
r` j/ | |S )5zthis is the main loop that both txt2img and img2img use; it calls func_init once inside all the scopes and func_sample once per batchr   N)Nry   Nonenone$Could not find checkpoint with name c                   s&   g | ]}t  jd kr|nd  qS )r   )r~   r   r  )rM  r   r/   r0   r!    rP   z(process_images_inner.<locals>.<listcomp>c                   s   g | ]}t  | qS r/   )r~   r  )r   r/   r0   r!    rP   z	Approx NNrK   Fr;   ra  )batch_numberr   r   r   z
params.txtrH   utf8)encodingzBatch z out of )r   r  r   r   r   r   rg  FullzVAE DecoderTrr  rs  r\   r   r`  minmax)r  c              	     s   t   j j j||  jdS )N)r  rD   r   )r^  r   r   r   r   )rD   r  rL  r/   r0   r_    s    z&process_images_inner.<locals>.infotextr   r2   ry   z-before-face-restorationr'   rM  suffixz-before-color-correction)r'   rM  
parametersmask_for_overlayr4   r:   RGBarM   z-maskz-mask-composite)r  grid)r'   Zshort_filenamerM  r  )rW  r   r'   r   rS  rV  )r   F)r  rz   r   r<   r	   torch_gcr{  r   r   r   r   face_restorationr   r   r  get_closet_checkpoint_matchr   rx  r   rp   r'  Zname_for_extrar   r   r  Zget_loaded_vae_namer   Zget_loaded_vae_hashr   modules	sd_hijackr   Zapply_circularZclear_commentsr%  r   ri  r   r   ospathexistsr   embeddings_dirr   Zembedding_dbZ!load_textual_inversion_embeddingsr   processrc   no_grad	ema_scoper.  r   live_previews_enableshow_progress_typer   ra   r   Z
apply_unetr   	job_countr   r   rC  rB  r  r   r   r   r   r   r   r   rb  opt_Cr?   opt_fr>   r   r   r   before_process_batchr>  r   r   activater   process_batchopenrQ  paths	data_pathrK  writer_  r;  r   r   r   updatejobunet_needs_upcastwithout_autocastr  r   r   r  sd_vae_decode_methodrw  cpustackr   clampr   
is_enabledsend_everything_to_cpupostprocess_batchPostprocessBatchListArgspostprocess_batch_listr@   rD  	enumeratebatch_indexr*   moveaxisnumpyr6   r3   #save_images_before_face_restoration
save_imager   r5   rw   samples_formatPostprocessImageArgspostprocess_imager.   r   #save_images_before_color_correctionrJ   r   r   r9   rp  enable_pnginfor'   hasattrr  any	save_masksave_mask_compositereturn_maskreturn_mask_compositer7   	compositer=   r   rA   nextjobgrid_only_if_multiplereturn_grid	grid_saver   
image_gridinsertrx   grid_formatgrid_extended_filenameZ
deactivatepostprocess)rM  rV  Zoutput_imagesr   r   	processedr   Zsamples_ddimZx_samples_ddimZbatch_paramsr_  rD  ru  x_sampler.   ppZimage_without_ccr   r	  Zimage_mask_compositerS  Z"unwanted_grid_because_of_img_countr  r  r/   )rM  r   r   r0   r    sL   








 

(    4
.

2
B  6

0.


4B00


x
r  c                 C  sN   d}| | }t || }t ||  d d } t || d d }| |fS )z2old algorithm for auto-calculating first pass sizei   @   )mathsqrtceil)r>   r?   Zdesired_pixel_countZactual_pixel_countscaler/   r/   r0   #old_hires_fix_first_pass_dimensions  s    r  c                      sF  e Zd ZU dZded< dZded< dZded	< dZded
< dZded< dZ	ded< dZ
ded< dZded< dZded< dZded< dZded< dZded< dZded< ddgZddgZedddZded< edddZded< edddZded< edddZded< edddZded< edddZded < edddZded!< edddZd"ed#< edddZd"ed$< edddZd%ed&< edddZd%ed'< edddZd%ed(< edddZ d%ed)< edddZ!d%ed*<  fd+d,Z"d-d. Z#d/d0 Z$d1d2 Z%d3d4 Z& fd5d6Z' fd7d8Z(d9d: Z) fd;d<Z* fd=d>Z+ fd?d@Z,  Z-S )A StableDiffusionProcessingTxt2ImgFr   r        ?r   r   r   r~   firstphase_widthfirstphase_heightr   hr_scaleNrv   hr_upscalerhr_second_pass_stepshr_resize_xhr_resize_yhr_checkpoint_namehr_sampler_namery   	hr_prompthr_negative_promptr   r   hr_checkpoint_infohr_upscale_to_xhr_upscale_to_y
truncate_x
truncate_yr   applied_old_hires_behavior_tolatent_scale_moder   hr_chr_ucr   all_hr_promptsall_hr_negative_prompts
hr_promptshr_negative_promptshr_extra_network_datac                   sR   t    | jdks| jdkr>| j| _| j| _| j| _| j| _tj	| _	tj
| _
d S )Nr   )superr   r  r	  r>   r  r?   r  r  cached_hr_uccached_hr_cr   	__class__r/   r0   r     s    
z.StableDiffusionProcessingTxt2Img.__post_init__c                 C  s  t jr\| j| j| jfkr\| j| _| j| _| j| _| j| _t	| j| j\| _| _| j| jf| _| jdkr| jdkr| j
| jd< t| j| j
 | _t| j| j
 | _n| j d| j | jd< | jdkr| j| _| j| j | j | _n| jdkr| j| j | j | _| j| _n| j}| j}| j| j }| j| j }||k rX| j| _| j| j | j | _n| j| j | j | _| j| _| j| t | _| j| t | _d S )Nr   zHires upscalerF   zHires resize)r   use_old_hires_fix_width_heightr  r>   r?   r  r  r  r  r  r
  r   r~   r  r  r  )r   target_wtarget_hZ	src_ratioZ	dst_ratior/   r/   r0   calculate_target_resolution!  s:    


z<StableDiffusionProcessingTxt2Img.calculate_target_resolutionc                   s   j r jrDt j _ jd u r6td j  jj jd<  jd urf j j	krf j jd< t
 jt
 jkr j jd< t
 jt
 jkr j jd<  jd urtj jd ntjtjd _ j r jd u rt fddtjD std	 j    tjshtjd
kr4 jt_tj j jpJ j tj  tjd t_dt_ jr| j jd<  jd ur j jd< d S )Nr  zHires checkpointzHires samplerzHires promptzHires negative promptnearestc                 3  s   | ]}|j  jkV  qd S r   )namer  r  r   r/   r0   rO  ]  rP   z8StableDiffusionProcessingTxt2Img.init.<locals>.<genexpr>zcould not find upscaler named rK   r2   TzHires stepszHires upscaler) r  r  r  r  r  rx  Zshort_titler   r  r   r   r  rz   r  r|   r  r   latent_upscale_modesrf   latent_upscale_default_moder  r  sd_upscalersr)  r    processing_has_refined_job_countr  r   
total_tqdmupdateTotalr   r  r  r/   r   r0   r   H  s6    
,"z%StableDiffusionProcessingTxt2Img.initc           
      C  s   t | j| j| _| j }| jj| |||| |d}~| j	sD|S | j
d u rttt| j|tjddjtjd}	nd }	t  tj| jd W d    n1 s0    Y  t  | ||	||||S )Nrq   Tr  r_   )r'   )r   create_samplerr   rp   r   r   rc  r  rr   r  r  rc   r  rw  r	   r  rk   r   r  ZSkipWritingToConfigr  r  r  sample_hr_pass)
r   r   r  r   r   r   r   rF   rt  decoded_samplesr/   r/   r0   r  p  s    

&
,z'StableDiffusionProcessingTxt2Img.samplec              	     s~  t jjr|S d_j}j} fdd}	jp8j}
t	|
j
_jd urt|jd D ]}|	|| qbtjjj||t |t fjd jd d}tdt jjd	k rʈtj
||}n
|}ntj|d	 d
 dd	d}g }t|D ]\}}dt|  dd }| tj!}t"#|}|	|| t$j%d|||j&d}t'| tj(d }t|dd}|)| qt*t'|}|j+t j,t-j.d}tj/dkrtj/j0d< t1|t23tj/}||}t j4  |d d d d j5d |jd j5d d  j6d |jd j6d d  f }t7j8|jdd  j9j:j;j<j=d_7j7> }t-?  j@st-A  tBCjD W d    n1 s0    Y  t-A  E  W d    n1 s0    Y  tFGj
jHdd jId urjIJ jjK||jLjMjNp8jO|d}tFGj
H  d _t-?  tPj
|t-jdd}d_|S )NTc                   sx     rtjsdS t| tjs.tj| |dd} tjj	j
g j|d}tj| jd|  | tj|dd	 dS )zsaves image before applying hires fix, if enabled in options; takes as an argument either an image or batch with latent space imagesNr   )approximation)r   r]  ry   z-before-highres-fixr  )rD  r   save_images_before_highres_fixr  r   r   sample_to_imager^  r   r   r   r   r@   r  rw   r  )r.   rD   r'   r   r   r   r/   r0   save_intermediate  s    zJStableDiffusionProcessingTxt2Img.sample_hr_pass.<locals>.save_intermediater   rQ   	antialias)r   rQ   r<  r   r\   r   r`  r  r   r2   )Zupscaler_namer3  r  VAE Encoderr;   rX   ra  r  )r   rq   r  F)Qr   r   rB  r   r  r  r  r   r   r4  rp   r   r  ri  re   rc   rh   ri   r   r  r  r   r   r  r   rr   r  r  r*   r  r  r  r6   r3   r   r5   r@   rA   r  r   r   rp  r   rk   rZ   r	   rl  rg   r   r   r   rf   r  r  r  r   rb  r   r   r   r   r   rc  r  r   r.  r   r  r   calculate_hr_condsr  r  r  r   	before_hrsample_img2imgr  r  r  r   rw  )r   rt  r6  r   r   r   r   Ztarget_widthZtarget_heightr;  Zimg2img_sampler_nameru  rq   Zlowres_samplesbatch_imagesr  r.   noiser/   r:  r0   r5    sh    
.


T,

.
((z/StableDiffusionProcessingTxt2Img.sample_hr_passc                   s4   t    d | _d | _tjs0d d gt_d d gt_d S r   )	r!  r  r  r  r   r  r  r"  r#  r   r$  r/   r0   r    s    

z&StableDiffusionProcessingTxt2Img.closec                   s   t     jsd S  jdkr& j _ jdkr8 j _t jtrN j _	n j
 j  jg  _	t jtrz j _n j
 j  jg  _ fdd j	D  _	 fdd jD  _d S )Nry   c                   s   g | ]}t j| jqS r/   r  r  r   r/   r0   r!    rP   zBStableDiffusionProcessingTxt2Img.setup_prompts.<locals>.<listcomp>c                   s   g | ]}t j| jqS r/   r"  r  r   r/   r0   r!    rP   )r!  r%  r  r  rz   r  r|   r  r   r  r   r   r  r   r$  r   r0   r%    s    




z.StableDiffusionProcessingTxt2Img.setup_promptsc                 C  s   | j d urd S tj| j| j| jd}tj| j| j| jdd}t| j	pJ| j
}| jpX| j}|rh||n|}| tj|| j| j| jg| j|| _| tj|| j| j| jg| j|| _ d S r3  )r  r
   r4  r  r  r  r  r   r5  r  r   r  r   r6  r2  r8  r7  r"  r   r   r  r9  r#  r   )r   r  r  r:  r   r6  r/   r/   r0   r>    s    
$z3StableDiffusionProcessingTxt2Img.calculate_hr_condsc                   s   | j rd | _|   d S t   d | _d | _| jr| jd u rtj	j
rP|   nttjrtjjt krt  t| | j W d    n1 s0    Y  |   t  t| | j W d    n1 s0    Y  d S r   )r   r  r>  r!  r;  r  r  r  r   r   hires_fix_use_firstpass_condsr   r  rp   r'  r  Zselect_checkpointr	   r.  r   r  r   r   r   r$  r/   r0   r;    s     


,
z,StableDiffusionProcessingTxt2Img.setup_condsc                   s   | j r| j| jfS t  S r   )r   r  r  r!  r<  r   r$  r/   r0   r<  /  s    z*StableDiffusionProcessingTxt2Img.get_condsc                   sp   t   }| jrl| j| j| j | jd | j  | _| j| j| j | jd | j  | _t	
| j\| _| _|S r   )r!  r>  r  r  r   r   r  r  r  r   r=  r   )r   r  r$  r/   r0   r>  5  s    
$$z<StableDiffusionProcessingTxt2Img.parse_extra_network_prompts).rE  rF  rG  r  rH  r   r  r	  r
  r  r  r  r  r  r  r  r  r"  r#  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r)  r   r  r5  r  r%  r>  r;  r<  r>  __classcell__r/   r/   r$  r0   r    sP   
'(ar  c                      sr  e Zd ZU dZded< dZded< dZded	< dZded
< dZded< dZ	ded< dZ
ded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< edddZded< edddZded< edddZded< edddZd ed!< edddZded"< edddZded#<  fd$d%Zed&d' Zejd(d' Zd)d* Zd+d, Zd/d-d.Z  ZS )0 StableDiffusionProcessingImg2ImgNr   init_imagesr   r~   resize_moder  r   r   rP  r   maskr$   mask_blur_xmask_blur_y	mask_blurinpainting_fillTr   inpaint_full_resinpaint_full_res_paddinginpainting_mask_invertinitial_noise_multiplierr   latent_maskFr   r	  ztorch.Tensornmaskrq   rv   r  r  init_latentc                   s4   t    | j| _d | _| jd u r(tjn| j| _d S r   )r!  r   rH  r	  rP  r   r   r$  r/   r0   r   Z  s    
z.StableDiffusionProcessingImg2Img.__post_init__c                 C  s   | j | jkr| j S d S r   )rI  rJ  r   r/   r/   r0   rK  a  s    z*StableDiffusionProcessingImg2Img.mask_blurc                 C  s   t |tr|| _|| _d S r   )r  r~   rI  rJ  r   r/   r/   r0   rK  g  s    
c                 C  s  t jjdkr| jnd | _t| j| j| _d }| j}|d urt	|}| j
rVt|}| jdkrt|}dtd| j d  d }t||df| j}t|}| jdkrt|}dtd| j d  d }t|d|f| j}t|}| jrv|| _|d}tt|| j}t|| j| j|j|j}|\}	}
}}||}t !d|| j| j}|	|
||	 ||
 f| _"nNt !| j#|| j| j}t|}t$|%tj&d dd%tj'}t|| _g | _(| j)d ur| j)n|}t*j+o| j,d u }|rg | _,g }| j-D ]Z}t*j.rBt/0|1 2 | _3t j4|t*j5d | j3d	d
 t 6|t*j7}|d u r|| j#dkr|t !| j#|| j| j}|d urt8d|j|jf}|j9|ddt| jdd | j(:|d |d ur||}t !d|| j| j}|d ur| j;dkrt<||}|r4| j,:t=| t|%tj&d }t>|dd}|:| qt?|dkrtj@|d ddjA| jBdd}| j(d ur| j(| jB | _(| j,d urt?| j,dkr| j,| jB | _,nBt?|| jBkrt?|| _Bt|}ntCdt?| d| jB dtDE|}|jFt jGtHjId}t*jJdkrNt*jJ| jKd< tL|tMNt*jJ| j| _OtHP  | j#dkrtDjQjRjS| jO| jtT | jtT fdd| _O|d ur|}|dU| jOjVd | jOjVd f}t>tj|tj&dddd }|d }tW|}tX|d  d}tDYd| Ft jGZ| jj[| _\tDY|Ft jGZ| jj[| _]| j;dkr| jO| j\ t^| jOjVdd  |d| jOjVd  | j]  | _On| j;dkr| jO| j\ | _O| _|d d | jO|| _`d S )Nr  r   r2   g      @r[   r;   rM   rL   F)r  basenameZforced_filenamesave_to_dirsrX   r  r:   )rH  r   )axiszbad number of images passed: z; expecting z or lessr3  r  r=  bilinear)r   rQ   r4   )r$   r;   r;   r\   )ar   rp   r  rP  r   r4  r   r   r	  rU   rO  r   invertrI  r*   r   r~   r(   GaussianBlurr   r5   rJ  rM  r  r7   r   Zget_crop_regionrN  Zexpand_crop_regionr>   r?   cropr@   rA   r   rG  clipr6   r   r3   r   rQ  r   img2img_color_correctionr   rF  save_init_imghashlibmd5tobytes	hexdigestr  r  outdir_init_imagesflattenimg2img_background_colorr=   rB   rp  rL  fillr1   r  r<   expand_dimsr    r   r$  rc   r   rk   rZ   r	   rl  rg   r   r   r   rf   rS  r  rh   ri   r   r  resizere   aroundtiler+   r  r_   rH  rR  re  r  rq   )r   r   r   r   Zcrop_regionr	  Znp_maskkernel_sizerH  x1y1x2y2rQ  Zadd_color_correctionsimgsimgr.   Zimage_maskedrA  Z	init_maskZlatmaskr/   r/   r0   r   m  s    










"
(




(
$
"<z%StableDiffusionProcessingImg2Img.initc           	      C  st   | j  }| jdkr*| j| jd< || j9 }| jj| | j|||| jd}| jd urf|| j	 | j| j  }~t
  |S )Nr\   zNoise multiplierr2  )r   rc  rP  r   r   r@  rS  rq   rH  rR  r	   r  )	r   r   r  r   r   r   r   rF   rt  r/   r/   r0   r    s    



z'StableDiffusionProcessingImg2Img.samplec                 C  s"   | j p d| jv rtj p tjp tj S )Nr  )r  r   r   token_merging_ratio_img2imgr  r/   r/   r0   r    s    z8StableDiffusionProcessingImg2Img.get_token_merging_ratio)F) rE  rF  rG  rF  rH  rG  r   rP  rH  rI  rJ  rK  rL  rM  rN  rO  rP  rQ  r   r	  rR  rq   r  r  rS  r   rI  rJ  r   r  r  rD  r/   r/   r$  r0   rE  A  s:   


 rE  )Nr`  r   r   N)NF)Nr   r   FNN)\
__future__r   rY  r&   r  r  r   r^  dataclassesr   r   rc   r  r*   PILr   r   ry  r(   Zskimager   typingr   modules.sd_hijackr  r	   r
   r   r   r   r   r   r   r   r   r   r   r   Zmodules.rngr   r   Zmodules.sd_samplers_commonr   r   r   Zmodules.sharedr   r   r   r   Zmodules.pathsr  Zmodules.face_restorationZmodules.imagesr@   Zmodules.stylesmodules.sd_modelsr  modules.sd_vaer  Zldm.data.utilr   ldm.models.diffusion.ddpmr   einopsr    r!   Zblendmodes.blendr"   r#   r  r  r1   r9   rJ   rU   rr   rt   rK  re  r   rf  rw  r{  r|  r  r^  r  r  r  r  rE  r/   r/   r/   r0   <module>   sx   <  qb

"

6( |  O