a
    Aþdã_  ã                   @  sZ  d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlZejjjjjZejjjjjZG dd	„ d	ƒZG d
d„ deƒZG dd„ deƒZ G dd„ de ƒZ!G dd„ deƒZ"G dd„ deƒZ#G dd„ deƒZ$G dd„ deƒZ%dd„ Z&ej'j(sJej'j)r€zd dl*Z(de_+W n" e,y~   ej-ddd Y n0 dd„ Z.dMdd „Z/dNd!d"„Z0e 1¡ j2d# Z3d$d%„ Z4d&d'„ Z5d(d)„ Z6d*d+„ Z7d,d-„ Z8d.d/„ Z9d0d1„ Z:d2d3„ Z;dOd4d5„Z<dPd6d7„Z=dQd9d:„Z>d;d<„ Z?dRd=d>„Z@dSd?d@„ZAdTdAdB„ZBdCdD„ ZCdEdF„ ZDdGdH„ ZEdIdJ„ ZFdKdL„ ZGdS )Ué    )ÚannotationsN)Úeinsum)Údefault©Ú	rearrange)ÚsharedÚerrorsÚdevicesÚsub_quadratic_attention)Úhypernetworkc                   @  s^   e Zd ZU dZded< dZded< dZded< dZded	< d
d„ Zdd„ Z	dd„ Z
dd„ ZdS )ÚSdOptimizationNÚstrÚnamez
str | NoneÚlabelÚcmd_optr   ÚintÚpriorityc                 C  s"   | j d u r| jS | j› d| j › S )Nz - )r   r   ©Úself© r   úG/var/www/html/stable-diffusion-webui/modules/sd_hijack_optimizations.pyÚtitle   s    
zSdOptimization.titlec                 C  s   dS )NTr   r   r   r   r   Úis_available%   s    zSdOptimization.is_availablec                 C  s   d S ©Nr   r   r   r   r   Úapply(   s    zSdOptimization.applyc                 C  s<   t jtjjj_ttjjj	j
_t jtjjj_ttjjj	j
_d S r   )r   Ú attention_CrossAttention_forwardÚldmÚmodulesÚ	attentionÚCrossAttentionÚforwardÚ(diffusionmodules_model_AttnBlock_forwardÚdiffusionmodulesÚmodelÚ	AttnBlockÚsgmÚ,sgm_diffusionmodules_model_AttnBlock_forwardr   r   r   r   Úundo+   s    zSdOptimization.undo)Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__r   r   r   r   r   r   r'   r   r   r   r   r      s   
r   c                   @  s(   e Zd ZdZdZdZdd„ Zdd„ ZdS )ÚSdOptimizationXformersÚxformerséd   c                 C  s:   t jjp8t jo8tj ¡ o8dtj t j¡  ko4dkS   S )N)é   r   )é	   r   )	r   Úcmd_optsÚforce_enable_xformersÚxformers_availableÚtorchÚcudar   Úget_device_capabilityÚdevicer   r   r   r   r   8   s    z#SdOptimizationXformers.is_availablec                 C  s8   t tjjj_ttjjjj	_t t
jjj_tt
jjjj	_d S r   )Úxformers_attention_forwardr   r   r   r   r    Úxformers_attnblock_forwardr"   r#   r$   r%   r   r   r   r   r   ;   s    zSdOptimizationXformers.applyN)r(   r)   r*   r   r   r   r   r   r   r   r   r   r,   3   s
   r,   c                   @  s,   e Zd ZdZdZdZdZdd„ Zdd„ Zd	S )
ÚSdOptimizationSdpNoMemz
sdp-no-memz5scaled dot product without memory efficient attentionÚopt_sdp_no_mem_attentionéP   c                 C  s   t tjjdƒottjjjƒS )NÚscaled_dot_product_attention)Úhasattrr4   ÚnnÚ
functionalÚcallabler=   r   r   r   r   r   H   s    z#SdOptimizationSdpNoMem.is_availablec                 C  s8   t tjjj_ttjjjj	_t t
jjj_tt
jjjj	_d S r   )Ú+scaled_dot_product_no_mem_attention_forwardr   r   r   r   r    Úsdp_no_mem_attnblock_forwardr"   r#   r$   r%   r   r   r   r   r   K   s    zSdOptimizationSdpNoMem.applyN)	r(   r)   r*   r   r   r   r   r   r   r   r   r   r   r:   B   s   r:   c                   @  s$   e Zd ZdZdZdZdZdd„ ZdS )ÚSdOptimizationSdpZsdpzscaled dot productÚopt_sdp_attentionéF   c                 C  s8   t tjjj_ttjjjj	_t t
jjj_tt
jjjj	_d S r   )Ú$scaled_dot_product_attention_forwardr   r   r   r   r    Úsdp_attnblock_forwardr"   r#   r$   r%   r   r   r   r   r   X   s    zSdOptimizationSdp.applyN©r(   r)   r*   r   r   r   r   r   r   r   r   r   rD   R   s
   rD   c                   @  s(   e Zd ZdZdZedd„ ƒZdd„ ZdS )ÚSdOptimizationSubQuadzsub-quadraticÚopt_sub_quad_attentionc                 C  s   t jjdkrdS dS ©NÚmpsiè  é
   )r   r7   Útyper   r   r   r   r   c   s    zSdOptimizationSubQuad.priorityc                 C  s8   t tjjj_ttjjjj	_t t
jjj_tt
jjjj	_d S r   )Úsub_quad_attention_forwardr   r   r   r   r    Úsub_quad_attnblock_forwardr"   r#   r$   r%   r   r   r   r   r   g   s    zSdOptimizationSubQuad.applyN©r(   r)   r*   r   r   Úpropertyr   r   r   r   r   r   rJ   _   s
   
rJ   c                   @  s$   e Zd ZdZdZdZdZdd„ ZdS )ÚSdOptimizationV1ÚV1zoriginal v1Úopt_split_attention_v1rN   c                 C  s   t tjjj_t tjjj_d S r   )Ú split_cross_attention_forward_v1r   r   r   r   r    r%   r   r   r   r   r   t   s    zSdOptimizationV1.applyNrI   r   r   r   r   rT   n   s
   rT   c                   @  s(   e Zd ZdZdZedd„ ƒZdd„ ZdS )ÚSdOptimizationInvokeAIZInvokeAIÚopt_split_attention_invokeaic                 C  s   t jjdkrtj ¡ sdS dS rL   )r   r7   rO   r4   r5   r   r   r   r   r   r   }   s    zSdOptimizationInvokeAI.priorityc                 C  s   t tjjj_t tjjj_d S r   )Ú&split_cross_attention_forward_invokeAIr   r   r   r   r    r%   r   r   r   r   r      s    zSdOptimizationInvokeAI.applyNrR   r   r   r   r   rX   y   s
   
rX   c                   @  s    e Zd ZdZdZdZdd„ ZdS )ÚSdOptimizationDoggettxZDoggettxÚopt_split_attentionéZ   c                 C  s8   t tjjj_ttjjjj	_t t
jjj_tt
jjjj	_d S r   )Úsplit_cross_attention_forwardr   r   r   r   r    Ú!cross_attention_attnblock_forwardr"   r#   r$   r%   r   r   r   r   r   ‹   s    zSdOptimizationDoggettx.applyN)r(   r)   r*   r   r   r   r   r   r   r   r   r[   †   s   r[   c              	   C  s*   |   tƒ tƒ tƒ tƒ tƒ tƒ tƒ g¡ d S r   )Úextendr,   r:   rD   rJ   rT   rX   r[   )Úresr   r   r   Úlist_optimizers’   s    ùrb   TzCannot import xformers)Úexc_infoc                  C  sb   t jjdkrTtj t j¡} | d }| d }tj tj ¡ ¡\}}|| }|| }|S t 	¡ j
S d S )Nr5   úactive_bytes.all.currentúreserved_bytes.all.current)r   r7   rO   r4   r5   Úmemory_statsÚmem_get_infoÚcurrent_deviceÚpsutilÚvirtual_memoryÚ	available)ÚstatsÚ
mem_activeÚmem_reservedÚmem_free_cudaÚ_Úmem_free_torchÚmem_free_totalr   r   r   Úget_available_vram¦   s    rs   c                   s  | j ‰ |  |¡}t||ƒ}t tj|¡\}}|  |¡}|  |¡}	~~~~‡ fdd„|||	fD ƒ\}
}}~~~	|
j	}tj
jr–|
 ¡ | ¡ | ¡   }
}}tjtj
j d² tj|
jd |
jd |jd |
j|
j	d}td|
jd dƒD ]^}|d }td|
||… |||… ƒ}|| j9 }|jd	d
}~td||||… ƒ|||…< ~qæ~
~~W d   ƒ n1 sb0    Y  | |¡}t|dˆ d}~|  |¡S )Nc                 3  s   | ]}t |d ˆ dV  qdS ©zb n (h d) -> (b h) n d©ÚhNr   ©Ú.0Útru   r   r   Ú	<genexpr>¿   ó    z3split_cross_attention_forward_v1.<locals>.<genexpr>©Údisabler   é   é   ©r7   Údtypeúb i d, b j d -> b i jéÿÿÿÿ)Údimúb i j, b j d -> b i dú(b h) n d -> b n (h d)ru   )ÚheadsÚto_qr   r   Úapply_hypernetworksr   Úloaded_hypernetworksÚto_kÚto_vr   ÚoptsÚupcast_attnÚfloatr	   Úwithout_autocastr4   ÚzerosÚshaper7   Úranger   ÚscaleÚsoftmaxÚtor   Úto_out)r   ÚxÚcontextÚmaskÚkwargsÚq_inÚ	context_kÚ	context_vÚk_inÚv_inÚqÚkÚvr   Úr1ÚiÚendÚs1Ús2Úr2r   ru   r   rW   ´   s6    



*
&
rW   c                   s’  | j ‰ |  |¡}t||ƒ}t tj|¡\}}|  |¡}|  |¡}	|j	}
tj
jrz| ¡ | ¡ |	jjdkrj|	n|	 ¡   }}}	tjtj
j dÎ || j }~~‡ fdd„|||	fD ƒ\}}}~~~	tj|jd |jd |jd |j|j	d}tƒ }d	}|jd |jd  |jd  | ¡  }| ¡ dkr.d
nd}|| }d}||krbdt t || d¡¡ }|dkrÀt t t |d ¡¡d ¡d }td|› d|› d|d | d›d|| d›d	ƒ‚|jd | }td|jd |ƒD ]f}t|| |jd ƒ}td|d d …||…f |ƒ}|j d|j	d}~td||ƒ|d d …||…f< ~qà~~~W d   ƒ n1 sd0    Y  | !|
¡}t"|dˆ d}~|  #|¡S )NrM   r|   c                 3  s   | ]}t |d ˆ dV  qdS rt   r   rw   ru   r   r   rz   ð   r{   z0split_cross_attention_forward.<locals>.<genexpr>r   r~   r   r€   é   @é   ç      @é@   é   z5Not enough memory, use lower resolution (max approx. r˜   z	). Need: z0.1fzGB free, Have:zGB freer‚   rƒ   ©r„   r   r…   r†   ru   )$r‡   rˆ   r   r   r‰   r   rŠ   r‹   rŒ   r   r   rŽ   r   r7   rO   r	   r   r”   r4   r‘   r’   rs   Úelement_sizeÚmathÚceilÚlogÚfloorÚsqrtÚRuntimeErrorr“   Úminr   r•   r–   r   r—   )r   r˜   r™   rš   r›   rœ   r   rž   rŸ   r    r   r¡   r¢   r£   r¤   rr   ÚgbÚtensor_sizeÚmodifierÚmem_requiredÚstepsÚmax_resÚ
slice_sizer¥   r¦   r§   r¨   r©   r   ru   r   r^   Ý   sV    



,
*&

"ÿÿ
&
r^   rª   c                 C  s(   t d| |ƒ}|jd|jd}t d||ƒS )Nr‚   rƒ   r¯   r…   )r   r•   r   )r¡   r¢   r£   Úsr   r   r   Úeinsum_op_compvis   s    rÀ   c                 C  sz   t j| jd | jd |jd | j| jd}td| jd |ƒD ]8}|| }t| ||… |||… |||… ƒ|||…< q<|S ©Nr   r~   r   r€   ©r4   r‘   r’   r7   r   r“   rÀ   ©r¡   r¢   r£   r¾   Úrr¥   r¦   r   r   r   Úeinsum_op_slice_0&  s
    *.rÅ   c                 C  sz   t j| jd | jd |jd | j| jd}td| jd |ƒD ]8}|| }t| d d …||…f ||ƒ|d d …||…f< q<|S rÁ   rÂ   rÃ   r   r   r   Úeinsum_op_slice_1.  s
    *.rÆ   c                 C  sh   | j d | j d  dkr$t| ||ƒS t d| j d | j d   ¡}|d dkrV|d8 }t| |||ƒS d S )Nr   r~   é   rª   i   )r’   rÀ   r±   r´   rÆ   )r¡   r¢   r£   r¾   r   r   r   Úeinsum_op_mps_v16  s    rÈ   c                 C  s>   t dkr,| jd | jd  dkr,t| ||ƒS t| ||dƒS d S )Nr®   r   r~   rÇ   )Úmem_total_gbr’   rÀ   rÅ   ©r¡   r¢   r£   r   r   r   Úeinsum_op_mps_v2@  s     rË   c                 C  sš   | j d | j d  |j d  |  ¡  d }||kr>t| ||ƒS dt|d | ƒ ¡ > }|| j d kr|t| ||| j d | ƒS t| ||t| j d | dƒƒS )Nr   r~   é   )r’   r°   rÀ   r   Ú
bit_lengthrÅ   rÆ   Úmax)r¡   r¢   r£   Zmax_tensor_mbÚsize_mbÚdivr   r   r   Úeinsum_op_tensor_memG  s    *rÑ   c           
      C  sV   t j | j¡}|d }|d }t j | j¡\}}|| }|| }	t| |||	d d ƒS )Nrd   re   gffffff
@rÌ   )r4   r5   rf   r7   rg   rÑ   )
r¡   r¢   r£   rl   rm   rn   ro   rp   rq   rr   r   r   r   Úeinsum_op_cudaQ  s    rÒ   c                 C  s|   | j jdkrt| ||ƒS | j jdkrntdkrb| jd d dkrb| jd | jd  dk rbt| ||ƒS t| ||ƒS t| ||dƒS )Nr5   rM   é    r   r~   i   )r7   rO   rÒ   rÉ   r’   rÈ   rË   rÑ   rÊ   r   r   r   Ú	einsum_op\  s    2rÔ   c                   s  | j ‰ |  |¡}t||ƒ}t tj|¡\}}|  |¡}|  |¡}	~~~~|j	}
tj
jr‚| ¡ | ¡ |	jjdkrr|	n|	 ¡   }}}	tjtj
j dD || j }‡ fdd„|||	fD ƒ\}}}	t|||	ƒ}W d   ƒ n1 sÞ0    Y  | |
¡}|  t|dˆ d¡S )NrM   r|   c                 3  s   | ]}t |d ˆ dV  qdS rt   r   rw   ru   r   r   rz   |  r{   z9split_cross_attention_forward_invokeAI.<locals>.<genexpr>r†   ru   )r‡   rˆ   r   r   r‰   r   rŠ   r‹   rŒ   r   r   rŽ   r   r7   rO   r	   r   r”   rÔ   r–   r—   r   )r   r˜   r™   rš   r›   r¡   r   rž   r¢   r£   r   rÄ   r   ru   r   rZ   j  s     



,
*
rZ   c              	   K  sn  |d u sJ dƒ‚| j }|  |¡}t||ƒ}t tj|¡\}}|  |¡}	|  |¡}
~~~~| 	d|df¡ 
dd¡jdd}|	 	d|df¡ 
dd¡jdd}	|
 	d|df¡ 
dd¡jdd}
|jjdkrà| ¡ |	 ¡ |
 ¡   }}	}
|j}tjjr| ¡ |	 ¡  }}	t||	|
tjjtjjtjj| jd}| |¡}| 	dd|f¡ 
dd¡jdd	}| j\}}||ƒ}||ƒ}|S )
NzLattention-mask not currently implemented for SubQuadraticCrossAttnProcessor.rƒ   r~   r   )Úend_dimrM   ©Úq_chunk_sizeÚkv_chunk_sizeÚchunk_thresholdÚuse_checkpointr   )Ú	start_dim)r‡   rˆ   r   r   r‰   r   rŠ   r‹   rŒ   Ú	unflattenÚ	transposeÚflattenr7   rO   Ú
contiguousr   r   rŽ   r   Úsub_quad_attentionr1   Úsub_quad_q_chunk_sizeÚsub_quad_kv_chunk_sizeÚsub_quad_chunk_thresholdÚtrainingr–   r—   )r   r˜   r™   rš   r›   rv   r¡   r   rž   r¢   r£   r   Úout_projÚdropoutr   r   r   rP   †  s.    



   
$
 
rP   é   c              
   C  s<  t  | j¡jd }| j\}	}
}|j\}}}|	| |
 | }|d u rx| jjdkrhdt ¡ dkr`dn| }q˜t	t
ƒ d ƒ}n |dkr†d }nt	d| t
ƒ  ƒ}|d u rÊ|d urÊ||	| |jd |jd    }n|dkrÖd }|d urê||krê|}tj| j|jkd	( tj| ||||||d
W  d   ƒ S 1 s.0    Y  d S )Nr®   rM   i   Úi386r   gffffffæ?r   g{®Gáz„?r|   )Zquery_chunk_sizerØ   Úkv_chunk_size_minrÚ   )r4   Úfinfor   Úbitsr’   r7   rO   ÚplatformÚ	processorr   rs   r	   r   r
   Zefficient_dot_product_attention)r¡   r¢   r£   r×   rØ   ré   rÙ   rÚ   Zbytes_per_tokenZbatch_x_headsZq_tokensrp   Zk_tokensZqk_matmul_size_bytesZchunk_threshold_bytesr   r   r   rà   «  s6    "ùrà   c              
   C  sx   t jjsd S z6tjj}|\}}| tjjj| ||d d¡r@|W S W n0 t	yr } zt
 |d¡ W Y d }~n
d }~0 0 d S )N)ÚqueryÚkeyÚvalueÚ	attn_biaszenabling flash attention)r   r1   Úxformers_flash_attentionr-   ÚopsZ(MemoryEfficientAttentionFlashAttentionOpZsupportsZfmhaZInputsÚ	Exceptionr   Údisplay_once)r¡   r¢   r£   Zflash_attention_opÚfwÚbwÚer   r   r   Úget_xformers_flash_attention_opÑ  s    
"rù   c              	     sÎ   | j ‰ |  |¡}t||ƒ}t tj|¡\}}|  |¡}|  |¡}	‡ fdd„|||	fD ƒ\}
}}~~~	|
j	}tj
jrŽ|
 ¡ | ¡ | ¡   }
}}tjj|
||d t|
||ƒd}| |¡}t|dˆ d}|  |¡S )Nc                 3  s   | ]}t |d ˆ dV  qdS )zb n (h d) -> b n h dru   Nr   rw   ru   r   r   rz   é  r{   z-xformers_attention_forward.<locals>.<genexpr>)rñ   Úopzb n h d -> b n (h d)ru   )r‡   rˆ   r   r   r‰   r   rŠ   r‹   rŒ   r   r   rŽ   r   r-   ró   Úmemory_efficient_attentionrù   r–   r   r—   )r   r˜   r™   rš   r›   rœ   r   rž   rŸ   r    r¡   r¢   r£   r   Úoutr   ru   r   r8   à  s    




r8   c                 K  sX  |j \}}}|d ur:|  |||¡}| || jd|j d ¡}| j}|  |¡}	t||ƒ}t tj	|¡\}
}|  
|
¡}|  |¡}|| }|	 |d||¡ dd¡}| |d||¡ dd¡}| |d||¡ dd¡}~	~~|j}tjjrú| ¡ | ¡ | ¡   }}}tjjj||||ddd}| dd¡ |d|| ¡}| |¡}| jd |ƒ}| jd |ƒ}|S )Nrƒ   r~   r   ç        F)Ú	attn_maskÚ	dropout_pÚ	is_causalr   )r’   Zprepare_attention_maskÚviewr‡   rˆ   r   r   r‰   r   rŠ   r‹   rŒ   rÝ   r   r   rŽ   r   r4   r?   r@   r=   Úreshaper–   r—   )r   r˜   r™   rš   r›   Ú
batch_sizeZsequence_lengthÚ	inner_dimrv   rœ   r   rž   rŸ   r    Úhead_dimr¡   r¢   r£   r   Úhidden_statesr   r   r   rG   ú  s4    



ÿ
rG   c                 K  sF   t jjjdddd t| |||ƒW  d   ƒ S 1 s80    Y  d S ©NTF)Úenable_flashÚenable_mathÚenable_mem_efficient)r4   Úbackendsr5   Ú
sdp_kernelrG   )r   r˜   r™   rš   r›   r   r   r   rB   #  s    rB   c                 C  sà  |}|   |¡}|  |¡}|  |¡}|  |¡}|j\}}}}	| ||||	 ¡}
~|
 ddd¡}~
| ||||	 ¡}~tj||j	d}t
ƒ }|jd |jd  |jd  | ¡  }|d }d}||krÜdt t || d¡¡ }|jd | dkrü|jd | n|jd }td|jd |ƒD ]œ}|| }t |d d …||…f |¡}|t|ƒd  }~tjjj|d|jd}~| ||||	 ¡}| ddd¡}~t ||¡|d d …d d …||…f< ~~q| ||||	¡}~|  |¡}~||7 }|S )Nr   r   r~   )r7   r¬   g      à¿r¯   )Únormr¡   r¢   r£   r’   r  Úpermuter4   Ú
zeros_liker7   rs   r°   r±   r²   r³   r“   Úbmmr   r?   r@   r•   r   Úproj_out)r   r˜   Úh_Úq1Úk1r£   ÚbÚcrv   ÚwÚq2r¡   r¢   rr   r¹   r»   r¼   r¾   r¥   r¦   Úw1Úw2Úw3Úv1Úw4Úh2Úh3r   r   r   r_   (  sL    



&*"
r_   c                 C  sö   zÔ|}|   |¡}|  |¡}|  |¡}|  |¡}|j\}}}}	dd„ |||fD ƒ\}}}|j}
tjjrv| 	¡ | 	¡  }}| 
¡ }| 
¡ }| 
¡ }tjj|||t|||ƒd}| |
¡}t|d|d}|  |¡}|| W S  tyð   t| |ƒ Y S 0 d S )Nc                 s  s   | ]}t |d ƒV  qdS ©zb c h w -> b (h w) cNr   rw   r   r   r   rz   k  r{   z-xformers_attnblock_forward.<locals>.<genexpr>)rú   úb (h w) c -> b c h wru   )r  r¡   r¢   r£   r’   r   r   r   rŽ   r   rß   r-   ró   rû   rù   r–   r   r  ÚNotImplementedErrorr_   ©r   r˜   r  r¡   r¢   r£   r  r  rv   r  r   rü   r   r   r   r9   c  s*    






r9   c                 C  sØ   |}|   |¡}|  |¡}|  |¡}|  |¡}|j\}}}}	dd„ |||fD ƒ\}}}|j}
tjjr~| 	¡ | 	¡ | 	¡   }}}| 
¡ }| 
¡ }| 
¡ }tjjj|||ddd}| |
¡}t|d|d}|  |¡}|| S )Nc                 s  s   | ]}t |d ƒV  qdS r   r   rw   r   r   r   rz   ‚  r{   z(sdp_attnblock_forward.<locals>.<genexpr>rý   F)rÿ   r   r!  ru   )r  r¡   r¢   r£   r’   r   r   r   rŽ   r   rß   r4   r?   r@   r=   r–   r   r  r#  r   r   r   rH   {  s$    





rH   c                 C  sB   t jjjdddd t| |ƒW  d   ƒ S 1 s40    Y  d S r  )r4   r  r5   r  rH   )r   r˜   r   r   r   rC     s    rC   c              	   C  s°   |}|   |¡}|  |¡}|  |¡}|  |¡}|j\}}}}	dd„ |||fD ƒ\}}}| ¡ }| ¡ }| ¡ }t|||tjj	tjj
tjj| jd}
t|
d|d}
|  |
¡}
||
 S )Nc                 s  s   | ]}t |d ƒV  qdS r   r   rw   r   r   r   rz   œ  r{   z-sub_quad_attnblock_forward.<locals>.<genexpr>rÖ   r!  ru   )r  r¡   r¢   r£   r’   rß   rà   r   r1   rá   râ   rã   rä   r   r  )r   r˜   r  r¡   r¢   r£   r  r  rv   r  rü   r   r   r   rQ   •  s    



$
rQ   )NN)NN)NN)NN)rç   NNNT)NN)NN)NN)HÚ
__future__r   r±   ri   rì   r4   r   Úldm.utilr   Úeinopsr   r   r   r   r	   r
   Úmodules.hypernetworksr   Úldm.modules.attentionr   Ú"ldm.modules.diffusionmodules.modelÚsgm.modules.attentionr%   Ú"sgm.modules.diffusionmodules.modelr"   r#   r$   r    r!   r&   r   r,   r:   rD   rJ   rT   rX   r[   rb   r1   r-   r2   Zxformers.opsr3   rô   Úreportrs   rW   r^   rj   ÚtotalrÉ   rÀ   rÅ   rÆ   rÈ   rË   rÑ   rÒ   rÔ   rZ   rP   rà   rù   r8   rG   rB   r_   r9   rH   rC   rQ   r   r   r   r   Ú<module>   sl   

)
@



%
&

)
;