a
    d~
                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ dd Zdd Z	dd Z
d	d
 Zdd Zdd ZdddZdd ZdS )    N)tqdmc                 C   sV   t jddd| |ddgt jt jd}|jdkrHtd|jd	  d
S td dS )NZawss3syncz	--excludez*epoch_latest.pt)stdoutstderrr   z%Error: Failed to sync with S3 bucket zutf-8Fz"Successfully synced with S3 bucketT)	
subprocessrunPIPE
returncodeloggingerrorr   decodeinfo)	local_dir
remote_dirresult r   \/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/training/file_utils.pyremote_sync_s3
   s    "

r   c                 C   s   t | }t |}|D ]}d|v r&qtd|  ||v rjt|| t|| krjtd| d qz"td| d || ||< W q ty } z&td| d|  W Y d }~ dS d }~0 0 qd	S )
Nzepoch_latest.ptzAttempting to sync zSkipping remote sync for .zSuccessful sync for zError during remote sync for z: FT)fsspecZ
get_mapperr   r   lendebug	Exception)r   r   abker   r   r   remote_sync_fsspec   s     

 r   c                 C   s@   t d |dkrt| |S |dkr.t| |S t d dS d S )NzStarting remote sync.r   r   zRemote protocol not knownF)r   r   r   r   r   )r   r   protocolr   r   r   remote_sync,   s    



r    c                 C   s   t |  t||| q d S )N)timesleepr    )
sync_everyr   r   r   r   r   r   keep_running_remote_sync6   s    
r$   c                 C   s   t jt| |||fd}|S )N)targetargs)multiprocessingProcessr$   )r#   r   r   r   pr   r   r   start_sync_process;   s    r*   c                 C   s@   t |d}|}t| | W d    n1 s20    Y  d S )Nwb)r   opentorchsave)Zpt_obj	file_pathoffr   r   r   pt_save@   s    r2   c                 C   sV   |  drtd t| d}|}tj||d}W d    n1 sH0    Y  |S )Nr   z0Loading remote checkpoint, which may take a bit.rb)map_location)
startswithr   r   r   r,   r-   load)r/   r4   r0   r1   outr   r   r   pt_loadE   s    

,r8   c                 C   sH   z.t |  W d    n1 s"0    Y  W n tyB   Y dS 0 dS )NFT)r   r,   FileNotFoundError)r/   r   r   r   check_existsM   s    "r:   )N)r   osr'   r   r!   r   r-   r   r   r   r    r$   r*   r2   r8   r:   r   r   r   r   <module>   s   


