a
    
dL                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlmZmZ d dlZddlmZmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& eG dd dZ'G dd dej(Z)G dd dej*j+Z,e-e-dddZ.dS )    N)	dataclass)datetime)	has_magic)AnyDictListOptionalTupleUnion)quoteunquote   )CommitOperationCopyCommitOperationDelete)DEFAULT_REVISIONENDPOINTREPO_TYPE_MODELREPO_TYPES_MAPPINGREPO_TYPES_URL_PREFIXES)HfApi)EntryNotFoundErrorHFValidationErrorRepositoryNotFoundErrorRevisionNotFoundErrorhf_raise_for_statushttp_backoffpaginateparse_datetimec                   @   s@   e Zd ZU dZeed< eed< eed< eed< edddZd	S )
HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path.	repo_typerepo_idrevisionpath_in_reporeturnc                 C   s2   t | jd| j  dt| j d| j dS )N @/)r   getr   r    
safe_quoter!   r"   rstripself r-   g/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/huggingface_hub/hf_file_system.py	unresolve$   s    ,z"HfFileSystemResolvedPath.unresolveN)__name__
__module____qualname____doc__str__annotations__r/   r-   r-   r-   r.   r      s   
r   c                       s  e Zd ZdZdZdZdddee ee d fddZeeee e	e
ee f dd	d
Zd-eee edddZd.ee ddddZd/eeee ddddZd0eee ddddZd1ee
ee ee ddddZd2ee
e
ee eeeeeef f  dddZd3eee d d!d"Zd4eeee dd#d$d%Zeedd&d'Zeeeef d fd(d)Zd5eeee f e
ee ee d*d+d,Z  ZS )6HfFileSystema  
    Access a remote Hugging Face Hub repository as if were a local file system.

    Args:
        endpoint (`str`, *optional*):
            The endpoint to use. If not provided, the default one (https://huggingface.co) is used.
        token (`str`, *optional*):
            Authentication token, obtained with [`HfApi.login`] method. Will default to the stored token.

    Usage:

    ```python
    >>> from huggingface_hub import HfFileSystem

    >>> fs = HfFileSystem()

    >>> # List files
    >>> fs.glob("my-username/my-model/*.bin")
    ['my-username/my-model/pytorch_model.bin']
    >>> fs.ls("datasets/my-username/my-dataset", detail=False)
    ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json']

    >>> # Read/write files
    >>> with fs.open("my-username/my-model/pytorch_model.bin") as f:
    ...     data = f.read()
    >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f:
    ...     f.write(data)
    ```
    r%   ZhfNendpointtokenc                   s:   t  j|i | |pt| _|| _t||d| _i | _d S )Nr7   )super__init__r   r8   r9   r   _api_repo_and_revision_exists_cache)r,   r8   r9   argsZstorage_options	__class__r-   r.   r;   M   s    
zHfFileSystem.__init__)r   r    r!   r$   c              
   C   s   |||f| j vrz| jj|||d W n ttfyv } z4d|f| j |||f< d|f| j ||d f< W Y d }~npd }~0  ty } z0d|f| j |||f< d| j ||d f< W Y d }~n*d }~0 0 d| j |||f< d| j ||d f< | j |||f S )N)r!   r   F)TN)r=   r<   Z	repo_infor   r   r   )r,   r   r    r!   er-   r-   r.   _repo_and_revision_exist_   s    (&z%HfFileSystem._repo_and_revision_exist)pathr!   r$   c                 C   s*  t t t t t t ddd}| |}|s6tdnH|dd d t v rzd|vr`td|dd\}}t| }nt}|	ddkrd	|v r|d	d\}}d|v r|dd\}}nd
}t
|}|||}| |||\}}	|st||	nd|dd d }
d|ddd  }|dd }d|ddd  }|
}|}| |||\}}	|s
t|	ttfr|}|}| |||\}}|st||	n
t||	nT|}d
}d	|v r|d	d\}}t
|}|||}| |||\}}|s
td|d ur|nt}t||||S )N)revision_in_pathr!   r$   c                 S   s8   |d ur0| d ur4| |kr4t d|  d| dn| }|S )NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.)
ValueError)rD   r!   r-   r-   r.   %_align_revision_in_path_with_revisionq   s    zHHfFileSystem.resolve_path.<locals>._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r'   r   z/Acces to repositories lists is not implemented.r   r&   r%      )r   r4   _strip_protocolNotImplementedErrorsplitr   valuesr   r   countr   rB   FileNotFoundErrorjoin
isinstancer   r   r   r   )r,   rC   r!   rF   r   r    rD   r"   Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r-   r-   r.   resolve_pathp   sb    





zHfFileSystem.resolve_path)rC   r$   c                 C   sJ   |s| j   | j  n,| | }|rF| j |d  | |}q(d S )N)dircacheclearZ$_repository_type_and_id_exists_cacherR   r/   pop_parent)r,   rC   r-   r-   r.   invalidate_cache   s    
zHfFileSystem.invalidate_cacherbHfFileSystemFile)rC   moder!   r$   c                 K   s(   |dkrt dt| |f||d|S )Nabz/Appending to remote files is not yet supported.)rZ   r!   )rI   rY   )r,   rC   rZ   r!   kwargsr-   r-   r.   _open   s    zHfFileSystem._openc              
   K   sR   | j ||d}| jj|j|j| j|j|j|d|dd | j	|
 d d S )Nr!   commit_messagecommit_description)r"   r    r9   r   r!   r_   r`   rC   )rR   r<   Zdelete_filer"   r    r9   r   r!   r(   rW   r/   )r,   rC   r!   r\   resolved_pathr-   r-   r.   _rm   s    	zHfFileSystem._rmF)rC   	recursivemaxdepthr!   r$   c              
      s   j ||d}t|jd|j  j||||jd} fdd|D }dd |D }	d| d}
|
|rnd	nd7 }
|
|d urd
| dnd7 }
jj|j|jj	|	|j|d|
|dd j
| d d S )Nr^   r%   )rd   re   r!   c                    s*   g | ]"} |s|t d  d qS )r   N)isdirlen).0rC   	root_pathr,   r-   r.   
<listcomp>       z#HfFileSystem.rm.<locals>.<listcomp>c                 S   s   g | ]}t |d qS ))r"   )r   )rh   r"   r-   r-   r.   rk      rl   zDelete  zrecursively zup to depth r_   r`   )r    r   r9   
operationsr!   r_   r`   ra   )rR   r   r(   r   r    expand_pathr!   r<   create_commitr9   rW   r/   )r,   rC   rd   re   r!   r\   rb   pathsZpaths_in_reporn   r_   r-   ri   r.   rm   s$    
	zHfFileSystem.rmT)rC   detailrefreshr!   r$   c                    sp  | j ||d}dt|j   |v }| }|| jvs<|r6t|j|j|jd d }|}	| j|	|jd}
zt	|
}W n: t
y   d|jv r| |}	| j|	|jd}
n Y n0 t|g|
}
g }|
D ]`}||d  |d |d d}|d d	kr ||d
 |dt|d d d || q|| j|	< | |}|sX fdd|D }|rb|S dd |D S )z!List the contents of a directory.r^   r&   r%   r'   rC   sizetypenameru   rv   fileoidlfsZ
lastCommitdate)Zblob_idr{   last_modifiedc              	      s*   g | ]"}i |d |d    ddiqS )rx   r%   r   )replacerh   orD   r-   r.   rk   !  rl   z#HfFileSystem.ls.<locals>.<listcomp>c                 S   s   g | ]}|d  qS )rx   r-   r   r-   r-   r.   rk   "  rl   )rR   r)   r!   r/   rS   r   r   r    
_iter_treenextr   r"   rV   	itertoolschainupdater(   r   appendZ_ls_from_cache)r,   rC   rs   rt   r!   r\   rb   has_revision_in_pathZpath_prefixZ	tree_pathZ	tree_iterZ	tree_itemZchild_infosZ
child_infooutr-   r   r.   ls   sP    




zHfFileSystem.ls)rC   r!   c              	   c   sj   | j ||d}| jj d|j d|j dt|j d|j 	d}| j	 }t
|ddi|dE d H  d S )	Nr^   z/api/zs/z/tree/r'   expandT)paramsheaders)rR   r<   r8   r   r    r)   r!   r"   r*   _build_hf_headersr   )r,   rC   r!   rb   r   r-   r-   r.   r   $  s    0
zHfFileSystem._iter_tree)path1path2r!   r$   c                 K   sD  | j ||d}| j ||d}|j|jko2|j|jk}|r| j||jdd d urd| d| }| jj|j|j|j|d||ddt|j	|j	|jdgd	 n~| j
|d
|jd}	|	 }
W d    n1 s0    Y  d| d| }| jj|
|j	|j| j|j|j|d||dd | j| d | j| d d S )Nr^   r{   zCopy z to r_   r`   r%   )Zsrc_path_in_repor"   Zsrc_revision)r    r   r!   r_   r`   rn   rX   Zpath_or_fileobjr"   r    r9   r   r!   r_   r`   ra   )rR   r   r    infor!   r<   rp   r(   r   r"   openreadupload_filer9   rW   r/   )r,   r   r   r!   r\   Zresolved_path1Zresolved_path2Z	same_repor_   fcontentr-   r-   r.   cp_file.  sD    

&

zHfFileSystem.cp_filec                 K   s*   | j |fi |}d|vr"t||d S )Nr}   )r   IsADirectoryError)r,   rC   r\   r   r-   r-   r.   modifiedX  s    zHfFileSystem.modifiedc                    sd   |  |}|jsPdt|j }||v }| }|s@||ddn|}|dddS t j|fi |S )Nr&   r%   r   r   	directoryrw   )rR   r"   r)   r!   r/   r~   r:   r   )r,   rC   r\   rb   rD   r   rx   r?   r-   r.   r   ^  s    
zHfFileSystem.info)rC   rd   re   r$   c           	         s  |d ur|dk rt dt|tr2 |g||S t } fdd|D }|D ]}t|rt j|fi |}||O }|rN|t jt|f||d|O }qNn*|rt j|f|ddd|}||O }||vrN|du s 	|rN|
| qN|st|tt|S )	Nr   zmaxdepth must be at least 1c                    s   g | ]}  |qS r-   )rH   )rh   pr+   r-   r.   rk   s  rl   z,HfFileSystem.expand_path.<locals>.<listcomp>)rd   re   TF)re   Zwithdirsrs   )rE   rO   r4   ro   setr   globlistfindexistsaddrM   sorted)	r,   rC   rd   re   r\   r   r   bitZrecr-   r+   r.   ro   h  s*    
$zHfFileSystem.expand_path)N)N)rX   N)N)FNN)TFN)N)N)FN) r0   r1   r2   r3   Zroot_markerprotocolr   r4   r;   r	   bool	ExceptionrB   r   rR   rW   r]   rc   intrr   r   r
   r   r   r   r   r   r   r   r   ro   __classcell__r-   r-   r?   r.   r6   +   s^   
E      1
* r6   c                       s^   e Zd Zdeeee d fddZeeedddZ	ddd	d
Z
deddddZ  ZS )rY   N)fsrC   r!   c                    s.   t  j||fi | |  |j||d| _d S )Nr^   )r:   r;   rR   rb   )r,   r   rC   r!   r\   r?   r-   r.   r;     s    zHfFileSystemFile.__init__)startendr$   c                 C   s   dd| d|d  i| j j }| j j dt| jjd| jj  dt	| jj
 dt	| jj }td||d	}t| |jS )
Nrangezbytes=-r   r'   r%   z	/resolve/GET)r   )r   r<   r   r8   r   r(   rb   r   r    r)   r!   r"   r   r   r   )r,   r   r   r   urlrr-   r-   r.   _fetch_range  s    
@zHfFileSystemFile._fetch_ranger#   c                 C   s   t jddd| _d S )Nzhffs-F)prefixdelete)tempfileNamedTemporaryFile	temp_filer+   r-   r-   r.   _initiate_upload  s    z!HfFileSystemFile._initiate_uploadF)finalr$   c                 C   s   | j d | j  }| j| |r| j  | jjj| jj	| j
j| j
j| jj| j
j| j
j| jd| jdd t| jj	 | jj| j
 d d S )Nr   r_   r`   r   ra   )bufferseekr   r   writecloser   r<   r   rx   rb   r"   r    r9   r   r!   r\   r(   osremoverW   r/   )r,   r   blockr-   r-   r.   _upload_chunk  s&    




zHfFileSystemFile._upload_chunk)N)F)r0   r1   r2   r6   r4   r   r;   r   bytesr   r   r   r   r   r-   r-   r?   r.   rY     s   rY   )sr$   c                 C   s   t | ddS )Nr%   )safe)r   )r   r-   r-   r.   r)     s    r)   )/r   r   r   Zdataclassesr   r   r   r   typingr   r   r   r   r	   r
   urllib.parser   r   ZfsspecZ_commit_apir   r   	constantsr   r   r   r   r   Zhf_apir   utilsr   r   r   r   r   r   r   r   r   ZAbstractFileSystemr6   specZAbstractBufferedFilerY   r4   r)   r-   r-   r-   r.   <module>   s&    (  ]+