a
    d                     @   s  U d dl mZmZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZmZ ddlmZ g Zee e d< ej!edddZ"ej#edddZ$eedddZ%eeedddZ&eeeedddZ'eej#edddZ(eedd d!Z)d"d# Z*d$d% Z+eeee ee d&d'd(Z,eed)d*d+Z-eeee d,d-d.Z.eeeee d/d0d1Z/dS )2    )ListAnyN)ShardMetadata)ShardedTensor)TensorProperties)Shard)"_check_shard_metadata_pair_overlap   )LoadItemTypeSavePlanReadItem	WriteItemWriteItemTypeTensorWriteData)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPES)+_shards_get_overlap_region_wrt_saved_tensor__all__)sizereturnc                 C   s   t dgt|  t| dS )Nr   )shard_offsetsshard_sizes)r   lenlist)r    r   u/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_shard_metadata%   s    r    )tensorr   c                 C   s   t | t|  dS )N)r!   metadata)r   r    r   )r!   r   r   r   _create_shard_from_tensor,   s    r#   )shard_mdr   c                 C   s   t t| jt| jdS )Noffsetssizes)r   torchSizer   r   )r$   r   r   r   _chunk_for_shard0   s    

r*   )sharded_tensorr$   r   c                 C   s   t t||  j|  jdS )NchunkZ
propertiesr   )r   r*   r"   Ztensor_propertiesr   )r+   r$   r   r   r   _sharded_tensor_metadata7   s
    r.   )fqnr+   r$   r   c                 C   s(   t |j}tt| |tjt||dS )NindextypeZtensor_data)r(   r)   r   r   r   r   ZSHARDr.   )r/   r+   r$   r&   r   r   r   _create_write_item_for_shardA   s    r3   )r/   r!   r   c                 C   sN   t dgt|  }tt| |tjtt	|| dt
|| ddS )Nr   r%   r,   r0   )r(   r)   r   r   r   r   r   TENSORr   r   r   Zcreate_from_tensor)r/   r!   r&   r   r   r   _create_write_item_for_tensorL   s    r5   r/   bytesc                 C   s   t t| tjdS )N)r1   r2   )r   r   r   BYTE_IOr6   r   r   r   _create_write_item_for_bytesioY   s    r9   c              	   C   s.   t tj| t|f|t|ft|fdS N)r2   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r
   r8   r(   r)   r;   Zdest_offsetr=   Zstorage_offsetlengthr   r   r   _create_read_item_for_byteio`   s    


rB   c              	   C   s(   t tj| t||t|t|dS r:   )r   r
   r4   r(   r)   r;   r<   r=   r>   r?   r   r   r   _create_read_item_for_tensorm   s    rD   )r/   checkpoint_mdlocal_shardsr   c                 C   s   g }t |D ]\}}t |jD ]\}}tt|jt|jd}t|j|sLqg }	g }
g }t||jdD ]*\}}}}|		| |
	| |	| qf|	t
t| |jj||
t| |j||	|d qq|S )N)r   r   )Zsaved_shardZcurrent_shardrC   )	enumeratechunksr   r   r'   r&   r   r"   r   appendrD   r   r   )r/   rE   rF   Z
read_itemsidxshardZstorage_idxZ
storage_mdZshard_md_from_storager>   r<   r?   ZdimZoffset_for_saved_tensorZoffset_for_current_tensorrA   r   r   r   _create_sharded_read_itemsz   sN    


rL   )
state_dictr   c                 C   sz   g }|   D ]d\}}t|trB| jD ]}|t||| q(qt|tjr`|t	|| q|t
|| qt|S )N)items
isinstancer   r"   Zshards_metadatarI   r3   r(   Tensorr5   r9   r   )rM   requestsr/   objr$   r   r   r   "_create_default_metadata_only_plan   s    

rS   )r/   objectr   c                    sJ   t tr" fdd D S t tjr:t gS t gS d S )Nc                    s   g | ]}t  |jqS r   )r3   r"   ).0rK   r/   rT   r   r   
<listcomp>   s   z'_create_write_items.<locals>.<listcomp>)rO   r   rF   r(   rP   r5   r9   rV   r   rV   r   _create_write_items   s    
rX   )r/   mdrR   r   c                 C   s|   t |tr&tt| dt| dddgS t |tr:| }n6t |tjrRt|g}nt	d|  ddt
|  t| ||S )Nr   r@   z Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found )rO   r   rB   r   r   rF   r(   rP   r#   
ValueErrorr2   rL   )r/   rY   rR   rF   r   r   r   _create_read_items   s&    
	


r[   )0typingr   r   r(   Z!torch.distributed._shard.metadatar   Z'torch.distributed._shard.sharded_tensorr   Z0torch.distributed._shard.sharded_tensor.metadatar   Z-torch.distributed._shard.sharded_tensor.shardr   Z1torch.distributed._shard.sharding_spec._internalsr   Zplannerr
   r   r   r   r   r   r"   r   r   r   r   r   r   Z
reshardingr   r   str__annotations__r)   r    rP   r#   r*   r.   r3   r5   r9   rB   rD   rL   rS   rX   r[   r   r   r   r   <module>   s>    	 	3