a
    dO                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlZd dlZd dlZd dlmZ d dlZd dlmZ d dlmZmZmZmZmZ d dlmZ d dlm Z  d dl!m"Z"m#Z#m$Z$m%Z% zd dl&mZ' W n e(y   dZ'Y n0 G d	d
 d
eZ)G dd dZ*e	G dd dZ+d7ddZ,dd Z-dd Z.dd Z/dd Z0dd Z1e"dddfddZ2e1fddZ3d8d d!Z4d"Z5d#Z6d$Z7d%Z8G d&d' d'ej9Z:G d(d) d)eZ;d9d+d,Z<d:d-d.Z=G d/d0 d0eZ>d;d1d2Z?d3d4 Z@d<d5d6ZAdS )=    N)	dataclass)Value)Image)Dataset
DataLoaderSubsetRandomSamplerIterableDatasetget_worker_info)DistributedSampler)_shuffle)base_plus_ext
url_openertar_file_expandervalid_samplec                   @   s&   e Zd Zd	ddZdd Zdd ZdS )

CsvDataset	Nc                 C   sV   t d| d tj||d}||  | _||  | _|| _t d || _d S )NzLoading csv data from .)sepzDone loading data.)	loggingdebugpdZread_csvtolistimagescaptions
transformstokenize)selfinput_filenamer   img_keycaption_keyr   	tokenizerZdf r!   V/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/training/data.py__init__   s    
zCsvDataset.__init__c                 C   s
   t | jS N)lenr   r   r!   r!   r"   __len__)   s    zCsvDataset.__len__c                 C   s<   |  tt| j| }| t| j| gd }||fS Nr   )r   r   openstrr   r   r   )r   idxr   textsr!   r!   r"   __getitem__,   s    zCsvDataset.__getitem__)r   N__name__
__module____qualname__r#   r'   r-   r!   r!   r!   r"   r      s   
r   c                   @   s,   e Zd Zd
edddZdd Zdd Zd	S )SharedEpochr   epochc                 C   s   t d|| _d S )Ni)r   shared_epochr   r4   r!   r!   r"   r#   3   s    zSharedEpoch.__init__c                 C   s   || j _d S r$   r6   valuer7   r!   r!   r"   	set_value6   s    zSharedEpoch.set_valuec                 C   s   | j jS r$   r8   r&   r!   r!   r"   	get_value9   s    zSharedEpoch.get_valueN)r   )r/   r0   r1   intr#   r:   r;   r!   r!   r!   r"   r2   2   s   r2   c                   @   s6   e Zd ZU eed< dZeed< dZeed< dd Z	dS )DataInfo
dataloaderNsamplerr6   c                 C   s<   | j d ur| j | | jd ur8t| jtr8| j| d S r$   )r6   r:   r?   
isinstancer
   	set_epochr7   r!   r!   r"   rA   C   s    
zDataInfo.set_epoch)
r/   r0   r1   r   __annotations__r?   r
   r6   r2   rA   r!   r!   r!   r"   r=   =   s   
r=   c           	         s   |d u rt j| }|d fS t| tr| d}|d}t|t|kshJ dt| dt| ddd |D }g g  }}t||D ]<\} tt		|} fdd|D }|
| |
| q||fS t| }||fS d S )Nz::z(Expected the number of data components (z) and weights(z) to match.c                 S   s   g | ]}t |qS r!   )float).0weightr!   r!   r"   
<listcomp>S       zexpand_urls.<locals>.<listcomp>c                    s   g | ]} qS r!   r!   )rD   _rE   r!   r"   rF   W   rG   )wdsZ
shardlistsexpand_urlsr@   r*   splitr%   ziplistbraceexpandextend)	urlsweightsZexpanded_urlsZurllistZall_urlsZall_weightsurlZexpanded_urlZexpanded_weightsr!   rI   r"   rK   J   s&    




rK   c                    s   t | \}}tj|d }tj|d}tj|d}tj|rltt|d t	 fdd|D }n&tj|rt
t|d }nd }t|}||fS )Nr   z
sizes.jsonr'   rc                    s    g | ]}t  tj| qS r!   )r<   ospathbasename)rD   Zshardsizesr!   r"   rF   g   rG   z$get_dataset_size.<locals>.<listcomp>)rK   rU   rV   dirnamejoinexistsjsonloadr)   sumastliteral_evalreadr%   )ZshardsZshards_listrH   Zdir_pathZsizes_filenameZlen_filenameZ
total_size
num_shardsr!   rX   r"   get_dataset_size`   s    rd   c                 C   s(  |dv sJ |dk}|\}}|dkr@ddl m} || j|d}n0|rP| j}|}	n
| j}|}	|sbJ tj||	d}|rtt	|j
}
t|j
}d}td	D ]B}||k}t	|
| }t|}d
|d |< tj| ||
|< q|
d}
tt|
d }nd }tjjj|| j| j|d}t||dS )N)trainvalv2re   rg   r   )ImageNetV2Dataset)location	transform)rj   2        r<   )
batch_sizenum_workersr?   )r>   r?   )Zimagenetv2_pytorchrh   imagenet_v2Zimagenet_trainimagenet_valdatasetsZImageFoldernpzerosr%   targetsarrayrangerandomshuffleZastyper   wheretorchutilsdatar   rn   workersr=   )argspreprocess_fnsrL   is_trainpreprocess_trainpreprocess_valrh   dataset	data_pathpreprocess_fnZidxsZtarget_arraykcmnZarrr?   r>   r!   r!   r"   get_imagenetv   sD    


r   c                 C   sP   dt jd< d\}}| D ]0\}}|d7 }|t|7 }t|t|ksJ q||fS )N0Z	WDS_EPOCH)r   r   rm   )rU   environr%   )r>   Z
n_elementsZ	n_batchesr   r,   r!   r!   r"   count_samples   s    
r   c                 C   s0   d| v }d| v p&d| v p&d| v p&d| v }|o.|S )NtxtZpngZjpgZjpegZwebpr!   )sampleZhas_captionZ	has_imager!   r!   r"   filter_no_caption_or_no_image   s     r   c                 C   s   t dt|  d dS )zTCall in an exception handler to ignore any exception, issue a warning, and continue.zHandling webdataset error (z). Ignoring.T)r   warningrepr)Zexnr!   r!   r"   log_and_continue   s    r   Tc                 c   s   d}| D ]}t |tsJ |d |d  }}||\}	}
|	du rBq|rN|
 }
|du sj|	|d ksj|
|v rt|rx|V  t|	|d d}|du s|
|v r|||
< qt|r|V  dS )zReturn function over iterator that groups key, value pairs into samples.

    :param keys: function that splits the key into key and extension (base_plus_ext)
    :param lcase: convert suffixes to lower case (Default value = True)
    Nfnamer}   __key____url__)r   r   )r@   dictlowerr   )r}   keysZlcasesuffixeshandlerZcurrent_sampleZ
filesampler   r9   prefixsuffixr!   r!   r"   group_by_keys_nothrow   s"    
r   c                 C   s(   t | |d}t||d}t||d}|S )Nr   )r   r   r   )srcr   ZstreamsfilesZsamplesr!   r!   r"   tarfile_to_samples_nothrow   s    r   c                 C   s:   t  }|dur0|j}| r,|| td|j 7 }|S tj S )z'get dataloader worker seed from pytorchNrm   )r	   seedmaxro   rJ   r|   pytorch_worker_seed)	incrementZworker_infor   r!   r!   r"   r      s    r   i  i  i  rl   c                   @   s   e Zd Zd
ddZdd Zd	S )detshuffle2rl   d   r   c                 C   s   || _ || _|| _|| _d S r$   bufsizeinitialr   r4   )r   r   r   r   r4   r!   r!   r"   r#      s    zdetshuffle2.__init__c                 C   sn   t | jtr| j }n|  jd7  _| j}t }| jdk rHt|}n
| j| }|| t|| j	| j
|S )Nrm   r   )r@   r4   r2   r;   rx   Randomr   r   r   r   r   )r   r   r4   rngr   r!   r!   r"   run   s    



zdetshuffle2.runN)rl   r   r   r   )r/   r0   r1   r#   r   r!   r!   r!   r"   r      s       
r   c                       s6   e Zd ZdZdejdddf fdd	Zdd Z  ZS )	ResampledShards2z,An iterable dataset yielding a list of urls.NFr   c                    s   t    t||\}}|| _|| _| jdurdt| jt| jksdJ dt| j dt| j dt| jd tsxJ || _t	
 | _|| _|| _|| _dS )zSample shards from the shard list with replacement.

        :param urls: a list of URLs as a Python list or brace notation string
        NzNumber of urls z and weights z should match.r   )superr#   rK   rQ   rR   r%   r@   r*   nshardsrx   r   r   worker_seeddeterministicr4   )r   rQ   rR   r   r   r   r4   	__class__r!   r"   r#     s    


zResampledShards2.__init__c                 c   s   t | jtr| j }n|  jd7  _| j}| jr^| jdu rFt|}n|  | }| j| t	| j
D ]H}| jdu rt| j| jdV  qht| jj| j| jddd dV  qhdS )z#Return an iterator over the shards.rm   N)rS   )rR   r   r   )r@   r4   r2   r;   r   r   r   r   r   rw   r   rR   r   choicerQ   choices)r   r4   r   rH   r!   r!   r"   __iter__0  s    


zResampledShards2.__iter__)	r/   r0   r1   __doc__sysmaxsizer#   r   __classcell__r!   r!   r   r"   r     s   r   Fc                    sH  |r
| j n| j}|d usJ t| ddo*|}d }|r`| jd urF| j}	qjt|\}	}|	sjtdn
| jphd}	t|d}
|rt|| j	d|
dg}n| j	d u sJ dt
|g}|r|s|ttt| j|
d	t
jt
jg |tt
jttd
g n|t
jt
jtdg |t
tt
jdtdt
jdddt
j| fdddt
ddt
j| j | dg t
j!| }|r|s|pt"t#|d }|| j$| j% ksJ d|rt&j'nt&j(}| j | j% }||	| }t)d| j$}||| }|| }|| }	|*|}nt&(|	| j  }t
j+|d d| j$| j$dkd}||_,|	|_-t.||
dS )NZdataset_resampledFzCurrently, the number of dataset samples must be specified for the training dataset. Please specify it via `--train-num-samples` if no dataset length info is present.r   r3   T)rR   r   r4   zl--train_data_upsampling_factors is only supported when sampling with replacement (with --dataset-resampled).r   )r   r   r   Zpilrgbzjpg;png;jpeg;webpr   )imagetextc                    s    | d S r(   r!   r   r    r!   r"   <lambda>  rG   z!get_wds_dataset.<locals>.<lambda>r   r   )partialz)number of shards must be >= total workersrm   )rn   ry   ro   Zpersistent_workers)r>   r6   )/
train_dataval_datagetattrtrain_num_samplesrd   RuntimeErrorZval_num_samplesr2   r   Ztrain_data_upsampling_factorsrJ   ZSimpleShardListrP   r   _SHARD_SHUFFLE_SIZE_SHARD_SHUFFLE_INITIALr   Zsplit_by_nodeZsplit_by_workerr   ry   _SAMPLE_SHUFFLE_SIZE_SAMPLE_SHUFFLE_INITIALZtarfile_to_samplesr   selectr   decoderenameZmap_dictZto_tupleZbatchedrn   ZDataPipeliner%   rK   r~   Z
world_sizemathfloorceilr   Z
with_epochZ	WebLoadernum_batchesnum_samplesr=   )r   Zpreprocess_imgr   r4   r   r    Zinput_shardsZ	resampledrc   r   r6   Zpipeliner   Zround_fnZglobal_batch_sizer   ro   Znum_worker_batchesr>   r!   r   r"   get_wds_datasetH  s    




	

	
r   c              	   C   s   |r
| j n| j}|sJ t||| j| j| j|d}t|}| jrL|rLt|nd }|oZ|d u }	t	|| j
|	| jd||d}
||
_t|
|
_t|
|S )N)r   r   r   r    Trn   ry   ro   Z
pin_memoryr?   Z	drop_last)r   r   r   Zcsv_img_keyZcsv_caption_keyZcsv_separatorr%   distributedr
   r   rn   r~   r   r   r=   )r   r   r   r4   r    r   r   r   r?   ry   r>   r!   r!   r"   get_csv_dataset  s2    	
r   c                   @   s&   e Zd ZdddZdd Zd	d
 ZdS )SyntheticDatasetN   r   Dummy captionr   c                    s8   || _ || _|| _td|| _|| _ fdd| _d S )NZRGBc                    s    | d S r(   r!   r   r   r!   r"   r     rG   z+SyntheticDataset.__init__.<locals>.<lambda>)rj   
image_sizecaptionr   newr   dataset_sizepreprocess_txt)r   rj   r   r   r   r    r!   r   r"   r#     s    zSyntheticDataset.__init__c                 C   s   | j S r$   )r   r&   r!   r!   r"   r'     s    zSyntheticDataset.__len__c                 C   s&   | j d ur|  | j}|| | jfS r$   )rj   r   r   r   )r   r+   r   r!   r!   r"   r-     s    
zSyntheticDataset.__getitem__)Nr   r   r   Nr.   r!   r!   r!   r"   r     s        
r   c              	   C   s|   |j d j}t||| j|d}t|}| jr8|r8t|nd }|oF|d u }	t|| j|	| j	d||d}
||
_
t|
|
_t|
|S )Nr   )rj   r   r   r    Tr   )r   sizer   r   r%   r   r
   r   rn   r~   r   r   r=   )r   r   r   r4   r    r   r   r   r?   ry   r>   r!   r!   r"   get_synthetic_dataset  s&    
	
r   c                 C   sv   |dkrt S |dkrtS |dkr$tS |dkrd| dd }|dv rFtS |dv rRt S td	| dntd
| d S )N
webdatasetcsv	syntheticautor   r   )r   Ztsv)tarz;Tried to figure out dataset type, but failed for extension zUnsupported dataset type: )r   r   r   rL   
ValueError)r   dataset_typeextr!   r!   r"   get_dataset_fn  s     
r   c                 C   s   |\}}i }| j s| jdkr<t| j | j| |d||d|d< | jr`t| j| j| |d|d|d< | jd urzt| |d|d< | jd urt| |d	|d
< |S )Nr   T)r   r4   r    re   F)r   r    rf   zimagenet-valrg   zimagenet-v2)r   r   r   r   rq   r   rp   )r   r   r4   r    r   r   r}   r!   r!   r"   get_data!  s    




r   )N)r   )r   FN)r   N)r   N)r   N)Br`   r]   r   r   rU   rx   r   rO   Zdataclassesr   multiprocessingr   numpyrs   Zpandasr   r{   Ztorchvision.datasetsrr   r   rJ   ZPILr   Ztorch.utils.datar   r   r   r   r	   Ztorch.utils.data.distributedr
   Zwebdataset.filtersr   Zwebdataset.tariteratorsr   r   r   r   Zhorovod.torchZhvdImportErrorr   r2   r=   rK   rd   r   r   r   r   r   r   r   r   r   r   r   ZPipelineStager   r   r   r   r   r   r   r   r!   r!   r!   r"   <module>   s`   

.

 6
u

