a  dTA@sdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZddlZddlmZddlZddlmZddlmZmZmZzdddde=fd"d#Z?d$d%Z@e!durGd&d'd'e!jAZBndZBGd(d)d)eZCGd*d+d+e/ZDdS).zR Dataset reader for webdataset Hacked together by / Copyright 2022 Ross Wightman N) dataclass)partial)islice)AnyCallableDictListOptionalTuple)Image)DatasetIterableDatasetget_worker_info)_shuffle) expand_urls) base_plus_ext url_openertar_file_expander valid_sample)load_class_map)Reader) SharedCountZWDS_SHUFFLE_SIZEi infoc Cstj||d}tj||d}d}z:t|}t|}Wdn1sT0Y|WSty}zt|}WYd}~n d}~00z:t|}t |}Wdn1s0Y|WStyYn0t d|d|d|diS)Nz.jsonz.yamlzDataset info file not found at z or z . Error: z.. Falling back to provided split and size arg.) ospathjoinwdsZgopenjsonload ExceptionstryamlZ safe_load_loggerwarning)rootbasenameZ info_jsonZ info_yamlerr_strfZ info_dicter+e/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/data/readers/reader_wds.py _load_info)s& ( ( r-c@sJeZdZUeed<eeed<dZeeed<dZeed<dZ eed<dS) SplitInfo num_samples filenamesr+ shard_lengthsr alt_labelnameN) __name__ __module__ __qualname__int__annotations__r r"r1r2r3r+r+r+r,r.?s   r.)splitrc szdd}d|vsd|vr$|d}d}d}t|dkrDt|d}|d}d |vr|d d }t|d}|rd |vr|||d vr||}t|}|r|d |}|s d dt|d|dDtfdd|D}t|d<t|d<||d<||}nt |||d}nRd |vs<||d vr^t d|d| d id|}|d |}||}|S)NcSs2t|dt|dt|d|dd|ddS)Nr/r0r1r2rr3)r/r0r1r2r3)r.tupleget)Z dict_infor+r+r, _info_convertIs   z(_parse_split_info.._info_converttarz..|rrrz::-splitscSsi|]\}}||qSr+r+).0r)cr+r+r, ez%_parse_split_info..r0r1c3s|]}|VqdSNr+rBr)Z_fcr+r, frEz$_parse_split_info..r/)r3r/r0zsplit z not found in info ()) r9lenr7rzipsumr:keysvaluesr. RuntimeErrorr;) r9rr<r/Z split_name split_partsZ split_idxZsplit_filenames split_infor+rHr,_parse_split_infoHsD        " rScCstdt|ddS)zUCall in an exception handler to ignore any exception, isssue a warning, and continue.zHandling webdataset error (z ). Ignoring.T)r$r%repr)exnr+r+r,log_and_continue{srVjpgRGBclsrc Cs|r,t|d}t||}|dkr8dSn t||}t||"}t|}|Wdn1sn0Y|r||}t ||| ddd} | S)z Custom sample decode * decode and convert PIL Image * cls byte string label to int * pass through JSON byte string (if it exists) without parse rrN)rWrYr) rloadsr7ioBytesIOr openr convertdictr;) sample image_key image_format target_keyr2metaZ class_labelbimgdecodedr+r+r,_decodes    & rhc cs|D]}zt|||||d}WnHtyf}z0||rDWYd}~qnWYd}~qWYd}~n d}~00|durt|trt|tr|d|d<|VqdS)zDecode samples with skip.)rarbrcr2N__key__)rhr! isinstancer_r;) datararbrcr2handlerr`resultrUr+r+r,_decode_sampless" $rncCst}|dur|jStjS)z'get dataloader worker seed from pytorchN)rseedrutilspytorch_worker_seed) worker_infor+r+r,rqsrqc@seZdZd ddZddZd S) detshuffle2drcCs||_||_||_||_dSrF)bufsizeinitialroepoch)selfrwrxroryr+r+r,__init__szdetshuffle2.__init__cCsft|jtr|jj}n|jd7_|j}|jdkr@t|}n |j|}t|}t||j |j |SNrr) rjryrvaluerorqrandomRandomrrwrx)rzsrcryrorngr+r+r,runs      zdetshuffle2.runN)rtrurrv)r4r5r6r{rr+r+r+r,rss  rscs4eZdZdZejdddffdd ZddZZS) ResampledShards2z,An iterable dataset yielding a list of urls.NTrvcsbttj|}||_t|jdts0J||_t |_ |durLt n||_ ||_||_dS)zSample shards from the shard list with replacement. :param urls: a list of URLs as a Python list or brace notation string rN)superr{rZ shardlistsrurlsrjr"nshardsr~rrrq worker_seed deterministicry)rzrrrrry __class__r+r,r{s   zResampledShards2.__init__ccst|jtr|jj}n|jd7_|j}|jrDt|||_t |j D].}|j dt |j d}t|j |dVqNdS)z#Return an iterator over the shards.rr)urlN)rjryrr}rr~rrrrangerrandintrKrr_)rzry_indexr+r+r,__iter__ s  zResampledShards2.__iter__) r4r5r6__doc__sysmaxsizer{r __classcell__r+r+rr,rsrc sveZdZdfd d Zd d Zdeed ddZddZddZ ddZ ddZ ddZ d ddZ d!ddZZS)" ReaderWdsFNr*rWrXrYrcs2ttdurtd||_||_||_||_||_d|_ |pDt |_ | |_ | |_ | |_d|_d|_t|j|_t||j|_|jj|_|jstdd|_|rt||_d|_ni|_d|_d |_trtrtd krt|_t|_d|_ d|_!||_"d |_#d|_$d |_%d|_&t'|_(d|_)dS) NzcPlease install webdataset 0.2.x package `pip install git+https://github.com/webdataset/webdataset`.ifilenamez.JPEGz+Invalid split definition, no samples found.FTrr)*rr{rrPr& is_training batch_sizerepeats common_seedshard_shuffle_size SHUFFLE_SIZEsample_shuffle_sizerarbrc filename_keykey_extr-rrSrRr/ remap_classr class_to_idx dist_rankdist_num_replicasdistZ is_availableZis_initializedZget_world_sizeZget_rankrr worker_idr num_workersglobal_worker_idglobal_num_workersZ init_countr epoch_countds)rzr&r3r9rrrro class_mapZ input_nameZ input_imageZ target_nameZ target_imageZ prefetch_sizeZ shuffle_sizerr+r,r{sR       zReaderWds.__init__cCs ||j_dSrF)rr})rzcountr+r+r, set_epochaszReaderWds.set_epoch)rcCs.|jdurdS|dur*||_|j|j|_dSrF)rrrr)rzrr+r+r,set_loader_cfgds  zReaderWds.set_loader_cfgc sjdur^tjj}|dur<|_|j_|j_|j _ j j _ j j j_ fddjjD}t|g}jr|tjjjdjtjtdtjjtjdgn|jtjtdg|tt j!j"jj#dgtj$|_%dS)z8 Lazily initialize worker (in worker processes) Ncsg|]}tjj|qSr+)rrrr&rGrzr+r, |rEz(ReaderWds._lazy_init..)rory)rl)r)rarbr2)&rrtorchrprkridrrorrrrrrrRr0rZSimpleShardListrextendrsrrr_split_by_node_and_workerZtarfile_to_samplesrVshufflerr~rrrnrarbr2Z DataPipeliner)rzrrZabs_shard_filenamesZpipeliner+rr, _lazy_initnsD       zReaderWds._lazy_initccs>|jdkr*t||jd|jD] }|Vqn|D] }|Vq.dSNr)rrr)rzrsr+r+r,rs   z#ReaderWds._split_by_node_and_workercCs\|jt|j|j}|js$|jdkr.t|}|jrT|jdurTt||j|j}t|Sr) r/maxrrrmathceilrr7)rznum_worker_samplesr+r+r,_num_samples_per_workers  z!ReaderWds._num_samples_per_workerccs|jdur||}|js*|jdkr8|j|}n|j}d}|D]6}||j}|jrd|j|}||j |fV|d7}qFdSr|) rrrrrZ with_epochrcrrra)rzrrir`targetr+r+r,rs   zReaderWds.__iter__cCs||j}|SrF)rr)rzr/r+r+r,__len__szReaderWds.__len__cCsds JddS)NFz Not supportedr+)rzrr'absoluter+r+r, _filenameszReaderWds._filenamecCs||jdur|g}|jD]Z}|j|vr6||j}n$d|vrN|d|j}n dsZJd||t||jkrqxq|S)z0 Return all filenames in dataset, overrides baseNriFzNo supported name field present)rrrrappendrKr/)rzr'rnamesr`r3r+r+r,r0s      zReaderWds.filenames) FNrrNrWrXrYrNN)N)FF)FF)r4r5r6r{rr r7rrrrrrrr0rr+r+rr,rs0D + r)r)rWrXrYr)Err[rloggingrrr~rZ dataclassesr functoolsr itertoolsrtypingrrrrr r rZtorch.distributedZ distributedrr#ZPILr Ztorch.utils.datar r rZ webdatasetrZwebdataset.filtersrZwebdataset.shardlistsrZwebdataset.tariteratorsrrrr ImportErrorrrreaderrZ shared_countr getLoggerr4r$r7environr;rr-r.r"rSrVrhrnrqZ PipelineStagersrrr+r+r+r,sf              3 %  ,