a d$@sddlZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z ddl m Z ddlmZddlZddlmZddlmZmZmZddlmZddlmZdd lmZerdd lmZd ZGd d d eZ eeedddZ!GdddeZ"dS)N)AnyCallableOptionalSizedTupleUnion) HTTPError)warn)Tensor) DataLoaderDataset random_split) _IS_WINDOWS)LightningDataModule)_TORCHVISION_AVAILABLE) transformsz./datacseZdZdZdZdZdZdZd%ee e e e dd fd d Z e eee fd d dZe dddZeedddZee dddZd&e ddddZeddddZed'ee eeeefddd Zed(eee efee efed"d#d$ZZS))_MNISTzCarbon copy of ``tests_pytorch.helpers.datasets.MNIST``. We cannot import the tests as they are not distributed with the package. See https://github.com/Lightning-AI/lightning/pull/7614#discussion_r671183652 for more context. )zChttps://pl-public-data.s3.amazonaws.com/MNIST/processed/training.ptz?https://pl-public-data.s3.amazonaws.com/MNIST/processed/test.ptz training.ptztest.ptZcompleteTg_)Ǻ?gGr?N)roottrain normalizedownloadkwargsreturnc sZt||_||_||_|||jr2|jn|j}|t j |j |\|_ |_dSN)super__init__rrr prepare_dataTRAIN_FILE_NAMETEST_FILE_NAME _try_loadospathjoincached_folder_pathdatatargets)selfrrrrrZ data_file __class__q/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/demos/mnist_datamodule.pyr5s  z_MNIST.__init__)idxrcCsV|j|d}t|j|}|jdurNt|jdkrN|j|g|jR}||fS)Nr)r%floatZ unsqueezeintr&rlennormalize_tensor)r'r,imgtargetr*r*r+ __getitem__Bs z_MNIST.__getitem__rcCs t|jSr)r0r%r'r*r*r+__len__Ksz_MNIST.__len__cCstj|jd|jS)NMNIST)r!r"r#rcache_folder_namer6r*r*r+r$Nsz_MNIST.cached_folder_path) data_folderrcCs4d}|j|jfD]}|o,tjtj||}q|S)NT)rrr!r"isfiler#)r'r:existingfnamer*r*r+ _check_existsRsz_MNIST._check_exists)rrcCs4|r||js||j||js0tddS)NzDataset not found.)r>r$ _download RuntimeError)r'rr*r*r+rXs  z_MNIST.prepare_datacCsRtj|dd|jD]8}td|tj|tj|}tj ||qdS)NT)exist_okz Downloading ) r!makedirs RESOURCESlogginginfor"r#basenameurllibrequest urlretrieve)r'r:urlZfpathr*r*r+r?^s  z_MNIST._download?) path_datatrialsdeltarc Csd\}}|sJdtj|s.Jd|t|D]R}zt|}Wn:ty}z"|}t|t WYd}~q6d}~00qq6|dusJ|dur||S)zHResolving loading from the same time from multiple concurrent processes.)NNz!at least some trial has to be setzmissing file: N) r!r"r;rangetorchload Exceptiontimesleeprandom)rMrNrOres exception_exr*r*r+r es  ( z_MNIST._try_load)tensormeanstdrcCs8tj||j|jd}tj||j|jd}|||S)N)dtypedevice)rQZ as_tensorr_r`subdiv)r\r]r^r*r*r+r1zsz_MNIST.normalize_tensor)TrT)T)rKrL)r[rL)__name__ __module__ __qualname____doc__rCrrr9strbooltuplerrr/rr r4r7propertyr$r>rr? staticmethodr.r rr1 __classcell__r*r*r(r+r%s*    r)argsrrc Osttdd }|rlzddlm}|tddWn8tyj}z td|dd}WYd}~n d}~00|s|td t}||i|S) NZPL_USE_MOCKED_MNISTFr)r8T)rzError z) downloading `torchvision.datasets.MNIST`zD`torchvision.datasets.MNIST` not available. Using our hosted version) rhr!getenvZtorchvision.datasetsr8_DATASETS_PATHrprintr)rmrZtorchvision_mnist_availabler8er*r*r+r8s r8c seZdZdZdZedddddfeeeeeee e dd fd d Z e ed d dZ dd ddZ eddddZed ddZed ddZed ddZe eed ddZZS)MNISTDataModulezStandard MNIST, train, val, test splits and transforms. >>> MNISTDataModule() # doctest: +ELLIPSIS <...mnist_datamodule.MNISTDataModule object at ...> ZmnistiF* N) data_dir val_split num_workersrseed batch_sizermrrc sVtj|i||r.tr.td|dd}||_||_||_||_||_||_ dS)an Args: data_dir: where to save/load the data val_split: how many of the training images to use for the validation split num_workers: how many workers to use for loading data normalize: If true applies image normalize seed: starting seed for RNG. batch_size: desired batch size. zYou have requested num_workers=zA on Windows, but currently recommended is 0, so we set it for yourN) rrrr rvrwrxrryrz) r'rvrwrxrryrzrmrr(r*r+rs zMNISTDataModule.__init__r5cCsdS)N r*r6r*r*r+ num_classesszMNISTDataModule.num_classescCs$t|jdddt|jddddS)zSaves MNIST files to `data_dir`TrrFN)r8rvr6r*r*r+rszMNISTDataModule.prepare_data)stagercCsf|jrt|jdni}t|jfddd|}t|tss&        \