a dUx@sdZddlZddlZddlZddlZddlZddlmZmZm Z ddl m Z ddl m Z ddlmZddlmZddlmZmZmZmZmZmZmZmZdd lmZddlZddlmZ dd l!m"Z"dd l#m$Z$dd l%m&Z&dd lm'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;mZ>m?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZKmLZLddlMmNZNddlOmPZPddlQmRZRmSZSdd lTmUZUmVZVmWZWmXZXdd!lYmZZZdd"l[m\Z\m]Z]m^Z^m_Z_m`Z`dd#lambZbmcZcdd$ldmeZedd%lfmgZgmhZhmiZimjZjdd&lkmlZldd'lmmnZndd(lompZpdd)lqmrZrdd*lsmtZtmuZumvZvdd+lwmxZxdd,lymzZzm{Z{m|Z|m}Z}dd-l~mZdd.lmZmZdd/lmZmZdd0lmZmZmZmZmZdd1lmZdd2lmZdd3lmZmZdd4lmZdd5lmZdd6lmZmZmZdd7lmZdd8lmZmZmZmZmZeeZejd9d:d;Gdd?Z4ee#dd@dAZ5dddBdCZ6dddDdEZ7dddFdGZ8dddHdIZ9ddJee)ede)e)dKdLdMZ:ee)e)e)dNdOdPZ;ee)e)ddNdQdRZeee)fddUdVdWZ?eee)fddUdXdYZ@eee)fddUdZd[ZAee)e)e)dNd\d]ZBeCedd^d_d`ZDdddadbZEdeddddcddZFdeddddedfZGdeddddgdhZHdeddddidjZIeJeddkdlZKeJeddmdnZLeJeMddodpZNeJe ddqdrZOeJe ddsdtZPeJe ddudvZQeJe ddwdxZReJe ddydzZSeJe e dd{d|ZTeJe dd}d~ZUeJddddZVeJe eWdddZXeXjYe eWddddZXeJe eZdddZ[eJe e dddZ\e\jYe e ddddZ\eJeedddZ]eJe^dddZ_eJee)dddZ`eJeeajbjcdddZdedjYeajbjcddddZdeJeedddZeeJedddZfeJeeee)fdddZgeJedddZheJedddZieJedddZjeJeekdddZleJe ekdddZmeJe endddZoeJeepdddZqeJe epdddZreJeesdddZteJeeeefdddZueJeedddZvde eee)ddddZwexe=dddZyexe)eeze{fe)e)dddZ|exee{ezfezdddZ}exezdddZ~exe{e)eee{fdddZeJedddZeJedddÄZejYeddĜddÄZeJedddDŽZejYeddĜddDŽZeJedddʄZejYeddĜddʄZeJeddd̈́ZejYeddĜdd̈́ZeJedddЄZejYeddĜddЄZeJedddӄZeJedddՄZejYeddĜddՄZeJe ddd؄ZeJe dddڄZeJee ddd܄ZeJee dddބZeJe dddZeJee dddZeJedddZeJedddZejYeddddZeJedddZejYeddddZeJedddZejYeddddZeJedddZejYeddddZeJedddZeJeeeeefdddZeJeedddZejYeeddddZeJe edddZejYee eddddZeJedddZeJedddZeJedddZeJeedddZddddZeddd ZeJee e fdd d ZZS(TrainerTNF2 rmax_size_cycle)4loggerenable_checkpointing callbacksdefault_root_dirgradient_clip_valgradient_clip_algorithm num_nodes num_processesdevicesgpusauto_select_gpus tpu_coresipusenable_progress_baroverfit_batchestrack_grad_normcheck_val_every_n_epoch fast_dev_runaccumulate_grad_batches max_epochs min_epochs max_steps min_stepsmax_timelimit_train_batcheslimit_val_batcheslimit_test_batcheslimit_predict_batchesval_check_intervallog_every_n_steps acceleratorstrategysync_batchnorm precisionenable_model_summarynum_sanity_val_stepsresume_from_checkpointprofiler benchmark deterministic!reload_dataloaders_every_n_epochs auto_lr_findreplace_sampler_ddpdetect_anomalyauto_scale_batch_sizeplugins amp_backend amp_levelmove_metrics_to_cpumultiple_trainloader_modeinference_modereturnc46spttdt|jjdtt |_ |durHt |}t ||2|_t|| | | || | ||!|'|+|(| |"|/|0|.d|_t||_t||_t||%|_t||_t||_t||d}4t||d}5|4j|5d|4|_t|_ t|_!t"|_#d|_$|j%|||||#||||j%||)||durRt&|t't(fsRt)d|d |durt*+|,st-d |d t*.d |d krt&|t't(fs|d krt(|dkst-d|d ||_/|durt*|,nd|_0t(||_1|3|_2|,|_3|4|j%|*|-t56||&||j%|||1|||||||t57|||||||||$ dS)a3 Customize every aspect of training via flags. Args: accelerator: Supports passing different accelerator types ("cpu", "gpu", "tpu", "ipu", "hpu", "mps", "auto") as well as custom accelerator instances. accumulate_grad_batches: Accumulates grads every k batches or as set up in the dict. Default: ``None``. amp_backend: The mixed precision backend to use ("native" or "apex"). Default: ``'native''``. .. deprecated:: v1.9 Setting ``amp_backend`` inside the ``Trainer`` is deprecated in v1.8.0 and will be removed in v2.0.0. This argument was only relevant for apex which is being removed. amp_level: The optimization level to use (O1, O2, etc...). By default it will be set to "O2" if ``amp_backend`` is set to "apex". .. deprecated:: v1.8 Setting ``amp_level`` inside the ``Trainer`` is deprecated in v1.8.0 and will be removed in v2.0.0. auto_lr_find: If set to True, will make trainer.tune() run a learning rate finder, trying to optimize initial learning for faster convergence. trainer.tune() method will set the suggested learning rate in self.lr or self.learning_rate in the LightningModule. To use a different key set a string instead of True with the key name. Default: ``False``. auto_scale_batch_size: If set to True, will `initially` run a batch size finder trying to find the largest batch size that fits into memory. The result will be stored in self.batch_size in the LightningModule or LightningDataModule depending on your setup. Additionally, can be set to either `power` that estimates the batch size through a power search or `binsearch` that estimates the batch size through a binary search. Default: ``False``. auto_select_gpus: If enabled and ``gpus`` or ``devices`` is an integer, pick available gpus automatically. This is especially useful when GPUs are configured to be in "exclusive mode", such that only one process at a time can access them. Default: ``False``. .. deprecated:: v1.9 ``auto_select_gpus`` has been deprecated in v1.9.0 and will be removed in v2.0.0. Please use the function :func:`~lightning_fabric.accelerators.cuda.find_usable_cuda_devices` instead. benchmark: The value (``True`` or ``False``) to set ``torch.backends.cudnn.benchmark`` to. The value for ``torch.backends.cudnn.benchmark`` set in the current session will be used (``False`` if not manually set). If :paramref:`~pytorch_lightning.trainer.Trainer.deterministic` is set to ``True``, this will default to ``False``. Override to manually set a different value. Default: ``None``. callbacks: Add a callback or list of callbacks. Default: ``None``. enable_checkpointing: If ``True``, enable checkpointing. It will configure a default ModelCheckpoint callback if there is no user-defined ModelCheckpoint in :paramref:`~pytorch_lightning.trainer.trainer.Trainer.callbacks`. Default: ``True``. check_val_every_n_epoch: Perform a validation loop every after every `N` training epochs. If ``None``, validation will be done solely based on the number of training batches, requiring ``val_check_interval`` to be an integer value. Default: ``1``. default_root_dir: Default path for logs and weights when no logger/ckpt_callback passed. Default: ``os.getcwd()``. Can be remote file paths such as `s3://mybucket/path` or 'hdfs://path/' detect_anomaly: Enable anomaly detection for the autograd engine. Default: ``False``. deterministic: If ``True``, sets whether PyTorch operations must use deterministic algorithms. Set to ``"warn"`` to use deterministic algorithms whenever possible, throwing warnings on operations that don't support deterministic mode (requires PyTorch 1.11+). If not set, defaults to ``False``. Default: ``None``. devices: Will be mapped to either `gpus`, `tpu_cores`, `num_processes` or `ipus`, based on the accelerator type. fast_dev_run: Runs n if set to ``n`` (int) else 1 if set to ``True`` batch(es) of train, val and test to find any bugs (ie: a sort of unit test). Default: ``False``. gpus: Number of GPUs to train on (int) or which GPUs to train on (list or str) applied per node Default: ``None``. .. deprecated:: v1.7 ``gpus`` has been deprecated in v1.7 and will be removed in v2.0. Please use ``accelerator='gpu'`` and ``devices=x`` instead. gradient_clip_val: The value at which to clip gradients. Passing ``gradient_clip_val=None`` disables gradient clipping. If using Automatic Mixed Precision (AMP), the gradients will be unscaled before. Default: ``None``. gradient_clip_algorithm: The gradient clipping algorithm to use. Pass ``gradient_clip_algorithm="value"`` to clip by value, and ``gradient_clip_algorithm="norm"`` to clip by norm. By default it will be set to ``"norm"``. limit_train_batches: How much of training dataset to check (float = fraction, int = num_batches). Default: ``1.0``. limit_val_batches: How much of validation dataset to check (float = fraction, int = num_batches). Default: ``1.0``. limit_test_batches: How much of test dataset to check (float = fraction, int = num_batches). Default: ``1.0``. limit_predict_batches: How much of prediction dataset to check (float = fraction, int = num_batches). Default: ``1.0``. logger: Logger (or iterable collection of loggers) for experiment tracking. A ``True`` value uses the default ``TensorBoardLogger`` if it is installed, otherwise ``CSVLogger``. ``False`` will disable logging. If multiple loggers are provided, local files (checkpoints, profiler traces, etc.) are saved in the ``log_dir`` of he first logger. Default: ``True``. log_every_n_steps: How often to log within steps. Default: ``50``. enable_progress_bar: Whether to enable to progress bar by default. Default: ``True``. profiler: To profile individual steps during training and assist in identifying bottlenecks. Default: ``None``. overfit_batches: Overfit a fraction of training/validation data (float) or a set number of batches (int). Default: ``0.0``. plugins: Plugins allow modification of core behavior like ddp and amp, and enable custom lightning plugins. Default: ``None``. precision: Double precision (64), full precision (32), half precision (16) or bfloat16 precision (bf16). Can be used on CPU, GPU, TPUs, HPUs or IPUs. Default: ``32``. max_epochs: Stop training once this number of epochs is reached. Disabled by default (None). If both max_epochs and max_steps are not specified, defaults to ``max_epochs = 1000``. To enable infinite training, set ``max_epochs = -1``. min_epochs: Force training for at least these many epochs. Disabled by default (None). max_steps: Stop training after this number of steps. Disabled by default (-1). If ``max_steps = -1`` and ``max_epochs = None``, will default to ``max_epochs = 1000``. To enable infinite training, set ``max_epochs`` to ``-1``. min_steps: Force training for at least these number of steps. Disabled by default (``None``). max_time: Stop training after this amount of time has passed. Disabled by default (``None``). The time duration can be specified in the format DD:HH:MM:SS (days, hours, minutes seconds), as a :class:`datetime.timedelta`, or a dictionary with keys that will be passed to :class:`datetime.timedelta`. num_nodes: Number of GPU nodes for distributed training. Default: ``1``. num_processes: Number of processes for distributed training with ``accelerator="cpu"``. Default: ``1``. .. deprecated:: v1.7 ``num_processes`` has been deprecated in v1.7 and will be removed in v2.0. Please use ``accelerator='cpu'`` and ``devices=x`` instead. num_sanity_val_steps: Sanity check runs n validation batches before starting the training routine. Set it to `-1` to run all batches in all validation dataloaders. Default: ``2``. reload_dataloaders_every_n_epochs: Set to a non-negative integer to reload dataloaders every n epochs. Default: ``0``. replace_sampler_ddp: Explicitly enables or disables sampler replacement. If not specified this will toggled automatically when DDP is used. By default it will add ``shuffle=True`` for train sampler and ``shuffle=False`` for val/test sampler. If you want to customize it, you can set ``replace_sampler_ddp=False`` and add your own distributed sampler. resume_from_checkpoint: Path/URL of the checkpoint from which training is resumed. If there is no checkpoint file at the path, an exception is raised. If resuming from mid-epoch checkpoint, training will start from the beginning of the next epoch. .. deprecated:: v1.5 ``resume_from_checkpoint`` is deprecated in v1.5 and will be removed in v2.0. Please pass the path to ``Trainer.fit(..., ckpt_path=...)`` instead. strategy: Supports different training strategies with aliases as well custom strategies. Default: ``None``. sync_batchnorm: Synchronize batch norm layers between process groups/whole world. Default: ``False``. tpu_cores: How many TPU cores to train on (1 or 8) / Single TPU to train on (1) Default: ``None``. .. deprecated:: v1.7 ``tpu_cores`` has been deprecated in v1.7 and will be removed in v2.0. Please use ``accelerator='tpu'`` and ``devices=x`` instead. ipus: How many IPUs to train on. Default: ``None``. .. deprecated:: v1.7 ``ipus`` has been deprecated in v1.7 and will be removed in v2.0. Please use ``accelerator='ipu'`` and ``devices=x`` instead. track_grad_norm: -1 no tracking. Otherwise tracks that p-norm. May be set to 'inf' infinity-norm. If using Automatic Mixed Precision (AMP), the gradients will be unscaled before logging them. Default: ``-1``. val_check_interval: How often to check the validation set. Pass a ``float`` in the range [0.0, 1.0] to check after a fraction of the training epoch. Pass an ``int`` to check after a fixed number of training batches. An ``int`` value can only be higher than the number of training batches when ``check_val_every_n_epoch=None``, which validates after every ``N`` training batches across epochs or during iteration-based training. Default: ``1.0``. enable_model_summary: Whether to enable model summarization by default. Default: ``True``. move_metrics_to_cpu: Whether to force internal logged metrics to be moved to cpu. This can save some gpu memory, but can make training slower. Use with attention. Default: ``False``. multiple_trainloader_mode: How to loop over the datasets when there are multiple train loaders. In 'max_size_cycle' mode, the trainer ends one epoch when the largest dataset is traversed, and smaller datasets reload when running out of their data. In 'min_size' mode, all the datasets reload when reaching the minimum length of datasets. Default: ``"max_size_cycle"``. inference_mode: Whether to use :func:`torch.inference_mode` or :func:`torch.no_grad` during evaluation (``validate``/``test``/``predict``). initz(: Initializing trainer with parameters: N)rurvryrzrrrwrtrrrrrxrZamp_typerr)rr)rr) epoch_loopz5`gradient_clip_val` should be an int or a float. Got .z`gradient_clip_algorithm` z! is invalid. Allowed algorithms: riinfrzJ`track_grad_norm` must be a positive number or 'inf' (infinity norm). Got )8super__init__rf_log_api_eventlogdetail __class____name__localsrIstateosfspathrA_data_connectorr>_accelerator_connectorrB_logger_connectorr?_callback_connectorr@_checkpoint_connectorrF_signal_connectorrMtunerr+r)connectfit_loopr* validate_loop test_loopr( predict_loop _ckpt_pathZon_trainer_init isinstanceintfloat TypeErrorrNZsupported_typelowerrXZsupported_typesrrrsr}_inference_mode_detect_anomaly_setup_on_initr9Z_init_profilerZ_init_debugging_flags)6selfrnrorprqrrrsrtrurvrwrxryrzr{r|r}r~rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrZtraining_epoch_looprj/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.pyrws$                   zTrainer.__init__)rcCsjt|d|_t|_td|_d|_g|_g|_ g|_ g|_ d|_ d|_ d|_td|_td|_dS)NFrz-inf)r9Z_log_device_infoZ should_stoprIrrnum_training_batchestrain_dataloadernum_sanity_val_batchesnum_test_batchesnum_val_batchesnum_predict_batchestest_dataloadersval_dataloaderspredict_dataloaders_last_train_dl_reload_epoch_last_val_dl_reload_epochrrrrr!s   zTrainer._setup_on_initzpl.LightningModule)modelrcCstts*t|tjs&tdt|jd|Sddlm}t||rJ| |St|tjrZ|Stdt|jddS)Nz*`model` must be a `LightningModule`, got ``r)OptimizedModulezM`model` must be a `LightningModule` or `torch._dynamo.OptimizedModule`, got `) rrplZLightningModulertype __qualname__Z torch._dynamorZ from_compiled)rrrrrr_maybe_unwrap_optimized5s     zTrainer._maybe_unwrap_optimized)rtrain_dataloadersr datamodule ckpt_pathrc Cs.||}||j_t||j|||||dS)a Runs the full optimization routine. Args: model: Model to fit. train_dataloaders: A collection of :class:`torch.utils.data.DataLoader` or a :class:`~pytorch_lightning.core.datamodule.LightningDataModule` specifying training samples. In the case of multiple dataloaders, please see this :ref:`section `. val_dataloaders: A :class:`torch.utils.data.DataLoader` or a sequence of them specifying validation samples. ckpt_path: Path/URL of the checkpoint from which training is resumed. Could also be one of two special keywords ``"last"`` and ``"hpc"``. If there is no checkpoint file at the path, an exception is raised. If resuming from mid-epoch checkpoint, training will start from the beginning of the next epoch. datamodule: An instance of :class:`~pytorch_lightning.core.datamodule.LightningDataModule`. N)rr_lightning_moduler8_call_and_handle_interrupt _fit_implrrrrrrrrrfitDs  z Trainer.fitcCstdt|jjdtj|j_ t j |j_ d|_ t|trJ|}d}|dusZ|durj|durjtd|jj||||d|p|j}|jj|jj |d|jdud|_|j||jd|jjsJd|_ dS) Nrz: trainer fit stageTzXYou cannot pass `train_dataloader` or `val_dataloaders` to `trainer.fit(datamodule=...)`)rrrmodel_providedZmodel_connectedrF)rfrrrrrrHFITTINGrfnrJRUNNINGstatustrainingrr%rXr attach_datarr_set_ckpt_pathlightning_moduler_runrstoppedrrrrrds4      zTrainer._fit_impl)r dataloadersrverboserrc CsF|dur|jdur.tdn||}||j_t||j|||||S)a Perform one evaluation epoch over the validation set. Args: model: The model to validate. dataloaders: A :class:`torch.utils.data.DataLoader` or a sequence of them, or a :class:`~pytorch_lightning.core.datamodule.LightningDataModule` specifying validation samples. ckpt_path: Either ``"best"``, ``"last"``, ``"hpc"`` or path to the checkpoint you wish to validate. If ``None`` and the model instance was passed, use the current weights. Otherwise, the best model checkpoint from the previous ``trainer.fit`` call will be loaded if a checkpoint callback is configured. verbose: If True, prints the validation results. datamodule: An instance of :class:`~pytorch_lightning.core.datamodule.LightningDataModule`. Returns: List of dictionaries with metrics logged during the validation phase, e.g., in model- or callback hooks like :meth:`~pytorch_lightning.core.module.LightningModule.validation_step`, :meth:`~pytorch_lightning.core.module.LightningModule.validation_epoch_end`, etc. The length of the list corresponds to the number of validation dataloaders used. Nz^`Trainer.validate()` requires a `LightningModule` when it hasn't been passed in a previous run)rrrrrr8r_validate_implrrrrrrrrrvalidates   zTrainer.validatecCstdt|jjdtj|j_ t j |j_ d|_ t|trJ|}d}|dur^|r^td|durr|j}d}nd}||j_|jj|||d|jj|jj |||jdud|_|j|_|j||jd}|jjsJd|_ |S) Nrz: trainer validate stageTzHYou cannot pass both `trainer.validate(dataloaders=..., datamodule=...)`F)rrrr)rfrrrrrrH VALIDATINGrrrJrr validatingrr%rXrrrrrrrrrZ_validated_ckpt_pathrrrrrrrrrresultsrrrrs0      zTrainer._validate_implc CsF|dur|jdur.tdn||}||j_t||j|||||S)a& Perform one evaluation epoch over the test set. It's separated from fit to make sure you never run on your test set until you want to. Args: model: The model to test. dataloaders: A :class:`torch.utils.data.DataLoader` or a sequence of them, or a :class:`~pytorch_lightning.core.datamodule.LightningDataModule` specifying test samples. ckpt_path: Either ``"best"``, ``"last"``, ``"hpc"`` or path to the checkpoint you wish to test. If ``None`` and the model instance was passed, use the current weights. Otherwise, the best model checkpoint from the previous ``trainer.fit`` call will be loaded if a checkpoint callback is configured. verbose: If True, prints the test results. datamodule: An instance of :class:`~pytorch_lightning.core.datamodule.LightningDataModule`. Returns: List of dictionaries with metrics logged during the test phase, e.g., in model- or callback hooks like :meth:`~pytorch_lightning.core.module.LightningModule.test_step`, :meth:`~pytorch_lightning.core.module.LightningModule.test_epoch_end`, etc. The length of the list corresponds to the number of test dataloaders used. NzZ`Trainer.test()` requires a `LightningModule` when it hasn't been passed in a previous run)rrrrrr8r _test_implrrrrtests!  z Trainer.testcCstdt|jjdtj|j_ t j |j_ d|_ t|trJ|}d}|dur^|r^td|durr|j}d}nd}||j_|jj|||d|jj|jj |||jdud|_|j|_|j||jd}|jjsJd|_ |S) Nrz: trainer test stageTzDYou cannot pass both `trainer.test(dataloaders=..., datamodule=...)`F)rrrr)rfrrrrrrHTESTINGrrrJrrtestingrr%rXrrrrrrrrrZ_tested_ckpt_pathrrrrrrrs0      zTrainer._test_impl)rrrreturn_predictionsrrc CsF|dur|jdur.tdn||}||j_t||j|||||S)as Run inference on your data. This will call the model forward function to compute predictions. Useful to perform distributed and batched predictions. Logging is disabled in the predict hooks. Args: model: The model to predict with. dataloaders: A :class:`torch.utils.data.DataLoader` or a sequence of them, or a :class:`~pytorch_lightning.core.datamodule.LightningDataModule` specifying prediction samples. datamodule: The datamodule with a predict_dataloader method that returns one or more dataloaders. return_predictions: Whether to return predictions. ``True`` by default except when an accelerator that spawns processes is used (not supported). ckpt_path: Either ``"best"``, ``"last"``, ``"hpc"`` or path to the checkpoint you wish to predict. If ``None`` and the model instance was passed, use the current weights. Otherwise, the best model checkpoint from the previous ``trainer.fit`` call will be loaded if a checkpoint callback is configured. Returns: Returns a list of dictionaries, one for each provided dataloader containing their respective predictions. See :ref:`Lightning inference section` for more. Nz]`Trainer.predict()` requires a `LightningModule` when it hasn't been passed in a previous run)rrrrrr8r _predict_impl)rrrrrrrrrpredictQs"  zTrainer.predictcCstdt|jjdtj|j_ t j |j_ d|_ ||j_t|trR|}d}|durf|rftd|durz|j}d}nd}|jj|||d|jj|jj |||jdud|_|j|_|j||jd}|jjsJd|_ |S) Nrz: trainer predict stageTzGYou cannot pass both `trainer.predict(dataloaders=..., datamodule=...)`F)rrrr)rfrrrrrrH PREDICTINGrrrJrr predictingrrrr%rXrrrrrrrZ_predicted_ckpt_pathrr)rrrrrrrrrrrrs0      zTrainer._predict_implr)rrrr) rrrrrscale_batch_size_kwargslr_find_kwargsmethodrc CsZ||}tdt,|jj||||||||d} Wdn1sL0Y| S)a Runs routines to tune hyperparameters before training. Args: model: Model to tune. train_dataloaders: A collection of :class:`torch.utils.data.DataLoader` or a :class:`~pytorch_lightning.core.datamodule.LightningDataModule` specifying training samples. In the case of multiple dataloaders, please see this :ref:`section `. val_dataloaders: A :class:`torch.utils.data.DataLoader` or a sequence of them specifying validation samples. dataloaders: A :class:`torch.utils.data.DataLoader` or a sequence of them specifying val/test/predict samples used for running tuner on validation/testing/prediction. datamodule: An instance of :class:`~pytorch_lightning.core.datamodule.LightningDataModule`. scale_batch_size_kwargs: Arguments for :func:`~pytorch_lightning.tuner.batch_size_scaling.scale_batch_size` lr_find_kwargs: Arguments for :func:`~pytorch_lightning.tuner.lr_finder.lr_find` method: Method to run tuner on. It can be any of ``("fit", "validate", "test", "predict")``. tune)rrrN)rrfrr^rZ_tune) rrrrrrrrrresultrrrrs"  $ z Trainer.tune)checkpoint_pathrcCsF|j||j|j|j|jjtjkrB|j dSN) rZ resume_startZ_restore_quantization_callbacksZ restore_modelZrestore_datamodulerrrHrZrestore_callbacks)rr rrr_restore_modules_and_callbackss     z&Trainer._restore_modules_and_callbacks)rrrcs|jdurdtttg}jdurdtfdd|Dsdddd|D}tdjjj d|dj j t j krtjjjj\}}|j_|j_t|drt|jj|jjttjj d j tjj d j!"#jj$sTtjj d |%|tjj d &d t'j(dt'j)dt'j*djj!dj+dj,dj-dj.dj.dj/0j/1j2j j t j kr3d4d5jj$r8tjj d |%|tjj dj67j68+}tjj d9j j t j kr3d4dtjj d:t;j<j _=dj _>|S)Nc3s|]}tj|VqdSr )rr.0srrr zTrainer._run..z, css|] }|jVqdSr )rr rrrrrzBUsing a compiled model is incompatible with the current strategy: z. Only zl support compilation. Either switch to one of the supported strategies or avoid passing in a compiled model.hparamsz: preparing dataz!: setting up strategy environmentz7: restoring module and callbacks from checkpoint path: z: configuring sharded modelz? Lightning internal flow looks like this: z or z || | || spawn processes || a6 || | || setup accelerator || and strategy || LIGHTNING | || zw || FLOW | || z; || DIRECTION or z- || or z || | || results \/ This is used to guide readers to the core loops: train, test, predict. zM is the simplest to understand, use `Go to Definition` to read it :) Z on_fit_startz: restoring training statez: trainer tearing downZ on_fit_endz: calling teardown hooks)?Z _compiler_ctxr6r4r3ranyjoin RuntimeErrorrrrrrHrr,rrrrrhasattrrOZclean_namespacerrrZ_attach_model_callbacksZ_attach_model_logging_functionsr:rrrZ prepare_dataZsetup_environment_Trainer__setup_profiler_call_setup_hookZrestore_checkpoint_after_setupr _call_configure_sharded_modelrfrrr _run_stage _run_train _run_evaluate _run_predictr reset_results reset_metricsr9_call_callback_hooks_call_lightning_module_hook_log_hyperparamsrZrestore_training_stateZ resume_end _teardown_call_teardown_hookrJFINISHEDrstage)rrrZsupported_strategiesZsupported_strategy_namesrrrrrrrs                           z Trainer._runc Cs<|js dSd}|jdur |jjnd}|jjr|r|jj}|jj}g}||@D]j}||||}}t|t|kr||qTt|t rt |t |kr||qT||krT||qT|rt d|di||}n |jjr|jj}n|r|jj}|jD].} |dur | || |j| qdS)NFz&Error while merging hparams: the keys zg are present in both the LightningModule's and LightningDataModule's hparams but have different values.)loggersrr!rhparams_initialkeysrappendrridrXZlog_hyperparamsZ log_graphsave) rr'Zdatamodule_log_hyperparamsZdatamodule_hparamsZlightning_hparamsZinconsistent_keyskeyZlm_valZdm_valrnrrrr!ms<          zTrainer._log_hyperparamscCs8|j|j}|dur ||j|jdS)zThis is the Trainer's internal teardown, unrelated to the `teardown` hooks in LightningModule and Callback; those are handled by :meth:`_call_teardown_hook`.N)rteardown _active_looprrrlooprrrr"s   zTrainer._teardowncCs@|jd|j||jr&|S|jr4|S|dS)Nz run-stage)rbarrierdispatch evaluatingrrrrrrrrrs  zTrainer._run_stagecCs|jd|jdS)NZsetup_training)rr1rZregister_signal_handlersrrrr_pre_training_routines zTrainer._pre_training_routinecCs|t|Wdn1s,0Y|jdusDJ|jtd||j_tj |j |j Wdn1s0YdS)NT) r4r^_run_sanity_checkrZtraintorchZset_grad_enabledrtrainerZautogradZset_detect_anomalyrrunrrrrrs&  zTrainer._run_trainc Cs|js J|j||j_|jd|jjdHt|j |j |j }Wdn1sd0YWdn1s0Y|D]:}t |t r|D]"\}}t |tr|||<qq|S)NZrun_Z _evaluation)r3_evaluation_loop_reload_evaluation_dataloadersr7rprofilerr%_evaluation_contextrrr8rdictitemsrcpuitem)rZeval_loop_resultsrkvrrrrs  F  zTrainer._run_evaluatecCsP||j||j_t|j|j|jWdS1sB0YdSr )reset_predict_dataloaderrrr7r<rrr8rrrrrs zTrainer._run_predictcsjjj}jo jdko |j }|rΈjj}d_j j  d| fddjD_t|Wdn1s0Y dj j t||j_dS)NrTZon_sanity_check_startcsg|]}tj|qSr)minr)r Z val_batchesrrr sz-Trainer._run_sanity_check..Zon_sanity_check_end)rrval_loopenable_validationrZ restartingrr%sanity_checkingrrrrr:rrr6no_gradr8r-)rrFZshould_sanity_checkr%rrrr5s.       &   zTrainer._run_sanity_checkcCsh|jjdusJ|jj}|jd|jdur<|jd|d|jd|d|jd|d|jddS)NZ pre_setupr9r%Z post_setup)rrrr1r_call_lightning_datamodule_hookrr )rrrrrr s  zTrainer._call_setup_hookcCsV|j8tdr*ddlm}||j|dWdn1sH0YdS)Nztorchdistx.deferred_initr)materialize_moduleZconfigure_sharded_model)rZmodel_sharded_contextrZtorchdistx.deferred_initrLrr )rrLrrrrs    z%Trainer._call_configure_sharded_modelcCs|jjdusJ|jj}|jdur0|jd|d|jd|d|jd|dd|j_d|j_|j D]}| dqb|j dS)Nr-rJsuccess) rrrrKrr r_current_fx_nameZ_metric_attributesr&finalizerZdescribe)rrrnrrrr#"s   zTrainer._call_teardown_hook) pl_module) hook_nameargsrPkwargsrcOs|p|j}|durtdt||}t|s0dS|j}||_|jd|jjd|||i|}Wdn1s|0Y||_|S)Nz3No `LightningModule` is available to call hooks on.z[LightningModule]r) rrgetattrcallablerNrr;rr)rrQrPrRrSr prev_fx_nameoutputrrrr 8s  ,z#Trainer._call_lightning_module_hook)rQrRrSrcOsv|jdurtdt|j|}t|rr|jd|jjjd|||i|WdS1sh0YdS)Nz7No `LightningDataModule` is available to call hooks on.z[LightningDataModule]r)rrrTrUrr;rr)rrQrRrSrrrrrKSs    z'Trainer._call_lightning_datamodule_hookc Ost|jjd||j}|r.|j}||_|jD]j}t||}t|r4|j d|j d|*|||jg|Ri|Wdq41s0Yq4|r||_dS)Nz: calling callback hook: [Callback]r) rdebugrrrrNrprTrUrr; state_key)rrQrRrSrPrVcallbackrrrrras  :zTrainer._call_callback_hookscCs*i}|jD]}|}|r |||j<q |S)z~Called when saving a model checkpoint, calls and returns every callback's `state_dict`, keyed by `Callback.state_key`.)rp state_dictrZ)rZcallback_state_dictsr[r\rrr_call_callbacks_state_dictxs   z"Trainer._call_callbacks_state_dict) checkpointrc Cs|j}|r|j}d|_|jD]f}|jd|jd |||j|}Wdn1s\0Y|durtd|jj dq|r||_dS)zXCalled when saving a model checkpoint, calls every callback's `on_save_checkpoint` hook.on_save_checkpointrXz.on_save_checkpointNzReturning a value from `z.on_save_checkpoint` was deprecated in v1.6 and is no longer supported as of v1.8. Please override `Callback.state_dict` to return state to be saved.) rrNrprr;rZr_ ValueErrorrr)rr^rPrVr[rrrr"_call_callbacks_on_save_checkpoints .z*Trainer._call_callbacks_on_save_checkpointc s|j}|r|j}d|_|d}|dur,dSt|dtdkfdd|jD}||}|rxtdt|d |jD]J}|j d |j d  | ||j|Wdq~1s0Yq~|r||_dS) zCalled when loading a model checkpoint. Calls every callback's `on_load_checkpoint` hook. We have a dedicated function for this rather than using `_call_callback_hooks` because we have special logic for getting callback_states. on_load_checkpointrpNzpytorch-lightning_versionz1.5.0devcsh|]}r|jn|jqSr)_legacy_state_keyrZr cbZis_legacy_ckptrr rz=Trainer._call_callbacks_on_load_checkpoint..zBe aware that when using `ckpt_path`, callbacks used to create the checkpoint need to be provided during `Trainer` instantiation. Please add the following callbacks: rrXz.on_load_checkpoint) rrNgetrrpr(r]listrr;rZrb)rr^rPrVcallback_statesZcurrent_callbacks_keys differencer[rrfr"_call_callbacks_on_load_checkpoints*   0z*Trainer._call_callbacks_on_load_checkpointcCsR|d}|durdS|jD]0}||j||j}|rt|}||qdS)zQCalled when loading a model checkpoint, calls every callback's `load_state_dict`.rpN)rhrprZrcrZload_state_dict)rr^rjr[rrrr_call_callbacks_load_state_dicts  z'Trainer._call_callbacks_load_state_dictcOs|j}|j}||_t|j|}t|s*dS|jd|jjjd|||i|}Wdn1sl0Y||_|S)Nz [Strategy]r) rrNrTrrUrr;rr)rrQrRrSrPrVrrWrrr_call_strategy_hooks  ,zTrainer._call_strategy_hook)eventrcCstjd|dS)Nzlightning.trainer.)r6Z_CZ_log_api_usage_once)rorrrrszTrainer._log_api_eventcCsN|jjdusJ|jdkr |jnd}t|j|j_|jj|jj||j ddS)Nrg)r% local_ranklog_dir) rr world_sizerprrrrr9rq)rrprrrZ__setup_profilerszTrainer.__setup_profilerc Cs|jj}|p|j}td|}|jdk}|r6|r6|s:dS|jtj|_ |j dkrj|jj |j tjd|_ t |j t tf|jjtjd|_ t|j tr|j jn|j }t |t |jjdt |t t|jdtrt |t tt|j tst||jj|_ |p|jp|j}t|j |j|r$t|j ntd}|_|dkr@dS|j|_t|jt rft!||j|_n6|jtdkrt ||j|_n|jdkrt"d t|j#t r|j#|_$|j$|jkr<|j%dur.rprrrrrLsz Trainer.early_stopping_callbackscCsdd|jDS)zA list of all instances of :class:`~pytorch_lightning.callbacks.prediction_writer.BasePredictionWriter` found in the Trainer.callbacks list.cSsg|]}t|tr|qSr)rr$rdrrrrEVrz7Trainer.prediction_writer_callbacks..rrrrrprediction_writer_callbacksRsz#Trainer.prediction_writer_callbackscCs|j}t|dkr|dSdS)zThe first :class:`~pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint` callback in the Trainer.callbacks list, or ``None`` if it doesn't exist.rN)checkpoint_callbacksryrrrrcheckpoint_callbackXszTrainer.checkpoint_callbackcCsdd|jDS)zA list of all instances of :class:`~pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint` found in the Trainer.callbacks list.cSsg|]}t|tr|qSr)rr!rrrrrEcrz0Trainer.checkpoint_callbacks..rrrrrr_szTrainer.checkpoint_callbackscCs"|jD]}t|tr|SqdS)zAn instance of :class:`~pytorch_lightning.callbacks.progress.base.ProgressBarBase` found in the Trainer.callbacks list, or ``None`` if one doesn't exist.N)rprr#)rrrrrprogress_bar_callbackes   zTrainer.progress_bar_callbackcCs |jj}|durtddd|S)Nz`trainer.resume_from_checkpoint` is deprecated in v1.5 and will be removed in v2.0. Specify the fit checkpoint path with `trainer.fit(ckpt_path=)` instead.r)rZresume_from_checkpoint_fit_pathr[)rrrrrrnszTrainer.resume_from_checkpointcCs|jS)aGSet to the path/URL of a checkpoint loaded via :meth:`~pytorch_lightning.trainer.trainer.Trainer.fit`, :meth:`~pytorch_lightning.trainer.trainer.Trainer.validate`, :meth:`~pytorch_lightning.trainer.trainer.Trainer.test`, or :meth:`~pytorch_lightning.trainer.trainer.Trainer.predict`. ``None`` otherwise.)rrrrrrzszTrainer.ckpt_path)filepath weights_onlystorage_optionsrcCs(|jdurtd|jj|||ddS)a* Runs routine to create a checkpoint. Args: filepath: Path where checkpoint is saved. weights_only: If ``True``, will only save the model weights. storage_options: parameter for how to save to storage, passed to ``CheckpointIO`` plugin NzSaving a checkpoint is only possible if a model is attached to the Trainer. Did you call `Trainer.save_checkpoint()` before calling `Trainer.{fit,validate,test,predict}`?)rr)rAttributeErrorrsave_checkpoint)rrrrrrrrs zTrainer.save_checkpointcCst|}dd|jDS)NcSsi|]\}}||jqSr)default)r rArBrrr rz.Trainer.default_attributes..)inspect signature parametersr>)clsZinit_signaturerrrdefault_attributess zTrainer.default_attributes)rrRrSrcKst||fi|Sr )rR)rrRrSrrrrRszTrainer.from_argparse_args) arg_parserrcCs t||Sr )rS)rrrrrrSszTrainer.parse_argparsercCst|Sr )rT)rrrrmatch_env_argumentsszTrainer.match_env_arguments) parent_parserrSrcKst||fi|Sr )rQ)rrrSrrrrQszTrainer.add_argparse_argscCs|jjtjkSr )rrrJZ INTERRUPTEDrrrr interruptedszTrainer.interruptedcCs|jjtjkSr )rr%rGrvrrrrrszTrainer.training)valrcCs"|rtj|j_n|jrd|j_dSr )rGrvrr%rrrrrrrs cCs|jjtjkSr )rr%rGrrrrrrszTrainer.testingcCs"|rtj|j_n|jrd|j_dSr )rGrrr%rrrrrrs cCs|jjtjkSr )rr%rGrrrrrrszTrainer.predictingcCs"|rtj|j_n|jrd|j_dSr )rGrrr%rrrrrrs cCstd|jjtjkS)NzM`Trainer.tuning` has been deprecated in v1.8.0 and will be removed in v2.0.0.)r[rr%rGTUNINGrrrrtuningszTrainer.tuningcCs*td|rtj|j_n|jr&d|j_dS)NzUSetting `Trainer.tuning` has been deprecated in v1.8.0 and will be removed in v2.0.0.)r[rGrrr%rrrrrrs  cCs|jjtjkSr )rr%rGrrrrrrszTrainer.validatingcCs"|rtj|j_n|jrd|j_dSr )rGrrr%rrrrrrs cCs|jjduo|jjjSr )rr%r3rrrrr3szTrainer.evaluatingcCs|jjtjkSr )rr%rGSANITY_CHECKINGrrrrrHszTrainer.sanity_checkingcCs"|rtj|j_n|jrd|j_dSr )rGrrr%rHrrrrrHs cCs |jjjS)zThe number of optimizer steps taken (does not reset each epoch). This includes multiple optimizers and TBPTT steps (if enabled). )rr global_steprrrrrszTrainer.global_stepcCs |jjjjS)z=The current epoch, updated after the epoch end hooks are run.)rZepoch_progresscurrentZ completedrrrrrz szTrainer.current_epochcCs|jjSr )rrrrrrrszTrainer.max_epochscCs|jjSr )rrrrrrrszTrainer.min_epochscCs|jjSr )rrrrrrrszTrainer.max_stepscCs|jjSr )rrrrrrrszTrainer.min_stepscCs |jjjjS)z,Whether trainer is executing the last batch.)rrZbatch_progress is_last_batchrrrrr szTrainer.is_last_batchcCs|jSr ) _fit_looprrrrr%szTrainer.fit_loop)r0rcCs||_||_dS)zAttach a custom fit loop to this Trainer. It will run with :meth:`~pytorch_lightning.trainer.trainer.Trainer.fit`. N)r7rr/rrrr)scCs|jSr )_validate_looprrrrr3szTrainer.validate_loopcCs||_||_dS)a-Attach a custom validation loop to this Trainer. It will run with :meth:`~pytorch_lightning.trainer.trainer.Trainer.validate`. Note that this loop is different from the one running during training inside the :meth:`pytorch_lightning.trainer.trainer.Trainer.fit` call. N)r7rr/rrrr7scCs|jSr ) _test_looprrrrrBszTrainer.test_loopcCs||_||_dS)zAttach a custom test loop to this Trainer. It will run with :meth:`~pytorch_lightning.trainer.trainer.Trainer.test`. N)r7rr/rrrrFscCs|jSr ) _predict_looprrrrrPszTrainer.predict_loopcCs||_||_dS)zAttach a custom prediction loop to this Trainer. It will run with :meth:`~pytorch_lightning.trainer.trainer.Trainer.predict`. N)r7rr/rrrrTscCsL|jjtjkr|jjjS|jjtjkr,|jS|jjtj kr@|j St ddS)NzPThe `Trainer._evaluation_loop` property isn't defined. Accessed outside of scope) rrrHrrrrFrrrrrrrrrr9^s zTrainer._evaluation_loopcCs.|jr |jS|js|jr|jS|jr*|jSdSr )rrrHr3r9rrrrrrr.hs  zTrainer._active_loopcCst|jdkr|jdSdSr)ryr&rrrrrnuszTrainer.logger)rnrcCs|s g|_n|g|_dSr )r&)rrnrrrrnyscCs|jSr Z_loggersrrrrr&szTrainer.loggers)r&rcCs|r|ng|_dSr r)rr&rrrr&scCs|jjSr )rcallback_metricsrrrrrszTrainer.callback_metricscCs|jjSr )rlogged_metricsrrrrrszTrainer.logged_metricscCs|jjSr )rprogress_bar_metricsrrrrrszTrainer.progress_bar_metricscCs|j}|dur|jSdSr )r._results)rZ active_looprrrrszTrainer._resultscCstr|sdStddSr)rY_should_terminate_gracefullyrWrrrr_exit_gracefully_on_signalsz"Trainer._exit_gracefully_on_signalcCs0tjt|j|jjd}t|jj|dddkS)N)rsum)Z reduce_opr)r6ZtensorrZ_terminate_gracefullyrrboolreduce)rvaluerrrrsz$Trainer._should_terminate_gracefullycCs|j}|jdgkrtd|jdkr<|jdkr6tdS|jS|jdurVtd||j }|tdkrn|jS|jdus|J| |j |_ |j }t ||t|jd}|jdkrt||jn|}|S)a Estimated stepping batches for the complete training inferred from DataLoaders, gradient accumulation factor and distributed setup. Examples:: def configure_optimizers(self): optimizer = ... scheduler = torch.optim.lr_scheduler.OneCycleLR( optimizer, max_lr=1e-3, total_steps=self.trainer.estimated_stepping_batches ) return [optimizer], [scheduler] rzkEstimated stepping batches cannot be computed with different `accumulate_grad_batches` at different epochs.rirNzBLoading `train_dataloader` to estimate number of stepping batches.rg)accumulation_schedulerZepochsrXrrrrr\rrZget_accumulate_grad_batchesrzrmathceilr{rD)rrZ total_batchesZeffective_batch_sizeZmax_estimated_stepsrrrestimated_stepping_batchess&    z"Trainer.estimated_stepping_batches)3TTNNNNrgNNNNNNTrhrirgFNNNriNNNNNNNrjNNFrkTrlNNNNrFTFFNNNFrmT)NNNN)NNNN)NNNTN)NNNTN)NNNTN)NNNTN)NNNNN)NNNNN)NNNNNNr)N)N)N)N)N)N)FN)r __module__rrPrr&r rrr r rrrstrr rrr7r<rr2r;r0rrobjectrrcr%rarrr_rr`rrrrrr rrLrr rr!r"rr4rrrr5rrr#r rKrr=r]rarlrmrn staticmethodrrrrrrCpropertyrrr1rrxrprrrrtrrrrrsetterrbrrrr=rrr6nnModulerrqrrrrGrqr"rrr$rr!rrr#rrrr classmethodrrrrRrSrrrQrrrrrrr3rHrrzrrrrrr+rr*rrr(rr9r.rnr&rrCrrDrrErrrr __classcell__rrrrrfvs     , # . / 5 0 5 1 4 3  '  -      "  s             rfT)rrrccsf|r0tr tr tdks0t|ts0tjntj}|dVWdn1sX0YdS)NZgloo) distZ is_availableZis_initialized get_backendrrr6rrI)rrZcontext_manager_classrrrr<s r<)T)__doc__rloggingrrwarningsargparserrr contextlibrcopyrdatetimerpathlibrtypingr r r r r rrrweakrefrr6Ztorch.distributedZ distributedrZ#lightning_utilities.core.apply_funcrZ lightning_utilities.core.importsrZpackaging.versionrrZ torch.optimrZtorch.utils.datarZtyping_extensionsrZpytorch_lightningrZ#lightning_fabric.utilities.cloud_iorZlightning_fabric.utilities.datarZ"lightning_fabric.utilities.importsrZ lightning_fabric.utilities.typesrZ#lightning_fabric.utilities.warningsrZpytorch_lightning.acceleratorsrrZpytorch_lightning.callbacksr r!r"r#Z-pytorch_lightning.callbacks.prediction_writerr$Z!pytorch_lightning.core.datamoduler%Zpytorch_lightning.loggersr&Z%pytorch_lightning.loggers.tensorboardr'Zpytorch_lightning.loopsr(r)Z2pytorch_lightning.loops.dataloader.evaluation_loopr*Z pytorch_lightning.loops.fit_loopr+Z!pytorch_lightning.loops.utilitiesr,r-Zpytorch_lightning.pluginsr.r/r0r1Zpytorch_lightning.profilersr2Zpytorch_lightning.strategiesr3r4r5r6r7Zpytorch_lightning.trainerr8r9Z1pytorch_lightning.trainer.configuration_validatorr:Z:pytorch_lightning.trainer.connectors.accelerator_connectorr;r<r=r>Z7pytorch_lightning.trainer.connectors.callback_connectorr?Z9pytorch_lightning.trainer.connectors.checkpoint_connectorr@Z3pytorch_lightning.trainer.connectors.data_connectorrAZ5pytorch_lightning.trainer.connectors.logger_connectorrBZs    (                                  r