a
    dq                     @  s   d Z ddlmZ ddlZddlZddlmZ ddlmZ erPddl	m
Z
mZmZ edZddd	d
dddZdddd	ddddZdS )z*Utilities that can be used with Deepspeed.    )annotationsN)_PATH)_DEEPSPEED_AVAILABLE)(get_fp32_state_dict_from_zero_checkpointget_model_state_fileget_optim_filescpur   z
str | Nonestr)checkpoint_dirtagreturnc                 C  s   |d u rft j| d}t j|rXt|}|  }W d    qf1 sL0    Y  ntd| t j| |}t j|st	dt
 d|S )NZlatestz Unable to find 'latest' file at zDirectory 'z' doesn't exist)ospathjoinisfileopenreadstrip
ValueErrorisdirFileNotFoundErrords_checkpoint_dir)r
   r   Zlatest_pathfd	directory r   n/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/pytorch_lightning/utilities/deepspeed.pyr   #   s    
,r   None)r
   output_filer   r   c                   s   t | |g d t| } t| }tj|d td}|d d }t| |}tj|td} fdd| D }fdd D |d	< t	d
|  t
|| dS )a3  Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be loaded with
    ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. It gets copied into the
    top level checkpoint dir, so the user can easily do the conversion at any point in the future. Once extracted,
    the weights don't require DeepSpeed and can be used in any application. Additionally the script has been
    modified to ensure we keep the lightning state inside the state dict for being able to run
    ``LightningModule.load_from_checkpoint('...')```.

    Args:
        checkpoint_dir: path to the desired checkpoint folder.
            (one that contains the tag-folder, like ``global_step14``)
        output_file: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
        tag: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt
            to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``

    Examples:

        >>> from pytorch_lightning.utilities.deepspeed import (
        ...     convert_zero_checkpoint_to_fp32_state_dict
        ... )
        >>> # Lightning deepspeed has saved a directory instead of a file
        >>> save_path = "lightning_logs/version_0/checkpoints/epoch=0-step=0.ckpt/" # doctest: +SKIP
        >>> output_path = "lightning_model.pt" # doctest: +SKIP
        >>> convert_zero_checkpoint_to_fp32_state_dict(save_path, output_path) # doctest: +SKIP
        Saving fp32 state dict to lightning_model.pt
    )moduleZ	optimizerZlr_schedulerZcsr_tensor_module_namesZskipped_stepsZglobal_stepsZdp_world_sizeZmp_world_sizer   )Zmap_locationZoptimizer_state_dict
zero_stagec                   s   i | ]\}}| vr||qS r   r   ).0keyvalue)deepspeed_statesr   r   
<dictcomp>d       z>convert_zero_checkpoint_to_fp32_state_dict.<locals>.<dictcomp>c                   s    i | ]}| d d  | qS )zmodule.   )	partition)r    k)
state_dictr   r   r$   g   r%   r)   zSaving fp32 state dict to N)r   r   r   torchload
CPU_DEVICEr   itemskeysprintsave)r
   r   r   Zoptim_filesZoptim_stater   Z
model_fileZclient_stater   )r#   r)   r   *convert_zero_checkpoint_to_fp32_state_dict4   s    


r1   )N)N)__doc__
__future__r   r   r*   Z lightning_fabric.utilities.typesr   Z&pytorch_lightning.strategies.deepspeedr   Zdeepspeed.utils.zero_to_fp32r   r   r   Zdevicer,   r   r1   r   r   r   r   <module>   s   
 