o
    *j                  
   @   s  d dl Z d dlZd dlmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ e	 Zdddd	ddd
dd	d
ddZdZdad2ddZdefddZdejdeeee jf deeee jf ddfddZdeeee jf ddfddZdeddfddZdejdeeee jf dedeeejf fd d!Zdejdeeee jf dedeeejf fd"d#Zd$eeejf deeee jf ddfd%d&Zd'ejd(ejdefd)d*Zdeeee jf d+edeeejf fd,d-Z d.ejded/edeej fd0d1Z!dS )3    N)DictListUnion)nn)
get_logger)	is_masterZmoe   )version
world_sizeZv1i  )r	   r
   Ztensor_model_parallel_sizeseed)r	   r   )zgpt-moeZplugzmglm-text-summarizationzmp_rank_XX_model_states.ptFc                 K   s   ddl m} ddlm} | du r|du rJ d| du rM||}z|j} W n' tyL   z|jj}W n ty?   |jj}Y nw |t	v rHt	| ni } Y nw | 
| ||  dadS )a  Initialize megatron_util environment for megatron_based model.

    If argument `megatron_cfg` is not specified, then the megatorn_cfg will be load
    from configuration.json file in the model_dir.

    Args:
        megatron_cfg (Dict, optional): Megatron Config will be send to megatron_util.
        model_dir (str, optional): The model path for configuration. Defaults to None.
    r   )read_config)initialize_megatronNzEcfg and model_dir cannot both be None when initializing megatron_utilT)Zmodelscope.utils.hubr   megatron_utilr   ZmegatronAttributeErrormodeltypeZpipeline_DEFAULT_CFG_WITH_MODEL_TYPEupdate_IS_MEGATRON_INITIALIZED)Zmegatron_cfgZ	model_dirkwargsr   r   cfgZ
model_type r   `/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/utils/megatron_utils.pyinit_megatron_util$   s.   


r   returnc                   C   s   t S N)r   r   r   r   r   is_megatron_initializedE   s   r   r   checkpoint_dir
target_dirc                 C   s  dt fdd}tjtj|drtj|d}tt|}ttd}t	| t
| |d| d|  ||k rWtj|dd	 t| ||| }t|| |d
 dS ||krutj|dd	 t| ||| }t|| |d dS t|| |d dS )a-  Split or Merge checkpoint for megatron_based model.

    Args:
        model (nn.Module): Any megatron_based model.
        checkpoint_dir (Union[str, bytes, os.PathLike]): The save path of origin checkpoint.
        target_dir (Union[str, bytes, os.PathLike]): The target path of new checkpoint.
    informationc                 S   s   t  r
t|  d S d S r   )r   loggerinfo)r   r   r   r   
log_masterT   s   z/convert_megatron_checkpoint.<locals>.log_masterr   Z
WORLD_SIZEzorigin_num_partitions: z, target_num_partitions: T)exist_okzSplit checkpoints succeeded.zMerge checkpoints succeeded.zCopy checkpoints succeeded.N)strospathexistsjoinlenlistdirintgetenv_check_origin_dir_check_target_num_partitionsmakedirs_split_checkpoint_save_converted_checkpoint_merge_checkpointshutilcopytree)r   r   r   r"   Zorigin_num_partitionsZtarget_num_partitions
state_dictr   r   r   convert_megatron_checkpointI   s8   

r6   
origin_dirc                 C   sh   t | }t|t|d @ dksJ dtt|D ]}td|d}||v s1J d| dqd S )N   r   z)The number of files must be a power of 2!XX02dzCan not find z file!)r%   r*   r)   range_CHECKPOINT_FORMATreplace)r7   	filenamesicheckpoint_namer   r   r   r-   w   s   



r-   num_partitionsc                 C   s   | | d @ dksJ dd S )Nr8   r   z5The number of target partitions must be a power of 2!r   )rA   r   r   r   r.      s   r.   c                 C   s   t td}|| }t||}i }|  D ](\}}t||| }	|	dkr,|| ||< qt|| ||	}
|
||   ||< q|S )NRANK)r+   r%   r,   _load_by_ranknamed_parameters_get_diff_dim_split_tensorclone)r   r   rA   target_rankZorigin_rankr5   target_state_dictname	parameterdimpartitions_listr   r   r   r0      s    
r0   c                    s   t tdfddtD } fdd|D }i }|  D ],\}t||d  }|dkr<|d  |< q"tjfdd|D |d |< q"|S )	NrB   c                    s   g | ]}  | qS r   r   .0r?   )rA   rI   r   r   
<listcomp>   s    z%_merge_checkpoint.<locals>.<listcomp>c                    s   g | ]}t  |qS r   )rD   rO   )r   r   r   rQ      s    
r   rC   c                    s   g | ]}|  qS r   r   )rP   r5   )rK   r   r   rQ      s    rM   )	r+   r%   r,   r;   rE   rF   torchcatrH   )r   r   rA   Zorigin_rank_listZstate_dict_listrJ   rL   rM   r   )r   rK   rA   rI   r   r2      s(   
r2   r5   c                 C   s8   t td}td|d}t| tj|| d S )NrB   r9   r:   )	r+   r%   r,   r<   r=   rS   saver&   r(   )r5   r   rI   target_namer   r   r   r1      s   r1   tensor1tensor2c                 C   s4   t t| j|jD ]\}\}}||kr|  S q	dS )NrC   )	enumeratezipshape)rW   rX   r?   s1s2r   r   r   rF      s
   rF   rankc                 C   sB   t d|d}tjtj| |dd dd}d|v r|d S |S )Nr9   r:   c                 S   s   | S r   r   )Zstoragelocr   r   r   <lambda>   s    z_load_by_rank.<locals>.<lambda>T)Zmap_locationZweights_onlymodule)r<   r=   rS   loadr%   r&   r(   )r   r^   r@   r5   r   r   r   rD      s   rD   tensorpartition_dimc                 C   s4   ddl m} |j| ||}tj| ||d}|S )Nr   )mpurR   )r   re   utilsdividesizerS   split)rc   rA   rd   re   Zper_partition_sizerN   r   r   r   rG      s   
rG   )NN)"r%   r3   typingr   r   r   rS   r   Zmodelscope.utils.loggerr   Zmodelscope.utils.torch_utilsr   r    r   r<   r   r   boolr   Moduler$   bytesPathLiker6   r-   r+   r.   ZTensorr0   r2   r1   rF   rD   rG   r   r   r   r   <module>   s   
!
.





