o
    *jN                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z
 ddlmZ ddlmZ 		dd	d
Zdd ZdddZ				dddZdd ZdS )    N)mpu)Float16Module)unwrap_model)DistributedDataParallel   )logger)MoEc              	   C   s   |du rt  }tj| |d|d}|d dkr5tj|d}tj| |d| d|dd}||fS tj|d	 }}||fS )
z8Determine the directory name for this rank's checkpoint.NZmp_rank_02dr   zmodel_rng.ptZ
expp_rank_	_mp_rank_z_optim_states.ptzmodel_optim_rng.ptr   Zget_tensor_model_parallel_rankospathjoin)checkpoints_pathpath_load_tagnum_expertsZtensor_rank	expp_rankZcommon_pathZ
model_nameZ
optim_name r   l/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/nlp/gpt_moe/checkpointing.pyget_checkpoint_names   s    

r   c              
   C   s:   t  }tjtj| dd| d| d|dd}|S )NmodelZlayer_Z_expert_r
   r	   z_model_states.ptr   )r   Zlayer_idZ	expert_idZmp_rankZ	ckpt_namer   r   r   _get_expert_ckpt_name4   s   r   c           	      C   sL   t  }t |}t| |||d}|\}}td|  tj|dd}|S )z Load the base state_dict from the given directory

    If rank0 is true, just loads rank 0 checkpoint, ignoring arguments.
    )r   r   r   zLoading model checkpoint from cpuZmap_location)r   Zget_max_expert_size_nameget_expert_parallel_rankr   r   infotorchload)	load_dirr   r   Zlargest_group_namer   Zcheckpoint_namesZmodel_checkpoint_nameZoptim_checkpoint_namemodel_state_dictr   r   r   _load_base_checkpoint=   s   
r    Tr   c                 C   s   t | ttf} t|||d}|d usJ |rt| |d | nt| |d | |r3| j|d |d n	| j|d |d tj rHtj	  d S d S )N)r   r   moduler   )strict)
r   torchDDPr   r    load_moe_checkpointZload_state_dictr   distributedZis_initializedZbarrier)r   r   r   r"   r   Zload_ds_ckptsr   r   r   r   load_checkpointQ   s   
r&   c                 C   s   d}|   D ]b\}}t|trh|j}|j}t|}t|D ]E}	|| |	 }
t|||
}t	
d|  tj|tdd}d}t| D ]}|| |
 | |	 }||||< qF|| q|d7 }qd S )Nr   zLoading expert states from r   r   z).deepspeed_moe.experts.deepspeed_experts.r   )Znamed_modules
isinstancer   Zexpert_group_namenum_local_expertsr   r   ranger   r   r   r   r   Zdevicelistkeysreplacepopupdate)r   Z
state_dictr   Zmoe_layer_idZn_moduler!   
group_namer(   r   Zlocal_expert_idZglobal_expert_idZmoe_load_pathZexpert_state_dictZmoe_str_prefixkeyZ	local_keyr   r   r   r$   l   s4   




r$   )NN)NTr   T)r   r   Zmegatron_utilr   Zmegatron_util.modelr   Zmegatron_util.utilsr   Ztorch.nn.parallelr   r#   configurationr   Z	moe.layerr   r   r   r    r&   r$   r   r   r   r   <module>   s&   

	
