o
    *j	                     @   sl   d Z ddlZddlmZ dddZdddZG dd	 d	ejjZG d
d dejjZ	dddZ
dddZdS )z-
Copyright 2020 The Microsoft DeepSpeed Team
    N)mpuc                    s^       t } fddtt D } ||< tjj| t d tj	||d  }|S )z5Gather tensors and concatenate them along a dimensionc                    s   g | ]}t  qS  )torchZ
empty_like).0_input_r   k/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/nlp/gpt_moe/moe/mappings.py
<listcomp>   s    z"_gather_tokens.<locals>.<listcomp>)group)dim)

contiguousr   get_tensor_model_parallel_rankrange$get_tensor_model_parallel_world_sizer   distributedZ
all_gatherZget_tensor_model_parallel_groupcat)r   r   rankZtensor_listoutputr   r   r	   _gather_tokens	   s   


r   c                 C   sf   t  }t  }| j| | dks"J d| d| j|  d| d| j| | }t| ||| |S )z/Divide a tensor among the tensor parallel ranksr   zinput dimension z (z2) is not divisible by tensor parallel world size ())r   r   r   shaper   Znarrow)r   r   Ztotal_chunksZ
this_chunk
chunk_sizer   r   r	   _drop_tokens   s   r   c                   @   4   e Zd ZdZedd Zedd Zedd ZdS )	_GatherTokensz1All gather tokens among the tensor parallel ranksc                 C   
   t ||S N)r   graphr   r   r   r   r	   symbolic-      
z_GatherTokens.symbolicc                 C      || _ t||S r   )r   r   ctxr   r   r   r   r	   forward1      
z_GatherTokens.forwardc                 C      t || jd fS r   )r   r   )r$   Zgrad_outputr   r   r	   backward6      z_GatherTokens.backwardN__name__
__module____qualname____doc__staticmethodr    r%   r(   r   r   r   r	   r   *       

r   c                   @   r   )	_DropTokensz5Divide tokens equally among the tensor parallel ranksc                 C   r   r   )r   r   r   r   r	   r    >   r!   z_DropTokens.symbolicc                 C   r"   r   )r   r   r#   r   r   r	   r%   B   r&   z_DropTokens.forwardc                 C   r'   r   )r   r   )r$   r   r   r   r	   r(   G   r)   z_DropTokens.backwardNr*   r   r   r   r	   r1   ;   r0   r1   c                 C   $   t d u s
t  dkr| S t| |S N   )r   r   r   applyr   r   r   r   r	   gather_tokensL      r7   c                 C   r2   r3   )r   r   r1   r5   r6   r   r   r	   drop_tokensS   r8   r9   )r   )r.   r   Zmegatron_utilr   r   r   ZautogradFunctionr   r1   r7   r9   r   r   r   r	   <module>   s    


