o
    "j+                     @   sT  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ ddlm	Z	m
Z
mZ ddlmZmZmZmZmZmZmZmZmZmZmZmZ g Zdadd Zi ad Zi ai ad	a g d
Z!da"da#e j$ddZ%d a&dd Z'dd Z(dd Z)dd Z*dd Z+dd Z,dd Z-dd Z.dd Z/dd  Z0	 d+d!d"Z1da2d#d$ Z3dde%fd%d&Z4d'd( Z5d)d* Z6dS ),    N)core)in_dynamic_mode   )Group_add_new_groupis_initialized)	_c_concat_c_identity_c_lookup_table_c_softmax_with_cross_entropy_c_split_Linear_linear_mp_allreduce_parallel_embedding_parallel_linear_set_var_distributedsplitc                   C   s   t stj a t S N)_global_envpaddledistributedZParallelEnv r   r   ^/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/collective.py_get_global_env.   s   
r   Z_default_pg)ncclglooheterxcclbkcli  )secondsc                 C      | a d S r   )_default_backend)backendr   r   r   _set_default_backendL      r$   c                 C   r!   r   )_default_store)storer   r   r   _set_default_storeQ   r%   r(   c                  C   s.   t tvrt } t| jdtt| jtt < tS )Nr   )_global_env_gid
_group_mapr   r   ranklistrange
world_sizegenvr   r   r   _get_group_mapV   s   r1   c                   C   s
   t  t S r   )r1   r)   r   r   r   r   _get_global_group`   s   
r2   c                   C   s   t S r   _group_map_by_namer   r   r   r   _get_group_map_by_named   s   r5   c                   C   s   t  sJ dt t S )NzZCall paddle.distributed.init_parallel_env first to initialize the distributed environment.)r   r5   _default_group_namer   r   r   r   _get_default_groupi   s   
r7   c                 C      | t vsJ |t | < d S r   )r*   )gidgroupr   r   r   _set_group_mapr      r;   c                 C   r8   r   r3   )namer:   r   r   r   _set_group_map_by_namex   r<   r>   c                 C   r8   r   )_group_map_backend)r:   r#   r   r   r   _set_group_map_backend~   r<   r@   c                   C   s8   t  rtd7 attt jd S tt tt jd S )Nr   	   )r   _start_ring_idmaxr   nringslenr1   r   r   r   r   _new_ring_id   s   rF   c           	      C   s   d }t  }| tv sJ d|  | dkrtj||||}|S | dkr/tj|||||j}|S | dkr@tj||j|||}|S | dkrMtj	||||}|S )NzUnsupported backend: %s.r   r   r   r   )
r   _valid_backend_listr   ZProcessGroupGloocreateZProcessGroupNCCLZ
pg_timeoutZProcessGroupCustomdevice_typeZProcessGroupBKCL)	r#   r'   r+   r.   
group_name
pg_optionsgroup_idpgr0   r   r   r   _new_process_group_impl   s&   		rN   c                 C   r!   r   )_custom_gid)r9   r   r   r   _set_custom_gid   r%   rP   c              	      s  t  rtrtnt }tt| }|dkrA| du st| dkrAt }|j}|j}|du r-t	n|}| du r5|} t| t|ksAJ dt| }t
| } |dkrh|| v rh|dkrWdn| |}	t|t|	||d|d}
nd}	d}
t|	|| |
|d}|t|< |t|< |t|< t| ttd	ddkrtjjtjdgtjd
|dd |S |sd}|dksJ dt   j}t }|| vrtd|| }|t|< ngt
| } | |}t| }t||| }|t|< |dkr*t }||_||_ fdd| D |_  j!|_!d|_"t# rt$ j%}t&||'| nt( r&t) j%}t*||'| nt+d|S t  r8tj,dgdd
ntj-dgddd
}tjj|dd tj.| |S )a  

    Creates a new distributed communication group.

    Args:
        ranks (list): The global ranks of group members.
        backend (str): The backend used to create group, only nccl is supported now.
        timeout (datetime.timedelta, optional): The waiting timeout for store relevant options, default is 30 minutes.

    Returns:
        Group: The group instance.

    Examples:
        .. code-block:: python

            >>> # doctest: +REQUIRES(env: DISTRIBUTED)
            >>> import paddle

            >>> paddle.distributed.init_parallel_env()
            >>> tindata = paddle.randn(shape=[2, 3])
            >>> gp = paddle.distributed.new_group([2, 4, 6])
            >>> paddle.distributed.all_reduce(tindata, group=gp, sync_op=False)

    r   Nr   zQSize of new group must be less than or equal to that of the default global group.r   )rK   rL   )rM   r=   Z$FLAGS_eager_communication_connection)ZdtypeT)r:   sync_opr   z,backend other than nccl is not supported yet   c                    s   g | ]} j | qS r   )trainer_endpoints).0ir/   r   r   
<listcomp>  s    
znew_group.<locals>.<listcomp>zno cuda device foundZint32)rR   )/r   rO   rF   r6   strrE   r7   r+   ranksr"   sortedindexrN   r&   r   r4   r*   r?   r   intosgetenvr   r   Z
all_reduceZzerosZuint8r   r   ZParallelStrategyZnranksZ
local_rankrT   Zcurrent_endpointrD   Zis_compiled_with_cudaZ	CUDAPlace	device_idZNCCLParallelContextZinit_with_ring_idZis_compiled_with_xpuZXPUPlaceZBKCLParallelContextAssertionErrorZ	to_tensorfullwait)rY   r#   timeoutr9   rJ   Zglobal_groupZglobal_rankZglobal_rankssizer+   rM   r:   Zring_idgpZ
group_rankZ
group_sizeZstrategyZplacetmpr   r/   r   	new_group   s   






rg   c                   C   s   t  S )a!  
    Check whether the distributed package is available.

    Returns:
        Returns True if the distributed package is available, otherwise False.

    Examples:
        .. code-block:: python

            >>> import paddle
            >>> print(paddle.distributed.is_available())

    )r   Zis_compiled_with_distr   r   r   r   is_available5  s   rh   c           
      C   sN  t  }t }|j}|j}|j}| dkrt j|d|| d S | dkrSd}|jD ]}||7 }q(|d	d7 }t
|jdd }t j| t j|d||| d S | dkrq|j}	tj|	 d	|  t j|d|||	 d S | d
krd}|jD ]}||7 }qz|d	d7 }t
|jdd }t j| t j|d||| d S d S )Nr   0r    z
ring_id:{}zUTF-8)encodingr   :r   )r   Zcreate_or_get_global_tcp_storer   r+   r.   r_   ZCommContextManagerZcreate_gloo_comm_contextrT   formathashlibmd5encode	hexdigestZset_device_idZcreate_nccl_comm_contextrI   r   ZdeviceZ
set_deviceZcreate_xccl_comm_contextZcreate_bkcl_comm_context)
r#   r'   Z
global_envr+   r.   Zdev_idZendpoints_strZendpointZendpoints_str_hashZdev_typer   r   r   _init_parallel_envF  sX   








rr   )r   )7datetimern   r]   r   Zpaddle.baser   Zpaddle.frameworkr   Zcommunication.groupr   r   r   Zfleet.layers.mpu.mp_opsr   r	   r
   r   r   r   r   r   r   r   r   r   __all__r   r   r*   r)   r4   r?   r6   rG   r&   r"   	timedeltaZ_default_timeoutrB   r$   r(   r1   r2   r5   r7   r;   r>   r@   rF   rN   rO   rP   rg   rh   rr   r   r   r   r   <module>   sN   8	
	
 