o
    "ju                     @   s   d dl Z d dlZd dlZd dlZd dlmZmZ d dlZddlm	Z	 eG dd deZ
eG dd deZG d	d
 d
ZG dd dZG dd dZG dd dZG dd dZe	ejZdddZdS )    N)IntEnumunique   )
get_loggerc                   @   s$   e Zd ZdZdZdZdZdZdZdS )
DeviceTyper         r         N)	__name__
__module____qualname__UNKNOWNCPUGPUXPUDCUNIC r   r   p/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/auto_parallel/static/cluster.pyr      s    r   c                   @   s0   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
S )LinkTyper   r   r   r      r	   r
         N)r   r   r   r   ZLOCZSYSPHBZPIXZPIBNVLZNVBNETr   r   r   r   r   $   s    r   c                   @   s  e Zd ZejejejgZdd Ze	dd Z
e
jdd Z
e	dd Zejdd Ze	d	d
 Zejdd
 Ze	dd Zejdd Ze	dd Zejdd Ze	dd Zejdd Ze	dd Zejdd Ze	dd Zejdd Ze	dd Zejdd Zdd Zd d! Zd"S )#Devicec                 C   s:   || _ || _|| _d | _d | _d | _d | _d | _d | _d S N)	
_global_id	_local_id_machine_type_model
_dp_gflops
_sp_gflops
_hp_gflops_memory)self	global_idlocal_idmachiner   r   r   __init__4   s   
zDevice.__init__c                 C      | j S r   r   r(   r   r   r   r)   E      zDevice.global_idc                 C   
   || _ d S r   r.   r(   valuer   r   r   r)   I      
c                 C   r-   r   r    r/   r   r   r   r*   M   r0   zDevice.local_idc                 C   r1   r   r5   r2   r   r   r   r*   Q   r4   c                 C   r-   r   r!   r/   r   r   r   r+   U   r0   zDevice.machinec                 C   r1   r   r6   r2   r   r   r   r+   Y   r4   c                 C   r-   r   r"   r/   r   r   r   type]   r0   zDevice.typec                 C   r1   r   r7   r2   r   r   r   r8   a   r4   c                 C   r-   r   r#   r/   r   r   r   modele   r0   zDevice.modelc                 C   r1   r   r9   r2   r   r   r   r:   i   r4   c                 C   r-   r   r$   r/   r   r   r   	dp_gflopsm   r0   zDevice.dp_gflopsc                 C   r1   r   r;   r2   r   r   r   r<   q   r4   c                 C   r-   r   r%   r/   r   r   r   	sp_gflopsu   r0   zDevice.sp_gflopsc                 C   r1   r   r=   r2   r   r   r   r>   y   r4   c                 C   r-   r   r&   r/   r   r   r   	hp_gflops}   r0   zDevice.hp_gflopsc                 C   r1   r   r?   r2   r   r   r   r@      r4   c                 C   r-   r   r'   r/   r   r   r   memory   r0   zDevice.memoryc                 C   r1   r   rA   r2   r   r   r   rB      r4   c                 C   s<   d}|d | j| j| jj| jj| j| j| j	| j
| j	7 }|S )N zvglobal_id: {}, local_id: {}, machine_id: {}, type: {}, model: {}, dp_flops: {}, sp_flops: {}, hp_flops: {}, memory: {})formatr)   r*   r+   idr8   namer:   r<   r>   r@   rB   r(   strr   r   r   __str__   s   zDevice.__str__c                 C      |   S r   rI   r/   r   r   r   __repr__      zDevice.__repr__N)r   r   r   r   r   r   r   NON_ACCELERATOR_TYPEr,   propertyr)   setterr*   r+   r8   r:   r<   r>   r@   rB   rI   rL   r   r   r   r   r   1   sR    

















r   c                   @   s   e Zd ZdZdZdd Zedd Zejdd Zedd	 Z	e	jd
d	 Z	edd Z
e
jdd Z
edd Zejdd Zedd Zejdd Zedd Zejdd Zdd Zdd ZdS )Linkr      c                 C   s(   || _ || _d | _d | _d | _d | _d S r   )_src_tgtr"   
_bandwidth_latency_hop)r(   sourcetargetr   r   r   r,      s   
zLink.__init__c                 C   r-   r   )rS   r/   r   r   r   rX      r0   zLink.sourcec                 C   r1   r   )_sourcer2   r   r   r   rX      r4   c                 C   r-   r   )rT   r/   r   r   r   rY      r0   zLink.targetc                 C   r1   r   )_targetr2   r   r   r   rY      r4   c                 C   r-   r   r7   r/   r   r   r   r8      r0   z	Link.typec                 C   r1   r   r7   r2   r   r   r   r8      r4   c                 C   r-   r   rU   r/   r   r   r   	bandwidth   r0   zLink.bandwidthc                 C   r1   r   r\   r2   r   r   r   r]      r4   c                 C   r-   r   rV   r/   r   r   r   latency   r0   zLink.latencyc                 C   r1   r   r^   r2   r   r   r   r_      r4   c                 C   r-   r   rW   r/   r   r   r   hop   r0   zLink.hopc                 C   r1   r   r`   r2   r   r   r   ra      r4   c                 C   s,   d}|d | jj| jj| j| j| j7 }|S )NrC   zPsource_global_id: {}, target_global_id: {}, type: {}, bandwidth: {}, latency: {})rD   rX   r)   rY   r8   r]   r_   rG   r   r   r   rI      s   zLink.__str__c                 C   rJ   r   rK   r/   r   r   r   rL      rM   zLink.__repr__N)r   r   r   default_hopdefault_nic_bandwidthr,   rO   rX   rP   rY   r8   r]   r_   ra   rI   rL   r   r   r   r   rQ      s<    












rQ   c                   @   s   e Zd Zdd Zedd Zejdd Zedd Zejdd Zed	d
 Zejdd
 Zedd Z	e	jdd Z	edd Z
edd Zedd Zdd Zdd Zdd Zdd Zdd ZdS ) Machinec                 C   s4   || _ d | _d | _d | _i | _i | _i | _d| _d S Nr   )_id	_hostname_addr_port_devices_links_accelerators!_non_accelerator_cumulative_count)r(   rE   r   r   r   r,      s   
zMachine.__init__c                 C   r-   r   rf   r/   r   r   r   rE      r0   z
Machine.idc                 C   r1   r   rn   r2   r   r   r   rE      r4   c                 C   r-   r   rg   r/   r   r   r   hostname   r0   zMachine.hostnamec                 C   r1   r   ro   r2   r   r   r   rp     r4   c                 C   r-   r   rh   r/   r   r   r   addr  r0   zMachine.addrc                 C   r1   r   rq   r2   r   r   r   rr     r4   c                 C   r-   r   ri   r/   r   r   r   port  r0   zMachine.portc                 C   r1   r   rs   r2   r   r   r   rt     r4   c                 C   r-   r   )rj   r/   r   r   r   devices  r0   zMachine.devicesc                 C   r-   r   )rk   r/   r   r   r   links  r0   zMachine.linksc                 C   r-   r   )rl   r/   r   r   r   accelerators   r0   zMachine.acceleratorsc                 C   s,   || j |j< |jtjvr|| j|j< d S d S r   )rj   r)   r8   r   rN   rl   r(   devicer   r   r   
add_device$  s   zMachine.add_devicec                 C   s   || j |jj|jjf< d S r   )rk   rX   r)   rY   r(   linkr   r   r   add_link*  s   zMachine.add_linkc                 C   s   | j ||fd S r   )rk   get)r(   source_global_idtarget_global_idr   r   r   get_link.  s   zMachine.get_linkc                 C   sD   d}| j  D ]	}|d| 7 }q| j D ]	}|d| 7 }q|S )NrC   z
, device: z, link: )ru   valuesrv   )r(   rH   ry   r|   r   r   r   rI   1  s   zMachine.__str__c                 C   rJ   r   rK   r/   r   r   r   rL   9  rM   zMachine.__repr__N)r   r   r   r,   rO   rE   rP   rp   rr   rt   ru   rv   rw   rz   r}   r   rI   rL   r   r   r   r   rd      s:    











rd   c                   @   sh   e Zd Zdd Zedd Zedd Zedd Zed	d
 Zedd Z	edd Z
edd ZdS )AlphaLatencyc                 C   s  t |tsJ |dd | _|dd | _|dd | _|dd | _| jd ur8zt| j| _W n   td| jd urD| jdd nd | _	| jd urS| jdd nd | _
| jd urb| jdd nd | _| j	d urzzt| j	| _	W n   td| j
d urzt| j
| _
W n   td| jdd | _| jdd | _| jdd | _| jdd | _| jd urt | jtr| jd	v sJ t| j | _nzt| j| _W n   td
| jd urt | jtr| jd	v sJ t| j | _nzt| j| _W n   td| jd ur.t | jtr| jdv sJ t| j | _nzt| j| _W n   td| jd ur\t | jtrK| jdv sCJ t| j | _d S z	t| j| _W d S    tdd S )NbaseinterZintraswitchz The switch latency must be floatringtreez$The base ring latency must be float.)r   z%The inter ring latency must be float.z%The inter tree latency must be float.)r   r   z%The intra ring latency must be float.z%The intra tree latency must be float.)
isinstancedictr~   Z_baseZ_interZ_intra_switchfloat	TypeError
_base_ring
_base_treeZ_base_inter_inter_ring_inter_tree_intra_ring_intra_treerH   r   )r(   alpha_latencyr   r   r   r,   >  s~   



zAlphaLatency.__init__c                 C   r-   r   )r   r/   r   r   r   	base_ring  r0   zAlphaLatency.base_ringc                 C   r-   r   )r   r/   r   r   r   	base_tree  r0   zAlphaLatency.base_treec                 C   r-   r   )r   r/   r   r   r   r     r0   zAlphaLatency.switchc                 C   r-   r   )r   r/   r   r   r   
inter_ring  r0   zAlphaLatency.inter_ringc                 C   r-   r   )r   r/   r   r   r   
inter_tree  r0   zAlphaLatency.inter_treec                 C   r-   r   )r   r/   r   r   r   
intra_ring  r0   zAlphaLatency.intra_ringc                 C   r-   r   )r   r/   r   r   r   
intra_tree  r0   zAlphaLatency.intra_treeN)r   r   r   r,   rO   r   r   r   r   r   r   r   r   r   r   r   r   =  s     L





r   c                   @   s   e Zd ZdZdd Z									
					d=ddZedd Zedd Zedd Z	dd Z
edd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Zd6d7 Zd8d9 Zd:d; Zd<S )>Clusterz
    The cluster is an abstract of the hardware resource for training, which contains the cluster topology and
    related hardware information. It will serve the task mapping, cost model and auto searching.
    c                 C   s4   d| _ i | _d | _d | _i | _i | _d | _d | _d S re   )_num_machines	_machinesZ	_topology_alpha_latency_rank_to_device_id_device_id_to_rank_num_devices_per_machine
_gpu_modelr/   r   r   r   r,     s   
zCluster.__init__V1006271Cr         rR      x  T=  z  K      c           .         s  g ddgdg    }|| _ || _ fdd}dd }dd	 }i }g |d
< d}i }i }t|D ]}i }dt| |d< d|d< d|d< g |d< g }d}t|D ]R}i }|dkrd|dkrd|n|d }|d7 }||}|||}|}||d< ||d< ||d< ||d< ||d< |
|d< |	|d< ||d< d|d< |||< |||< || qVi }||\} }!}|}"|}#|d7 }d}|}d}| |d< |!|d< ||d< |"|d< |#|d< ||d< ||d< ||d< ||d< |||< |||< || i }$|d7 }d }d!}%d}&d}||$d< ||$d< ||$d< |||< |||< ||$ ||d"< |d
 | q6td|d D ]Y}td|d D ]N}||kr=q4|| }'|| }(|| })|| }*i }+|},|}-|,|+d#< |-|+d$< |'|(krn|)|*krnd%|+d< ||+d&< nd'|+d< ||+d&< |d
 |' d |+ q4q+| | d(S ))z#Generate cluster by default config.)r   A100ZH100A2ZA10ZA16A30ZA40r   r   c                    sB   d }| v r	d}n| v rd}n	|  v rd}nd}|d usJ |S )Nr   r   r   r   )	gpu_modelr8   Z
dcu_modelsZ
gpu_modelsZ
xpu_modelsr   r   _convert_to_type  s   z<Cluster.gen_default_config_cluster.<locals>._convert_to_typec                 S   sr   d }| dkrdt | d }n"| dkrdt | d }n| dkr)dt | d }n| t | d }|d us7J |S )Nr   zTesla V100-SXM2-GBr   zTesla A100-SXM-r   zTesla A30-SXM-)rH   )r   
gpu_memoryr:   r   r   r   _convert_to_model  s   z=Cluster.gen_default_config_cluster.<locals>._convert_to_modelc                 S   sb   d\}}}| dkrd}d}d}n
| dkrd}d}d}|d us J |d us&J |d us,J |||fS )N)NNNr   x86_64ZGenuineIntelz'Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GZ6148z&Intel(R) Xeon(R) Gold 6148 CPU @ 2.40Gr   )	cpu_modelarchvendorr:   r   r   r   _convert_to_cpu_info  s   

z@Cluster.gen_default_config_cluster.<locals>._convert_to_cpu_infomachinesr   Zhost_rp   z	127.0.0.1rr   ii  rt   rv   r   r)   r*   r8   r:   rB   r>   r<   r@   r   r   r   r   r   g      )@ru   r   r   r   r]   r   N)r   r   rangerH   append_build_from_dict).r(   r   r   
node_countdevice_countr   Z
cpu_memoryZinter_bandwidthZintra_bandwidthgpu_dp_gflopsgpu_sp_gflopsgpu_hp_gflopsZcpu_dp_gflopsZcpu_sp_gflopsZall_gpu_modelsr   r   r   cluster_infor)   Zglobal_id_to_device_typeZglobal_id_to_nodeir+   ru   r*   jry   r8   r:   rB   Z
cpu_devicer   r   r>   r<   Z
nic_devicewidthipZ	node_id_iZ	node_id_jZdevice_type_iZdevice_type_jr|   r   r   r   r   r   gen_default_config_cluster  s   




z"Cluster.gen_default_config_clusterc                 C   r-   r   )r   r/   r   r   r   rank_to_device_idk  r0   zCluster.rank_to_device_idc                 C   r-   r   )r   r/   r   r   r   device_id_to_ranko  r0   zCluster.device_id_to_rankc                 C   r-   r   )r   r/   r   r   r   r   s  r0   zCluster.machinesc                 C   s   t |tsJ || j|j< |jdkrJ| j|jd  }|j}|jD ]}|j| jtjvr9|| }|| j	|< || j
|< q t|jt|j |j |_d S |jD ]}|j| jtjvrl|}|| j	|< || j
|< |j| |j|< qMt|jt|j |_d S )Nr   r   )r   rd   r   rE   rm   ru   r8   r   rN   r   r   lenrw   )r(   r+   Zprev_machineoffsetr)   Zrank_idr   r   r   add_machinew  sB   









zCluster.add_machinec                 C   r-   r   )r   r/   r   r   r   r     r0   zCluster.alpha_latencyc                 C   s   t |tsJ |j| d S r   )r   r   r+   rz   rx   r   r   r   rz     s   zCluster.add_devicec                 C   s    t |tsJ |jj| d S r   )r   rQ   rX   r+   r}   r{   r   r   r   r}     s   zCluster.add_linkc                 C   s0   d }| j  D ]}||j v r|j| }q|S r   )r   r   ru   keys)r(   device_global_idry   r+   r   r   r   
get_device  s   
zCluster.get_devicec                 C   s"  |d }|D ]}|   }t|}|d|_|d|_|d|_|dg }|D ]W}|d}|d}	t||	|}
|dd }|d urKt| }ntj}||
_	|d	d |
_
t|d
d|
_t|dd|
_t|dd|
_t|dd|
_| |
 q*| | q|D ]r}|dg }|D ]g}|d}|d}| |}| |}t||}|dd }|d urt| }ntj}||_	t|dd|_t|dd|_|dd |_|jd u r|j}|j}|j|jkrd|_ntj|_| | qqd|v rt|d| _d S d | _d S )Nr   rp   rr   rt   ru   r)   r*   r8   r:   r<   r   r>   r@   rB   rv   r   r   r]   r_   ra   r   )_generate_machine_idrd   r~   rp   rr   rt   r   r   r   r8   r:   r   r<   r>   r@   rB   rz   r   r   rQ   r   r]   r_   ra   r+   rE   rb   r}   r   r   Z_alpha_latecy)r(   r   Zmachines_infoZmachine_info
machine_idr+   Zdevices_infoZdevice_infor   Zdevice_local_idry   device_typeZ
links_infoZ	link_infor   r   rX   rY   r|   Z	link_typeZsource_machineZtarget_machiner   r   r   r     sj   












zCluster._build_from_dictc                 C   s@   t |}t|}W d    n1 sw   Y  | | d S r   )openjsonloadr   )r(   Zjson_file_pathZ	json_filer   r   r   r   build_from_file  s   
zCluster.build_from_filec                 C   s   | j }|  j d7  _ |S )Nr   )r   )r(   Zcur_machine_idr   r   r   r     s   zCluster._generate_machine_idc                 C   s@   g }| j  D ]}|j D ]}|jt| kr|| qq|S r   )r   r   ru   r8   r   r   )r(   r   ru   r+   ry   r   r   r   get_all_devices  s   
zCluster.get_all_devicesc           	      C   sf   d }d}|  |}|j}|||}d }|d u rtj}n|j}|dkr'd}|S d||d d   }|S )Ni  g        r   r   r   i@B )r   r+   r   rQ   rc   r]   )	r(   source_device_idtarget_device_idbetaZconvert_basery   r+   r|   r]   r   r   r   get_beta  s   
zCluster.get_betac                 C   s@   d }d }|  |}|j}|||}|d ur|j}|S tj}|S r   )r   r+   r   ra   rQ   rb   )r(   r   r   r   ra   ry   r+   r|   r   r   r   get_hop  s   
zCluster.get_hopc                 C   s@   t  }|D ]}| |}|jj}|| qt|dkrdS dS )Nr   FTsetr   r+   rE   addr   )r(   
device_idsmachine_ids	device_idry   r   r   r   r   cross_machine  s   
zCluster.cross_machinec                 C   s"   g }|D ]
}| | j|  q|S r   )r   r   )r(   Zgroup_ranksr   rankr   r   r   convert_rank_to_device_id(  s   z!Cluster.convert_rank_to_device_idc                 C   sD   t  }|D ]}| |}|jj}|| qt|}|dks J |S re   r   )r(   r   r   r   ry   r   countr   r   r   get_involved_machine_count0  s   
z"Cluster.get_involved_machine_countc                 C   s
   t | jS r   )r   r   r/   r   r   r   get_num_machines:  s   
zCluster.get_num_machinesc                 C   s   | j sJ | j S r   )r   r/   r   r   r   get_num_devices_per_machine=  s   
z#Cluster.get_num_devices_per_machinec                 C   s(   d}| j  D ]
}|d| d7 }q|S )NrC   z	machine: 
)r   r   )r(   rH   r+   r   r   r   rI   C  s   zCluster.__str__c                 C   rJ   r   rK   r/   r   r   r   rL   I  rM   zCluster.__repr__N)r   r   r   r   r   r   rR   r   r   r   r   r   r   )r   r   r   __doc__r,   r   rO   r   r   r   r   r   rz   r}   r   r   r   r   r   r   r   r   r   r   r   r   rI   rL   r   r   r   r   r     sV    
 /


#
<
r   c                 C   s  dd }t  }| r7|| r7d| d v r|| d d  |S | d d }| d d }| d d }| d d }n~td	}|d u rCd
}nt|}td}|d u rSd
}nt|}|| dks_J t|| }tdd dkrtdd }|}ttjjj	|d }n3tj
j }	|	sJ d|	j}ztd|}
|
d
 }t|
d d d }W n   t|	jd }|}Y td||||tj tdd  ddddddddd}|dkr|d n|d  }|j|||||d! |d" |d# d$ |S )%Nc                 S   sh   | sdS d| vr
dS d| d vr2d| d vrdS d| d vr dS d| d vr(dS d| d vr0dS dS dS )	NFclusterpath	num_nodesnum_gpusr   r   Tr   )json_configr   r   r   is_by_json_configQ  s   z.get_default_cluster.<locals>.is_by_json_configr   r   r   r   r   r   ZPADDLE_LOCAL_SIZEr   ZPADDLE_GLOBAL_SIZEr   ZPADDLE_DISTRI_BACKENDZxcclZPADDLE_XCCL_BACKENDi ʚ;z#Auto parallel just runs on gpu now.z[ , -]zeNode Count: {}, Local Device Size: {}, GPU Model: {}, GPU Memory: {}GB, World size: {}, EndPoint: {}.ZPADDLE_CURRENT_ENDPOINTr   r   iH )dpsphpi%  i,L  i	 )r   r   r   r   r   r   r   )r   r   r   r   r   r   r   )r   r   osgetenvintpaddler   coreZ	libpaddleZ_get_device_total_memoryry   cudaZget_device_propertiesrF   resplitZtotal_memoryloggerinforD   distributedZget_world_sizer   )r   r   r   r   Zlocal_device_countr   rB   Zglobal_device_countZgpu_nameZgpu_infoZ	re_resultZgflops_infoZdefault_gflopsr   r   r   get_default_clusterP  s~   




	r  r   )r   loggingr   r   enumr   r   r   Zutils.log_utilsr   r   r   r   rQ   rd   r   r   INFOr  r  r   r   r   r   <module>   s*   	oMPj   
)