o
    "Õj_  ã                   @   sB   d dl Z d dlZd dlmZ ddlmZmZmZ G dd„ dƒZdS )é    N)ÚThreadé   )Úget_gpu_infoÚget_gpu_processÚget_gpu_utilc                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚWatcherc                 C   s²   || _ d| _g | _| j jjsd S | j jjp| j jjj| _	t
| j	ƒdkrEtj | j jj| j jj› d¡}tjtj |¡dd t|dƒ| _nd S t| jd| _d| j_| j ¡  d S )Né   r   z.gpu.logT)Úexist_okÚw)Útarget)ÚctxÚintervalZgpu_utilÚargsZenable_gpu_logZdevicesÚnodeZdeviceÚlabelsÚgpusÚlenÚosÚpathÚjoinZlog_dirZjob_idÚmakedirsÚdirnameÚopenÚgpu_fdr   ÚwatchÚprocÚdaemonÚstart)Úselfr   Úfn© r    ún/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/launch/controllers/watcher.pyÚ__init__   s    
ÿzWatcher.__init__c                 C   s€   t | jƒdks	d S |  ¡  d}| j |¡ | j d¡ | jj ¡ s2|  |¡ t	 
| j¡ | jj ¡ r!t| dƒr>| j ¡  d S d S )Nr   zDindex,utilization_gpu,memory_total,memory_used,memory_free,timestampÚ
r   )r   r   Ú_print_gpu_infor   Úwriter   ÚstatusZis_doneÚ_save_gpu_logÚtimeÚsleepr   ÚhasattrÚclose)r   Úutil_keyr    r    r!   r   1   s   
þ
ÿzWatcher.watchc                 C   sØ   z^d}| j  |¡ | j  d¡ t| jƒD ]}| j  | |¡¡ | j  d¡ q| j  d¡ d}| j  |¡ | j  d¡ t| jƒD ]}| j  | |¡¡ | j  d¡ q?| j  d¡ | j  ¡  W d S    | jj 	d¡ Y d S )NzEindex,uuid,driver_version,name,gpu_serial,display_active,display_moder#   z.pid,process_name,gpu_uuid,gpu_name,used_memoryzsave gpu info failed)
r   r%   r   r   Ústrr   Úflushr   ÚloggerÚwarning)r   Zinfo_keyÚlineZprocess_keyr    r    r!   r$   B   s$   zWatcher._print_gpu_infoc                 C   sZ   zt | jƒD ]}| j | |¡¡ | j d¡ q| j ¡  W d S    | jj d¡ Y d S )Nr#   zsave gpu log failed)	r   r   r   r%   r-   r.   r   r/   r0   )r   r,   r1   r    r    r!   r'   X   s   zWatcher._save_gpu_logc                 C   s   t | dƒr	 d S d S )Nr   )r*   )r   r    r    r!   Ústopa   s   
ýzWatcher.stopN)Ú__name__Ú
__module__Ú__qualname__r"   r   r$   r'   r2   r    r    r    r!   r      s    	r   )	r   r(   Ú	threadingr   Zutils.nvsmir   r   r   r   r    r    r    r!   Ú<module>   s
   