o
    "j"                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZ d dlmZ d dlmZmZ d dlmZ e  dd Zd	d
 Zdd Zdd Z	dddZdd Zdd Zdd Zdd Zedkrej ddi e Z!ee! dS dS )    N)"DistributedDataLoaderFromGenerator)get_all_process_groupsnew_process_group)_get_global_env)Program_current_expected_place)Operatorc                 C   s*   |   dv rdS |   dv rdS td)N)yestruety1T)nofalsefn0FzUnsupported value encountered.)lowerargparseArgumentTypeError)v r   w/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/auto_parallel/static/tuner/profiler.pystr2bool&   s
   
r   c                  C   sn   t  } | jddtdd | jddtdd | jdtd	d
d | jdtd	dd | jdtd	dd |  }|S )Nz--profile_start_step
   z:integer indicates the warmup step before starting profile.)defaulttypehelpz--profile_end_step   z-integer indicates at the end step of profile.z--rankTz the rank id of the this process.)r   requiredr   z--device_idz"the device id of the this process.z--ctx_filenamezDthe filename to the profile context file saved by optimization tuner)r   ArgumentParseradd_argumentintstr
parse_args)parserargsr   r   r   r$   /   sB   r$   c                 C   sL   |   D ]\}}|dkrqt||d qt }|D ]
}t| |  qd S )Nr   )ranksgroup_id)itemsr   r   printZinstantiate)	group_maprankr(   r'   Zall_process_groupsZprocess_groupr   r   r   init_process_groupsU   s   
r-   c                 C   sD   t |  }g d}d}|D ]}|D ]}||v r|    S qq|S )N)ZInvalidArgumentErrorZNotFoundErrorZOutOfRangeErrorZAlreadyExistsErrorZResourceExhaustedErrorZPreconditionNotMetErrorZPermissionDeniedErrorZExecutionTimeoutErrorZUnimplementedErrorZUnavailableError
FatalErrorZExternalErrorr.   )r#   
splitlines)errormsgZcpp_error_types
error_typeetliner   r   r   get_cpp_error_typeb   s   r5      c                 C   sd  |d }|   }g }|jD ]}||jv r||j|  qt|j}	|jd jdkr<|	d8 }	tdD ]	}
|jddd q2t	j
 }t	j
| | t||d||j|||j|jd	}W d    n1 scw   Y  t|j}t|d	 |	d	 d
D ]$}
|j|d	  }|j }||j t||| d}|jd| qwt||	 D ]	}
|j|dd q|  |S )Ndatasetr   Zcreate_py_reader   F)syncF   )	r7   	feed_listcapacityplaces
batch_sizeepochssteps_per_epochZdata_parallel_world_sizeZdata_parallel_rankr6   )r   )global_blockZinput_namesvarsappendlenopsr   rangeZ
_remove_oppaddlestaticZcuda_placesZprogram_guardr   r>   Zdp_world_sizeZdp_rankdescZ_prepend_opZ	copy_fromr   insertZ_sync_with_cpp)main_programstartup_programprofile_ctxr?   r@   r7   Z
main_blockr;   nameZop_size_r=   Z
dataloaderZnew_op_sizeopZnew_op_descZnew_opr   r   r   create_dataloaderz   sJ   





rR   c                 C   sB   | d }t  }|}td|j|j|j | d }t|tj d S )NZdistributed_envz0current process rank: {}, device_id: {}, ip: {}.r+   )r   r*   formatr,   	device_idZcurrent_endpointr-   r&   )rN   Zdist_envgenvr+   r   r   r   	init_comm   s   	rV   c                 C   sV   | d }t |}| d }t |}| d }| |sJ | |}|||fS )NZmain_program_decsZstartup_program_decsloss_var_name)r   Zparse_from_stringrB   Zhas_varvar)rN   Zmain_program_desc_strrL   Zstartup_program_decs_strrM   rW   loss_varr   r   r   load_programs   s   


rZ   c                  C   s<   t  } t| tjstdt }t|j}tj|}|S )Nz2OptimizationTuner only support CUDA GPU right now.)	r   
isinstancerH   Z	CUDAPlaceRuntimeErrorr   rT   rI   ZExecutor)Z
place_typerU   Zplaceexer   r   r   get_executor   s   r^   c                 C   sr  t j| jstd| j dt| jd}tj|dd}W d   n1 s(w   Y  t| t	|\}}}t
|||}|d }t }z}|| d}	d}
|j  |
| jk rt }|j||gd	d
}t }|
| jkru|	|| 7 }	td|
|d f  |
d7 }
|
| jk sYd| j| j  |	 }|dd}tj dkrt|d}t|| W d   n1 sw   Y  td| d W nm tjjjy   |j  Y n] ty/ } zPt|}d|d}t j|st|d}t|| W d   n	1 sw   Y  td| d t| tt  |j  |`t !d W Y d}~nd}~ww |j  |`dS )zL
    main function to profile experiment for each pass hyper-parameter.
    z"There is no profile context named .rblatin1)encodingNZresult_filenamer   T)Z
fetch_listZuse_program_cachezstep: %d, loss_print: %fr6   g      ?)ZThroughtputZ	ErrorTypewzprofile done! avg speed : z
 step / s.rA   zprofile failed with error: [])"ospathisfileZctx_filename
ValueErroropenpickleloadrV   rZ   rR   r^   runZ_inner_dataloaderstartZprofile_end_steptimeZprofile_start_stepr*   rH   distributedZget_rankjsondump	frameworkcoreZEOFExceptionreset	Exceptionr5   	traceback
format_excsysexit)r&   r   rN   rL   rM   rY   Zdata_loaderresult_pathr]   durationZ	eval_step
start_timeZlossZend_timeZavg_tputZresult_dictfper2   r   r   r   profiler   sz   






r   __main__Z!FLAGS_new_executor_sequential_run)r6   N)"r   rp   re   rj   rx   rn   rv   rH   Z3paddle.distributed.auto_parallel.static.dist_loaderr   Z5paddle.distributed.auto_parallel.static.process_groupr   r   Zpaddle.distributed.collectiver   Zpaddle.frameworkr   r   Zpaddle.staticr   Zenable_staticr   r$   r-   r5   rR   rV   rZ   r^   r   __name__rr   Z	set_flagsr&   r   r   r   r   <module>   s:   	&
.U