o
    "jb                     @   s8  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZ	d dl
Z
d dl
mZ d dlmZmZ d dlmZ d dlmZ ddlmZmZ dd	lmZmZmZ d
dlmZ d
dlmZmZ d
dlm Z m!Z! d
dl"m#Z#m$Z$m%Z%m&Z&m'Z' da(dd Z)e*e) G dd dZ+G dd de+Z,G dd de+Z-dS )    N)profiler)_current_expected_place_set_expected_place)	benchmark)in_profiler_mode   )corein_dynamic_mode   )MP_STATUS_CHECK_INTERVALCleanupFuncRegistrar_set_SIGCHLD_handler   )_InfiniteIterableSampler)default_collate_fndefault_convert_fn)_flatten_batch_restore_batch)_DatasetKind_IterableDatasetStopIteration_ResumeIteration_worker_loop_WorkerExceptionc                   C   s*   t d urzt   b W d S    Y d S d S N)_loader__del__ r   r   e/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/io/dataloader/dataloader_iter.py_clear_loaderC   s   r   c                   @   sD   e Zd ZdZdd Zedd Zdd Zdd	 Zd
d Z	dd Z
dS )_DataLoaderIterBasez
    Iterator implement of DataLoader, will load and feed mini-batch
    data by setting in given dataloader.

    Args:
        loader(instance of DataLoader): instance of `paddle.io.DataLoader`
    c                 C   s   |j | _|jpg | _|j| _|j| _|j| _	|j
| _|j| _|j| _|j| _|j| _|j| _|jdkr6|jnt| _|j| _|j| _|j| _t| j | _!| jrU|j"pRt#| _$n|j"pYt%| _$d | _&d | _'t() | _*d S )Nr   )+Zdataset_datasetZ	feed_list
_feed_listZplaces_placesZreturn_list_return_listbatch_sampler_batch_samplerZ	drop_last
_drop_lastZauto_collate_batch_auto_collate_batchZnum_workers_num_workersZuse_buffer_reader_use_buffer_readerZprefetch_factor_prefetch_factorZuse_shared_memory_use_shared_memorytimeoutr   _timeoutZworker_init_fn_worker_init_fnZdataset_kind_dataset_kindZ
pin_memory_pin_memoryiter_index_sampler_sampler_iterZ
collate_fnr   _collate_fnr   _blocking_queue_thread	threadingEvent_thread_done_eventselfloaderr   r   r   __init__Y   s.   z_DataLoaderIterBase.__init__c                 C   s6   | j r| jS | jtjkrttt| jS t	| jdS Nr   )
r'   r%   r/   r   ZMAPlistrangelenr    r   r;   r   r   r   r2   {   s
   z"_DataLoaderIterBase._index_samplerc                 C   s   | S r   r   rB   r   r   r   __iter__   s   z_DataLoaderIterBase.__iter__c                 C   s
   t | jS r   )rA   r%   rB   r   r   r   __len__   s   
z_DataLoaderIterBase.__len__c                 C   "   | j   | jr| j  d S d S r   )r9   setr5   closerB   r   r   r   _exit_thread_expectedly      
z+_DataLoaderIterBase._exit_thread_expectedlyc                 C   rE   r   )r9   rF   r5   killrB   r   r   r   _exit_thread_unexpectedly   rI   z-_DataLoaderIterBase._exit_thread_unexpectedlyN)__name__
__module____qualname____doc__r=   propertyr2   rC   rD   rH   rK   r   r   r   r   r   P   s    "
	r   c                       sP   e Zd ZdZ fddZdd Zdd Zdd	 Zd
d Zdd Z	dd Z
  ZS )_DataLoaderIterSingleProcesszg
    Single process implement of DataLoaderIter, loading data from
    loader.data in main process
    c                    sX   t  | t| j| j| j| j| j| _	g | _
| jt| j | _|   d| _| ad S NF)superr=   r   Zcreate_fetcherr/   r    r'   r4   r&   _dataset_fetcher_structure_infosr*   rA   r"   _blocking_queue_capacity_init_thread	_shutdownr   r:   	__class__r   r   r=      s   z%_DataLoaderIterSingleProcess.__init__c                 C   s   dd | j D | _dd | j D | _dd | j D | _dd | j D | _tt | jt	| j
dk| _t| j| j| j| j| j| j
| jd| j	| _tj| jt fd| _d| j_| j  d S )	Nc                 S      g | ]}|j qS r   name.0vr   r   r   
<listcomp>       z=_DataLoaderIterSingleProcess._init_thread.<locals>.<listcomp>c                 S   r[   r   shaper^   r   r   r   ra      rb   c                 S   r[   r   Zdtyper^   r   r   r   ra      rb   c                 S      g | ]}|j  qS r   descZneed_check_feedr^   r   r   r   ra          
r   Ttargetargs)r!   
_var_names_shapes_dtypes_need_check_feedr   init_lod_tensor_blocking_queueVariablerV   rA   r"   r5   create_py_readerr)   r0   _readerr7   Thread_thread_loopr   r6   daemonstartrB   r   r   r   rW      s6   
z)_DataLoaderIterSingleProcess._init_threadc           	   
   C   sr  t dtt|   t| | j szt| j}| j	
|| j}W n ty1   |   Y d S w |d u s;| j r<nwt|\}}| j| | j rNnezMt  }|D ],}t|tjt jjfrh|  }nt|t js|t  }||t   |}|| qU| j rW n*z| j| W n   |   Y W n ty } z|   |d }~ww | j r|   d S )NDataloader_)r   set_current_thread_namestridr   r9   is_setnextr3   rT   fetchStopIterationrH   r   rU   appendLoDTensorArray
isinstancepaddleTensoreagervalue
get_tensor	LoDTensorrF   CPUPlacer5   push	ExceptionrK   )	r;   legacy_expected_placeindicesbatch	structurearrayslottmper   r   r   rv      sT   





1z)_DataLoaderIterSingleProcess._thread_loopc                    sD  t  rtjdtjjd}|  zzvt   t   t	 r5t
j j d }t| jd}nB jrr j }tt|D ]
}||  ||< qC fddtt jD }dd t||D }t jdkrq|d }n j }t   |W W t  r|  S S  ty    j      w t  r|  w w )NrQ   r]   Z
event_typer   c                       g | ]} j d qS r   rU   popr_   _rB   r   r   ra   ,      
z9_DataLoaderIterSingleProcess.__next__.<locals>.<listcomp>c                 S      g | ]	\}}t ||qS r   r   r_   dsr   r   r   ra   0      r   )r   r   RecordEventTracerEventType
Dataloaderbeginr   check_if_need_recordbefore_readerr	   r   r   read_next_tensor_listrt   read_next_listr   rU   r   r#   r@   rA   _move_to_listr"   zip	read_nextafter_readerendr   shutdown_try_shutdown_allr;   Ztrace_eventdataistructsr   rB   r   __next__  sN   







z%_DataLoaderIterSingleProcess.__next__c                 C   s\   | j r,| j  tdD ]}| j  rtd q n| j t ur'| j 	  d | _ d S d S )Nr   r   )
r6   r9   rF   r@   is_alivetimesleepr7   current_threadjoinr;   r   r   r   r   _shutdown_threadC  s   



z-_DataLoaderIterSingleProcess._shutdown_threadc                 C   s>   | j sz| jr| j  d | _|   W d| _ d S d| _ w d S )NT)rX   r5   rG   r   rB   r   r   r   r   S  s   

z._DataLoaderIterSingleProcess._try_shutdown_allc                 C      |    d S r   r   rB   r   r   r   r   b     z$_DataLoaderIterSingleProcess.__del__)rL   rM   rN   rO   r=   rW   rv   r   r   r   r   __classcell__r   r   rY   r   rQ      s     <,rQ   c                       s   e Zd Z fddZdd Zdd Zdd Zd	d
 ZdddZd ddZ	dd Z
dd Zdd Zdd Zdd Zdd Zdd Z  ZS )!_DataLoaderIterMultiProcessc                    s"  t  | |j| _d| _| jdksJ d| j dd | _d| _d| _d| _i | _	g | _
| jt| jt| j | _t | _tjjdtjd| _tjdddv rlzdt| jd  | _W n   d| _td	 Y nd| _| jd
 | j | _ | !  t"| jD ]}| #  q| $  d| _%d S )Nr   z-Multi-process DataLoader invalid num_workers())lowhighZFLAGS_use_shm_cacheF)r   1TTruetruer   zUSetting the shm cache buffer size to 0, equivalent to not using the shm cache policy.r
   )&rS   r=   _persistent_workers_resume_worker_cntr(   _data_queue	_send_idx	_rcvd_idx_batches_outstanding_task_infosrU   r*   maxrA   r"   _outstanding_capacityr7   Lock_thread_locknprandomrandintsysmaxsize
_base_seedosenvirongetr    _worker_shm_buffer_sizewarningswarn_main_thread_shm_buffer_size_init_workersr@   _try_put_indicesrW   rX   )r;   r<   r   rY   r   r   r=   g  sF   


z$_DataLoaderIterMultiProcess.__init__c                 C   s  ddl m} g | _g | _g | _tt| j| _	|
 | _| | _t | _t| jD ]D}|
 }|  | j| |jt| j| j|| j| j| j| j| j| j|| j| j| j| jfd}d|_|  | j| | jd q,t t!| t"dd | jD  t#  d S )Nr   )multiprocessingrj   Tc                 s   s    | ]}|j V  qd S r   )pidr_   wr   r   r   	<genexpr>  s    z<_DataLoaderIterMultiProcess._init_workers.<locals>.<genexpr>)$Zpaddle.incubater   _workers_worker_status_indices_queues	itertoolscycler@   r(   _workers_idx_cycleQueuer   r8   _workers_done_eventr7   r9   cancel_join_threadr   Processr   r    r/   r'   r4   r&   r.   r+   r   r   rw   rx   r   Z_set_process_pidsr|   tupler   )r;   r   r   Zindices_queueworkerr   r   r   r     sH   


 
z)_DataLoaderIterMultiProcess._init_workersc                 C   sB   | j d ur	 z| j   W n   | j   | j   Y d S qd S r   )r   
get_nowaitr   rG   rB   r   r   r   _clear_and_remove_data_queue  s   


z8_DataLoaderIterMultiProcess._clear_and_remove_data_queuec                 C   s   dd | j D | _dd | j D | _dd | j D | _dd | j D | _tt | jt	| j
dk| _t| j t| j| j| j| j| j| j
| jd| j	| _t | _tj| jt fd| _d| j_| j  d S )	Nc                 S   r[   r   r\   r^   r   r   r   ra     rb   z<_DataLoaderIterMultiProcess._init_thread.<locals>.<listcomp>c                 S   r[   r   rc   r^   r   r   r   ra     rb   c                 S   r[   r   re   r^   r   r   r   ra     rb   c                 S   rf   r   rg   r^   r   r   r   ra     ri   r   Trj   )r!   rm   rn   ro   rp   r   rq   rr   r   rA   r"   r5   Z(_set_max_memory_map_allocation_pool_sizer   rs   r)   r0   rt   r7   r8   r9   ru   rv   r   r6   rw   rx   rB   r   r   r   rW     s:   

z(_DataLoaderIterMultiProcess._init_threadc                 C   s2  | j $ | j| _t| jD ]}| j| t  |  jd7  _qW d    n1 s*w   Y  | jdkr>t	d | jdks4| j
 t| jkrot rWtj| j d }n| jr`| j  n| j }| j
 t| jksHd| _d| _d| _i | _g | _dg| j | _t| j| _t| jD ]}|   qd S )Nr   r   g      ?T) r   r(   r   r@   r   putr   r   r   r   r5   sizerA   r"   r	   r   r   r   rt   r   r#   r   r   r   r   rU   r   r1   r2   r3   r   r   )r;   	worker_idr   r   r   r   r   _reset  s:   




z"_DataLoaderIterMultiProcess._resetFc                 C   s:   | j | s
| jr|r| j| d  d| j |< d S d S d S rR   )r   r   r   r   )r;   r   r   r   r   r   _shutdown_worker3  s   
z,_DataLoaderIterMultiProcess._shutdown_workerNc              
   C   s   | j sazR|   |   | j  t| jD ]	}| j|dd q| j s<| jD ]}|	| q&| j
D ]}|  |  q1W tt|  d| _ d S W tt|  d| _ d S tt|  d| _ w d S )NT)r   )rX   rH   r   r   rF   r@   r(   r   r   r   r   r   rG   r   Z_erase_process_pidsr|   )r;   r,   r   r   qr   r   r   r   :  s,   





z-_DataLoaderIterMultiProcess._try_shutdown_allc              
   C   sl  t dtt|   t| | j s|  }| j s|d u r&|   nt	|t
r:| jdks2J |  jd8  _qzjzNt  }| jrN|D ]}|| qEn/|D ],}t	|tjt jjfrc|  }nt	|t jswt  }||t   |}|| qP| j|s| j  W n ty } z|   |d }~ww W |  jd7  _n|  jd7  _w | j rd S d S )Nry   r   r   )r   rz   r{   r|   r   r9   r}   	_get_datarH   r   r   r   r   r+   r   r   r   r   r   r   r   rF   r   r5   r   rG   r   rK   r   )r;   r   r   r   Ztensorr   r   r   r   r   r   rv   Q  sL   




 z(_DataLoaderIterMultiProcess._thread_loopc              
   C   s  | j  sf| jtjkrL| j| jk r?| j| j }t|dks%| j	|d  r&n&| j| j= |  jd7  _|  j
d8  _
| j| jk s| jsL| j
t| jk rLd S | j| jv rot| j| j dkro| j| j}| j|d  |d S z
| jj| jd}W ny ty } zm| j  rW Y d }~q g }t| jD ]\}}| j	| r| s|| | | qt|dkr|   ddd |D }td	t|| W Y d }~d S t|ttj frW Y d }~q |   t!d
| d |d }~ww | jtjkrt|t"r| jrd| j	|j#< n| |j# |  j
d8  _
| $  q |\}}	}
t|t%r4|	d u r4|
d u r4|S t|	t&rB|   |	'  || jkrZ|| jv rR| j|= | j|
 |	S | j|  |	|
f7  < q d S )Nr   r   r   r
   )r,   z, c                 s   s    | ]}t |jV  qd S r   )r{   r   r   r   r   r   r     s    z8_DataLoaderIterMultiProcess._get_data.<locals>.<genexpr>z1DataLoader {} workers exit unexpectedly, pids: {}z DataLoader reader thread failed(z*) to read data from workers' result queue.F)(r9   r}   r/   r   ZITERr   r   r   rA   r   r   r   r"   r   rU   r   r   r   r-   r   	enumerater   r   r   rK   r   loggingwarningformatr   IOErrorqueueEmptyerrorr   r   r   r   r   reraise)r;   infor   r   Zfailed_workersr   r   Zpidsidxr   r   r   r   r   r     s   
	



 


z%_DataLoaderIterMultiProcess._get_datac              	   C   s   | j | jks
J d| j^ zt| j}W n ty%   Y W d    d S w t| jD ]}t| j}| j	| r9 n
q+	 W d    d S | j
| | j|f |f| j| j< |  j d7  _ |  jd7  _W d    d S 1 snw   Y  d S )Nz'too many indices have been put to queuer   )r   r   r   r~   r3   r   r@   r(   r   r   r   r   r   r   )r;   r   r   Z
worker_idxr   r   r   r     s,   

"z,_DataLoaderIterMultiProcess._try_put_indicesc                 C   r   r   r   rB   r   r   r   r     r   z#_DataLoaderIterMultiProcess.__del__c                 C   s   |  d d S r>   r   rB   r   r   r   _shutdown_on_exit  s   z-_DataLoaderIterMultiProcess._shutdown_on_exitc                    s  t  rtjdtjjd}|  zzt   t    j	t
 jk r4 jr*t j   j  t rLtj j d }t| jd}nB jr j }tt
|D ]
}||  ||< qZ fddtt
 jD }dd t||D }t
 jdkr|d }n j }    t !  |W W t  r|"  S S  ty    js j#   $   w t  r|"  w w )Nr   r   r   c                    r   r   r   r   rB   r   r   ra   B  r   z8_DataLoaderIterMultiProcess.__next__.<locals>.<listcomp>c                 S   r   r   r   r   r   r   r   ra   F  r   r   )%r   r   r   r   r   r   r   r   r   r   rA   r"   r   r   r9   rF   r5   rG   r	   r   r   r   rt   r   r   rU   r   r#   r@   r   r   r   _on_output_batchr   r   r   r   r   r   rB   r   r      s\   









z$_DataLoaderIterMultiProcess.__next__c                 C   s.   t t| jD ]}|  jd8  _|   qd S r>   )r@   rA   r"   r   r   r   r   r   r   r	  Z  s   
z,_DataLoaderIterMultiProcess._on_output_batch)Fr   )rL   rM   rN   r=   r   r   rW   r   r   r   rv   r   r   r   r  r   r	  r   r   r   rY   r   r   f  s    E0
"
+
/{:r   ).r   r   r   r  r   r7   r   r   numpyr   r   r   Zpaddle.base.frameworkr   r   Zpaddle.profiler.timerr   Zpaddle.profiler.utilsr   Z	frameworkr   r	   Zmultiprocess_utilsr   r   r   r$   r   Zcollater   r   Zflatr   r   r   r   r   r   r   r   r   r   registerr   rQ   r   r   r   r   r   <module>   s6   

F Q