o
    "j7                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZddlmZ ddl	m
Z
mZmZ ddlmZmZ ddlmZ G d	d
 d
ZG dd dZG dd dZG dd dZdadd ZG dd dZG dd dZdZdZdZdZdZdZe ej!j"d d Z#dZ$d#dd Z%	 d$d!d"Z&dS )%    N   )core   )MP_STATUS_CHECK_INTERVALCleanupFuncRegistrar_cleanup_mmap   )_IterableDatasetFetcher_MapDatasetFetcher)_flatten_batchc                   @   s   e Zd Zdd ZdS )_IterableDatasetStopIterationc                 C   s
   || _ d S N)	worker_id)selfr    r   \/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/io/dataloader/worker.py__init__#   s   
z&_IterableDatasetStopIteration.__init__N)__name__
__module____qualname__r   r   r   r   r   r   "   s    r   c                   @   s   e Zd ZdS )_ResumeIterationN)r   r   r   r   r   r   r   r   '   s    r   c                   @   s    e Zd ZdZdZedd ZdS )_DatasetKindr   r   c                 C   s>   | t jkrt||||S | t jkrt||||S td|  )Nzunknown Dataset kind )r   MAPr
   ITERr	   NotImplementedError)kinddatasetauto_collate_batch
collate_fn	drop_lastr   r   r   create_fetcher/   s   

z_DatasetKind.create_fetcherN)r   r   r   r   r   staticmethodr    r   r   r   r   r   +   s
    r   c                   @   s   e Zd Zdd Zdd ZdS )ParentWatchDogc                 C   s   t  | _d| _d S NT)osgetppid_parent_pid_parent_aliver   r   r   r   r   @   s   

zParentWatchDog.__init__c                 C   s   | j rt | jk| _ | j S r   )r'   r$   r%   r&   r(   r   r   r   is_aliveD   s   zParentWatchDog.is_aliveN)r   r   r   r   r)   r   r   r   r   r"   ?   s    r"   c                   C   s   t S )a	  
    Get DataLoader worker process information function, this function is
    used to split data copy in worker process for IterableDataset
    (see :code:`paddle.io.IterableDataset`), worker information contains
    following fields:

    :attr:`num_workers`: total worker process number, see `paddle.io.DataLoader`

    :attr:`id`: the worker processs id, count from 0 to :attr:`num_workers - 1`

    :attr:`dataset`: the dataset object in this worker process

    Returns:
        WorkerInfo: an instance of WorkerInfo which contains fields above.

    Notes:
        For more usage and examples, please see :code:`paddle.io.IterableDataset`

    Example:

        .. code-block:: python

            >>> import math
            >>> import paddle
            >>> import numpy as np
            >>> from paddle.io import IterableDataset, DataLoader, get_worker_info

            >>> class SplitedIterableDataset(IterableDataset):
            ...     def __init__(self, start, end):
            ...         self.start = start
            ...         self.end = end
            ...
            ...     def __iter__(self):
            ...         worker_info = get_worker_info()
            ...         if worker_info is None:
            ...             iter_start = self.start
            ...             iter_end = self.end
            ...         else:
            ...             per_worker = int(
            ...                 math.ceil((self.end - self.start) / float(
            ...                     worker_info.num_workers)))
            ...             worker_id = worker_info.id
            ...             iter_start = self.start + worker_id * per_worker
            ...             iter_end = min(iter_start + per_worker, self.end)
            ...
            ...         for i in range(iter_start, iter_end):
            ...             yield np.array([i])
            ...
            >>> place = paddle.CPUPlace()
            >>> dataset = SplitedIterableDataset(start=2, end=9)
            >>> dataloader = DataLoader(
            ...     dataset,
            ...     places=place,
            ...     num_workers=2,
            ...     batch_size=1,
            ...     drop_last=True)
            ...
            >>> for data in dataloader:
            ...     print(data) # doctest: +SKIP("The output depends on the environment.")
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[2]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[6]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[3]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[7]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[4]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[8]])
            Tensor(shape=[1, 1], dtype=int64, place=Place(cpu), stop_gradient=True,
            [[5]])

    )_worker_infor   r   r   r   get_worker_infoO   s   Lr+   c                       s(   e Zd ZdZdd Z fddZ  ZS )
WorkerInfoFc                 K   s(   |  D ]
\}}t| || qd| _d S r#   )itemssetattr_WorkerInfo__initialized)r   kwargskvr   r   r   r      s   
zWorkerInfo.__init__c                    s(   | j rtd| jj dt ||S )NzCannot assign attributes to z objects)r/   RuntimeError	__class__r   super__setattr__)r   keyvalr4   r   r   r6      s
   zWorkerInfo.__setattr__)r   r   r   r/   r   r6   __classcell__r   r   r9   r   r,      s    r,   c                   @   s   e Zd ZdddZdd ZdS )_WorkerExceptionNc                 C   s2   || _ |pt }|d | _dtj| | _d S )Nr    )r   sysexc_infoexc_typejoin	tracebackformat_exceptionexc_msg)r   r   r>   r   r   r   r      s   
z_WorkerException.__init__c                 C   s:   d | j| jj| j}t| jdd r| j|d| |)Nz0DataLoader worker({}) caught {} with message:
{}message)rD   )formatr   r?   r   rC   getattr)r   msgr   r   r   reraise   s   
z_WorkerException.reraiser   )r   r   r   r   rH   r   r   r   r   r;      s    
r;   iװCl   u=& l   y iXl   y isI   l    c                    s   t t}fdd dd }|| t@ | d? dg} fdd|D }tt|D ]}tt|D ]}||krB|||  || ||< q/q'g }|D ]!}	|	|A t@ }
|t t@ }|
| t@ }
|
|
t? A t@ }
||
 qH|S )	Nc                    s8   |  A t @ }  t t @  |   t @ } | | t? A t @ } | S r   )MASK32MULT_AXSHIFT)value)hash_const_Ar   r   hash   s
   z_generate_states.<locals>.hashc                 S   s8   t |  t@ }t| t@ }|| t@ }||t? A t@ }|S r   )
MIX_MULT_LrJ   
MIX_MULT_RrL   )xyZresult_xZresult_yresultr   r   r   mix   s
   z_generate_states.<locals>.mix    r   c                    s   g | ]} |qS r   r   ).0Zentropy)rO   r   r   
<listcomp>   s    z$_generate_states.<locals>.<listcomp>)INIT_AINIT_BrJ   rangelenMULT_BrL   append)	base_seedr   Zhash_const_BrU   ZentropyspoolijZstatespstater   )rO   rN   r   _generate_states   s(   re   c              
      s  z]zFt t t  t| z
dd l}dd l W n	 ty%   Y nw ||	 }|	| t
	|  j	t||	 t|	|
| |dad }z|d urP||	 t|| |||}W n   t|	}Y d}t }| rHz|t}W n
 tjy~   Y qhw t|tr||d d f d}t|| ||d}qh|d u r| s|sJ dn| s|rqh|\}}z*|d ur|}d }nt
jjjt
 d ||}W d    n1 sw   Y  W n1 t y } z$t|t!r|tj"kr|t#|	 d}n
||t|	d f W Y d }~n;d }~ww t|tr|||d f t$|\}}|r<dd  fd	d
|D }||||f n||||f | smW n t%yS   Y n    W |r^t  n|rft  w w | rv|&  |'  d S d S )Nr   )idnum_workersr   seedFTz#get None when worker done_event set)Zplacec                 S   s   t  }|| t   |S r   )r   ZTensorsetCPUPlace)ZarrZ	lodtensorr   r   r   numpy2lodtensor  s   z%_worker_loop.<locals>.numpy2lodtensorc                    s(   g | ]}t | jr|n| qS r   )
isinstanceZndarrayZ
get_tensor)rW   bnprk   r   r   rX     s    

z _worker_loop.<locals>.<listcomp>)(r   registerr   r   Z_set_process_signal_handlerZ(_set_max_memory_map_allocation_pool_sizerandomnumpyImportErrorrh   paddlere   r,   r*   r   r    r;   r"   r)   getr   queueEmptyrl   r   putis_setbaseZdygraphguardrj   fetch	ExceptionStopIterationr   r   r   KeyboardInterruptZcancel_join_threadclose)r   Zdataset_kindZindices_queueZ	out_queueZ
done_eventr   r   r   Zinit_fnr   rg   Zuse_shared_memoryr_   Zshm_cahce_sizerq   rh   Zinit_exceptionfetcherZiterator_drainedZparent_watch_dogdataidxindicesbatcheZ	structureZtensor_listr   rn   r   _worker_loop  s   










C
r   )r   r   )r   )'r$   rv   r=   rA   rr   ro   rt   Z	frameworkr   Zmultiprocess_utilsr   r   r   r   r	   r
   Zflatr   r   r   r   r"   r*   r+   r,   r;   rY   rK   rZ   r]   rP   rQ   ZdtypeZuint32itemsizerL   rJ   re   r   r   r   r   r   <module>   s:   O0
7