o
    "j$                     @   s   d dl Z d dlZd dlZd dlmZmZ d dlmZm	Z	 d dl
mZmZmZ G dd de jdZG dd	 d	eZG d
d deZdS )    N)BatchSamplerIterableDataset)DistributedBatchSampler_InfiniteIterableSampler)_DatasetKinddefault_collate_fndefault_convert_fnc                   @   s   e Zd Zejdd ZdS )DistributedDataLoaderBasec                 C   s   t N)NotImplementedErrorself r   t/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/auto_parallel/static/dist_loader.py__iter__!   s   z"DistributedDataLoaderBase.__iter__N)__name__
__module____qualname__abcabstractmethodr   r   r   r   r   r	       s    r	   )	metaclassc                   @   sb   e Zd Zdddddddddddddg g dfddZdd Zd	d
 Zdd Zedd Zdd Z	dS )"DistributedDataLoaderFromGeneratorNTF   c                 C   sP  || _ || _|| _|| _|| _|| _|| _|| _|	| _|
| _	|| _
|| _|| _|| _t|t|ks4J t|t|ks>J || _|| _|| _t|trQtj| _ntj| _| j	d u r^d | _nt|trjt||
| _n	t||
d|d| _| jd u| _t| j| _| jr|pt| _n|pt | _t!| j| j | j| j| j| _"| # | _$| % | _&d S )NF)
batch_sizeshuffle	drop_last)'dataset	feed_listcapacityuse_double_bufferiterablereturn_listuse_multiprocessr   placesr   epochssteps_per_epoch
collate_fn
split_datalendp_world_sizesdp_ranks	acc_steps
isinstancer   r   ZITERdataset_kindMAPbatch_samplerr   r   auto_collate_batchiterindex_samplersampler_iterr   r   create_fetcherdataset_fetcher_infer_steps_steps_create_inner_dataloader_inner_dataloader)r   r   r   r   r   r    r!   r"   r   r#   r   r$   r%   r&   r'   data_parallel_world_sizedata_parallel_rankr+   r   r   r   __init__'   s`   





z+DistributedDataLoaderFromGenerator.__init__c                 C   s   d| _ | j  | S )Nr   )	_cur_stepr9   startr   r   r   r   r   v   s   
z+DistributedDataLoaderFromGenerator.__iter__c                 C   sP   | j s|  jd7  _d S | j| j k r|  jd7  _d S | j  t| j| _tNr   )r7   r=   r9   resetr1   r2   r3   StopIterationr   r   r   r   __next__{   s   
z+DistributedDataLoaderFromGenerator.__next__c                 C   s~   t | jtr| jdkr| jS z)t | jtrd }W |S | jd u r*t| j| j }W |S t| j| j | j }W |S    td)Nr   zLPlease set `steps_per_epoch` or implement `__len__` method in dataset class.)	r,   r%   intr   r   r   r(   r+   
ValueError)r   r%   r   r   r   r6      s    
	z/DistributedDataLoaderFromGenerator._infer_stepsc                 C   s6   | j r| jS | jtjkrttt| jS t	| jdS r?   )
r0   r/   r-   r   r.   listranger(   r   r   r   r   r   r   r2      s
   z0DistributedDataLoaderFromGenerator.index_samplerc              	      sH    fdd}t jjjj j j jd j j	 j
d}|| j |S )Nc               
   3   s    	 zt  j}  j| }|d u rW d S W n ty/   t j j j	 j
 j _Y d S w g }t|D ]@\}}t|} jsH|| q6|jd }| j|  dkseJ dt|t j| |t| j|  j|   q6|V  q)NTr   z6batch_size [{}] is not divisible by dp_world_size [{}])nextr3   r5   fetchrA   r   r4   r-   r   r0   r&   r   	enumeratenparrayr'   appendshaper)   formatstrsplitr*   )indicesbatchZpartial_dataidrK   r   r   r   r   data_generator   sJ   




zSDistributedDataLoaderFromGenerator._create_inner_dataloader.<locals>.data_generatorF)r   r   r   r    r!   r"   r   )paddlebaseio
DataLoaderZfrom_generatorr   r   r   r!   r"   r   Zset_batch_generatorr#   )r   rU   Z
dataloaderr   r   r   r8      s   
&
z;DistributedDataLoaderFromGenerator._create_inner_dataloader)
r   r   r   r<   r   rB   r6   propertyr2   r8   r   r   r   r   r   &   s0    
O
	r   c                   @   sP   e Zd Zdddddddddddddddg g fddZdd	 Zd
d Zdd ZdS )DistributedDataLoaderNTr   Fr   c                 C   s   || _ || _|| _|| _|| _|| _|| _|| _|	| _|
| _	|| _
|| _|| _|| _|| _|| _|| _|| _t| j | j| jd | jd | j| jd| _tjj| j | j| j| j| j| j| j| j	| j
| j| jd| _d S )Nr   )r   r   Znum_replicasrankr   r   )
r   r#   r!   r/   r&   num_workersuse_buffer_readeruse_shared_memorytimeoutworker_init_fn)r   r   r!   r#   r   r   r   r&   r]   r^   r_   r`   ra   r$   r%   r)   r*   r'   r   r/   rV   rX   rY   _dataloader)r   r   r   r#   r!   r   r   r   r&   r]   r^   r_   r`   ra   r$   r%   r'   r:   r;   r   r   r   r<      sN   zDistributedDataLoader.__init__c                 C   s
   t | jS r
   )r(   rb   r   r   r   r   __len__     
zDistributedDataLoader.__len__c                 C   
   | j  S r
   rb   r   r   r   r   r   r     rd   zDistributedDataLoader.__iter__c                 C   re   r
   rf   r   r   r   r   __call__  rd   zDistributedDataLoader.__call__)r   r   r   r<   rc   r   rg   r   r   r   r   r[      s,    
>r[   )r   numpyrJ   rV   Z	paddle.ior   r   Z"paddle.io.dataloader.batch_samplerr   r   Z$paddle.io.dataloader.dataloader_iterr   r   r   ABCMetar	   r   r[   r   r   r   r   <module>   s    4