o
    *j                     @   sL   d dl Z d dlmZ d dlmZ d dlmZ d dlm	Z	 G dd de
ZdS )    N)Pool)tqdm)OssUtilities)
UploadModec                   @   sb   e Zd ZdededefddZdededed	efd
dZdedededededed	efddZ	dS )DatasetUploadManagerdataset_name	namespaceversionc                 C   s    ddl m} t|||d| _d S )Nr   )HubApi)r   r   revision)Zmodelscope.hub.apir
   r   oss_utilities)selfr   r   r	   r
    r   i/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/utils/upload_utils.py__init__   s   zDatasetUploadManager.__init__object_namelocal_file_pathupload_modereturnc                 C   s   | j j||d|d}|S )NTZoss_object_namer   Zindicate_individual_progressr   r   upload)r   r   r   r   Z
object_keyr   r   r   r      s   zDatasetUploadManager.uploadobject_dir_namelocal_dir_pathnum_processes	chunksizefilter_hidden_filesc              
      s    fdd}g }t |D ]0\}	}
}|D ](}|r|drqt j||	|ddd|}t j|	|}|||f qqt|d}t	t
|j|||dt|d	}W d    t|S 1 sbw   Y  t|S )
Nc                    s     j j| d | d dd d S )Nr      Fr   r   )argsr   r   r   r   
run_upload!   s   
z3DatasetUploadManager.upload_dir.<locals>.run_upload. r   /)Z	processes)r   )total)oswalk
startswithpathjoinreplacestripappend
ThreadPoollistr   imaplen)r   r   r   r   r   r   r   r    Z
files_listrootdirsfiles	file_namer   r   poolresultr   r   r   
upload_dir   s0   
zDatasetUploadManager.upload_dirN)
__name__
__module____qualname__strr   r   r   intboolr7   r   r   r   r   r      s$    


	r   )r%   Zmultiprocessing.dummyr   r-   Z	tqdm.autor   Z%modelscope.msdatasets.utils.oss_utilsr   Zmodelscope.utils.constantr   objectr   r   r   r   r   <module>   s   