o
    *j                     @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlmZ d dlmZmZ d d	lmZmZmZmZ G d
d deZdS )    N)Union)DatasetBuilder)HubApi)DatasetContextConfig)CsvDatasetBuilderIterableDatasetBuilderTaskSpecificDatasetBuilder)DataDownloadConfig)DataDownloadManagerDataStreamingDownloadManager)META_FILES_FORMATDatasetPathNameDownloadModeMetaDataFieldsc                   @   s<   e Zd ZdZdefddZdeedf fddZd	d
 Z	dS )DataFilesManagerz"The modelscope data-files manager.dataset_context_configc                 C   s$  |j | _ |j| _|j| _|j| _|j| _|jj| _|jj| _|jj| _|j	| _	|j
| _
|j| _|jp4t }|j |_ |j|_|j|_|j|_tj|j| j| j | jtj|_|j	tjk}t||_t||_d|_t }|| j | j| j| _| j|_| jdd|_ ||_|| _!tj"|jdd d S )NFnum_proc   T)exist_ok)#Zdataset_name	namespaceversionZsubset_namesplitdata_meta_configmeta_data_filesmeta_args_mapZzip_data_filesdownload_modeuse_streamingZconfig_kwargsinput_config_kwargsdownload_configr	   ospathjoinZcache_root_dirr   ZDATA_FILES_NAME	cache_dirr   ZFORCE_REDOWNLOADboolZforce_downloadZforce_extractZuse_etagr   Zget_dataset_access_configZ
oss_configgetr   r   makedirs)selfr   r   Zis_force_downloadapi r(   t/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/data_files/data_files_manager.py__init__   sB   




zDataFilesManager.__init__returnNc                 C   s   | j r
tj| jdS | jsdS tt| j }tt| j }|du r'i }|r/|	t
jrB|| j || jj_t| jd}|S |rVtj|d tv rVt| jd}|S tdtj|d  d)z Build download manager. )r   NzDataset meta file extensions "z" is not implemented yet)r   r   Zget_builder_instancer   r   nextitervaluesr   r$   r   ZARGS_BIG_DATAupdater   r   r   r   r    splitextr   r   NotImplementedError)r&   Zmeta_data_fileZmeta_args_map_filebuilderr(   r(   r)   get_data_files_builderB   sB   


z'DataFilesManager.get_data_files_builderc                 C   sZ   | j jrt| j jd}||S | j jj| j j_t| j jd}|j|| j	j
dd | S )z( Fetch the data-files from dataset-hub. )r   F)
dl_managerr   Ztry_from_hf_gcs)r   r   r   r   Zas_streaming_datasetr   r   r
   Zdownload_and_preparer   valueZ
as_dataset)r&   r3   r5   r(   r(   r)   fetch_data_filesb   s    
z!DataFilesManager.fetch_data_files)
__name__
__module____qualname____doc__r   r*   r   r   r4   r7   r(   r(   r(   r)   r      s
    + r   )r   typingr   Zdatasetsr   Zmodelscope.hub.apir   Z4modelscope.msdatasets.context.dataset_context_configr   Z.modelscope.msdatasets.download.dataset_builderr   r   r   Z.modelscope.msdatasets.download.download_configr	   Z/modelscope.msdatasets.download.download_managerr
   r   Zmodelscope.utils.constantr   r   r   r   objectr   r(   r(   r(   r)   <module>   s   