o
    *j                     @   s   d dl Z d dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ e ZG d	d
 d
e jZG dd de jZG dd deZG dd deZG dd deZdS )    N)ABCabstractmethod)load_dataset)HubApi)DatasetContextConfig)OssDownloader)EXTENSIONS_TO_LOAD)
get_loggerc                   @   s   e Zd ZdZdZdZdZdS )LocalDataLoaderTypezQ Supported data loader types for local dataset: huggingface, PyTorch, Tensorflow hf_data_loaderZtorch_data_loaderZtf_data_loaderN)__name__
__module____qualname____doc__HF_DATA_LOADERZTORCH_DATA_LOADERZTF_DATA_LOADER r   r   v/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/data_loader/data_loader_manager.pyr
      s
    r
   c                   @   s   e Zd ZdZdZdZdS )RemoteDataLoaderTypezI Supported data loader types for remote dataset: huggingface, modelscope r   Zms_data_loaderN)r   r   r   r   r   MS_DATA_LOADERr   r   r   r   r      s    r   c                   @   s2   e Zd ZdZdefddZedejfddZ	dS )	DataLoaderManagerz Data loader manager, base class.dataset_context_configc                 C   s
   || _ d S Nr   selfr   r   r   r   __init__#   s   
zDataLoaderManager.__init__data_loader_typec                 C   s   d S r   r   )r   r   r   r   r   r   &   s   zDataLoaderManager.load_datasetN)
r   r   r   r   r   r   r   enumEnumr   r   r   r   r   r       s
    r   c                       6   e Zd ZdZdef fddZdejfddZ  Z	S )LocalDataLoaderManagerz+Data loader manager for loading local data.r   c                    s   t  j|d d S )Nr   )superr   r   	__class__r   r   r   .   s   zLocalDataLoaderManager.__init__r   c                 C   s   | j j}| j j}| j j}| j j}| j j}| j j}| j j}| j j}	| j j	}
| j j
}| j j}tj|rKtj|d d}|tv rKd }|g}t|}|tjkrm|r[td| d t|f|||||||	j|
d|S dtjj d)N   .3Use trust_remote_code=True. Will invoke codes from 9. Please make sure that you can trust the external codes.)namerevisionsplitdata_dir
data_files	cache_dirdownload_mode	streamingz!Expected local data loader type: )r   dataset_namesubset_nameversionr*   r+   r,   cache_root_dirr.   use_streamingtrust_remote_codeconfig_kwargsospathisfilesplitextstripr   getr
   r   loggerwarningr   value)r   r   r0   r1   r2   r*   r+   r,   r3   r.   r4   r5   input_config_kwargsZfile_extr   r   r   r   1   sJ   



z#LocalDataLoaderManager.load_dataset
r   r   r   r   r   r   r   r   r   __classcell__r   r   r"   r   r    +   s    r    c                       r   )RemoteDataLoaderManagerz,Data loader manager for loading remote data.r   c                    s    t  j|d t|jd| _d S )Nr   )token)r!   r   r   rD   apir   r"   r   r   r   a   s   z RemoteDataLoaderManager.__init__r   c                 C   s  | j j}| j j}| j j}| j j}| j j}| j j}| j j}| j jj	}	| j j
}
| j j}| j j}| j j}|tjkr]|rAtd| d t|f||||||	|
|d|}| jj|||
d |S |tjkrxt| j d}|  | jj|||
d |jS dtjj	 dtjj	 d| d	)
Nr&   r'   )r(   r)   r*   r+   r,   r.   r/   rD   )r0   	namespacer4   r   z"Expected remote data loader type: /z
, but got z .)r   r0   rF   r1   r2   r*   r+   r,   r.   r?   r4   r6   r5   rD   r   r   r=   r>   r   rE   Zdataset_download_statisticsr   r   processZdataset)r   r   r0   rF   r1   r2   r*   r+   r,   Zdownload_mode_valr4   r@   r5   rD   Zdataset_retZoss_downloaderr   r   r   r   f   sj   




z$RemoteDataLoaderManager.load_datasetrA   r   r   r"   r   rC   ^   s    rC   )r   r7   abcr   r   Zdatasetsr   r   Zmodelscope.hub.apir   Z4modelscope.msdatasets.context.dataset_context_configr   Z-modelscope.msdatasets.data_loader.data_loaderr   Zmodelscope.utils.constantr   Zmodelscope.utils.loggerr	   r=   r   r
   r   r   r    rC   r   r   r   r   <module>   s   3