o
    *j#                     @   s   d dl Z d dlZd dlmZ d dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZmZmZmZ G d	d
 d
eZdS )    N)defaultdict)FileLock)HubApi)DatasetContextConfig)DataMetaConfig)get_dataset_filesget_target_dataset_structure)REPO_TYPE_DATASETDatasetFormationsDatasetPathNameDownloadModec                   @   s\   e Zd ZdZdefddZdddZd	d
 ZdddZdd Z	de
de
de
de
fddZdS )DataMetaManagerzData-meta manager.dataset_context_configc                 C   s   || _ t | _d S )N)r   r   api)selfr    r   m/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/meta/data_meta_manager.py__init__   s   zDataMetaManager.__init__returnNc              
   C   sN  | j j}| j j}| j j}| j j}| j j}| j j}| j j}tj	
||||}tj	
|tj}	| j jp3t }
|s:tj}n|}|sBtj}n|}tj tj | tj | tj | tj | d
}tj	
||}tj|dd |tjkrtj	|	rt|	r| |	\}}nwt|d tj|	dd | ||||	\}}W d    n1 sw   Y  nP|tjkrtj	|	rt|	rtj|	dd t|d tj|	dd | ||||	\}}W d    n1 sw   Y  ntdtjj dtjj d| d	|	|
_||
_||
_ d
|v r|d
 }t!|dkr|d |
_"|
| j _|| j _#|| j _$d S )Nz.lockT)exist_ok)	lock_file)ignore_errorsz"Expected values of download_mode: z or z
, but got  ..pyr   )%r   dataset_name	namespacedownload_modeversioncache_root_dirsubset_namesplitospathjoinr   Z	META_NAMEdata_meta_configr   ZLOCK_FILE_NAME_ANYZLOCK_FILE_NAME_DELIMITERmakedirsr   ZREUSE_DATASET_IF_EXISTSexistslistdir_fetch_meta_from_cacher   _fetch_meta_from_hubFORCE_REDOWNLOADshutilrmtree
ValueErrorvaluemeta_cache_dirdataset_scriptsdataset_formationlendataset_py_scriptdataset_version_cache_root_dirZglobal_meta_lock_file_path)r   r   r   r   r   r   r   r    r4   r/   r$   Zlock_subset_nameZ
lock_splitZlock_file_nameZlock_file_pathr0   r1   Ztmp_py_scriptsr   r   r   fetch_meta_files   s   






z DataMetaManager.fetch_meta_filesc              	   C   sF  | j j}| j j}| j j}| j j}| j j}| j jpt }d }d }|j}	|	r*t	|	dkr.t
dd|	v r8|	d d }|	d D ]'}
|
| drct|
dd}t|}W d    n1 s\w   Y   nq<|ss|sst
d| d| d	|ry||_n$t|||\}}t|||| j |\}}}}||_||_||_||_||_|| j _d S )
Nr   zFCannot find dataset meta-files, please fetch meta from modelscope hub.r   z.jsonzutf-8)encodingzFile z
.json and z4.py not found,please specify at least one meta-file.)r   r   r   r    r   r   r$   r   r0   r2   FileNotFoundErrorendswithopenjsonloadr3   r   r   Zmeta_data_filesZzip_data_filesZmeta_args_mapZmeta_type_maptarget_dataset_structure)r   r   r   r    r   r   r$   Zdataset_jsonr3   r0   Z	json_pathZdataset_json_fileZtarget_subset_namer<   Zmeta_mapZfile_mapZargs_mapZtype_mapr   r   r   parse_dataset_structurei   sR   z'DataMetaManager.parse_dataset_structurec                 C   s6   | j j}t| j j}| jj||d}| j j| d S )N)
dataset_idr   )r   r   intr   r   Zget_virgo_metaZconfig_kwargsupdate)r   Zvirgo_dataset_idr   Zmeta_contentr   r   r   fetch_virgo_meta   s   z DataMetaManager.fetch_virgo_metac                 C   s   t t}d }t|D ]'}tj|d }|tjjkr&t	tj|d }q|| 
tj|| q|sCttjj dtjj d|t|fS )Nr   z! file does not exist, please use r   )r   listr!   r'   r"   splitextr
   Zformation_mark_extr.   r?   appendr#   r7   r   r*   )r   r/   local_pathsdataset_typeZmeta_file_nameZfile_extr   r   r   r(      s"   
z&DataMetaManager._fetch_meta_from_cacher   r   revisionr/   c                 C   sd   t  }|j|d | td}| j|||\}}| j||||}	| j||||||	\}
}|
|fS )N/)Zrepo_idZ	repo_type)r   Zget_endpoint_for_readr	   r   Zget_dataset_id_and_typeZget_dataset_meta_file_listZ"get_dataset_meta_files_local_paths)r   r   r   rH   r/   Z_apiZendpointr>   rG   	file_listrF   r1   r   r   r   r)      s    

z$DataMetaManager._fetch_meta_from_hub)r   N)__name__
__module____qualname____doc__r   r   r5   r=   rA   r(   strr)   r   r   r   r   r      s    
N
/
r   )r!   r+   collectionsr   r:   filelockr   Zmodelscope.hub.apir   Z4modelscope.msdatasets.context.dataset_context_configr   Z+modelscope.msdatasets.meta.data_meta_configr   Z)modelscope.msdatasets.utils.dataset_utilsr   r   Zmodelscope.utils.constantr	   r
   r   r   objectr   r   r   r   r   <module>   s   