o
    *j0                     @   st  d Z ddlZddlZddlZddlmZ ddlmZmZm	Z	 ddl
mZmZ zddlmZmZmZmZmZmZmZmZ ddlmZ dZW n eyS   d	ZdZdZY nw es8ddlZddlZddlZddlZddlZdd
l m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) de*dee	e*e*e*ee* f  fddZ+de*de	e*ee* f fddZ,ej-dfde*de*fddZ.de*fddZ/de*de*de*de*de*f
ddZ0de*d e*de*d!e*d"ee	e*e*f  d#ee	e*e*f  de*fd$d%Z1d&e*d"ee	e*e*f  d#ee	e*e*f  de*de*de*de*ddfd'd(Z2de*de*de*de*de	e*e*f f
d)d*Z3d+e*fd,d-Z4e+Ze.Ze/Ze4Ze0Ze2Ze3ZdS dS ).a  Compatibility shims for datasets>=4.0 script-based dataset loading.

Script-based dataset loading was removed in datasets 4.0. This module
provides minimal re-implementations of the necessary helpers so that
ModelScope can still load datasets that ship a custom builder .py script.

When running with datasets<4.0 the real implementations are simply
re-exported from datasets.load / datasets.utils.py_utils.
    N)Path)ListOptionalTuple)DownloadModeconfig)!HubDatasetModuleFactoryWithScript#LocalDatasetModuleFactoryWithScriptresolve_trust_remote_code_get_importable_file_path_create_importable_file_load_importable_fileinit_dynamic_modulesfiles_to_hash)get_importsTF)urlparse)_hash_python_lines)url_or_path_join)hf_dataset_url)FileLock	file_pathreturnc           	         sr  t | dd}| }W d   n1 sw   Y  g }d}|D ]}td|}t|dkr1| }|r4q tjd|tjd  du rJtjd	|tjd  du rOq  drt fd
d|D r`q  dr} d}t	|\}}|
d d||f q  dr|
d d ddf q  dr|
d d ddf q |
d d ddf q |S )zHParse a dataset script for import statements (ported from datasets<4.0).utf-8encodingNFz[\s\S]*?"""[\s\S]*?   z=^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*))flagszQ^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)c                 3   s"    | ]}|d    dkV  qdS )r      N)group).0impmatch d/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/utils/_compat.py	<genexpr>U   s     z&_compat_get_imports.<locals>.<genexpr>   Zexternalr   ZinternalZlibrary)open	readlinesrefindalllenr"   	MULTILINEr   any_compat_convert_github_urlappend)	r   flinesZimportsZis_in_docstringlineZdocstr_start_matchurl_pathsub_directoryr#   r!   r$   _compat_get_imports;   s`   





r5   r3   c                 C   s   t | }d }|jdv r[|jdkr[d| v r+| ds!td|  d| dd} | |fS |jdd  }d	|v r;|d	n|d
f\}}|d\}}d| d| d| d} | d| }| |fS )N)httphttpsZs3z
github.comZblob.pyzExternal import from github at z should point to a .py filerawr   z/tree/Zmaster/zhttps://github.com/z	/archive/z.zip-)r   schemenetlocendswith
ValueErrorreplacepathsplit)r3   parsedr4   Zgithub_pathZ	repo_infobranchZ
repo_owner	repo_namer#   r#   r$   r.   i   s&   



	r.   namec                 C   s   t |ptj}|tjvrtj| tj|dd tj|d}tj	|s?t
|d W d    n1 s6w   Y  t  tj|| }tj|dd tj|d}tj	|spt
|d	 W d    |S 1 skw   Y  |S )NTexist_ok__init__.pyw)strr   ZHF_MODULES_CACHEsysrA   r/   osmakedirsjoinexistsr'   	importlibinvalidate_caches)rF   Zhf_modules_cacheZ	init_pathdynamic_modules_pathZ
init_path2r#   r#   r$   _compat_init_dynamic_modules   s(   

rT   c              	   C   s   g }| D ]}t j|r|tt|d q|| qg }|D ]}t|dd}||	  W d    n1 s=w   Y  q#t
|S )Nz
*.[pP][yY]r   r   )rM   rA   isdirextendlistr   rglobr/   r'   r(   r   )
file_pathsZto_use_filesfpr1   r0   r#   r#   r$   _compat_files_to_hash   s   r[   rS   module_namespacesubdirectory_namec                 C   s6   t j| ||dd}t j|||dd d S )Nr:   --r8   )rM   rA   rO   r@   rB   )rS   r\   r]   rF   importable_dirr#   r#   r$    _compat_get_importable_file_path   s   


ra   importable_directory_pathoriginal_local_pathlocal_importsadditional_filesc              	   C   sd  t j||}t j|| d }|d }	t|	 |tjkr*t j|r*t| t j	|dd t j|d}
t j|
sSt
|
d W d    n1 sNw   Y  t j	|dd t j|d}t j|s|t
|d W d    n1 sww   Y  t j|st|| t j|d d }t j|s||d	}t
|dd
d}t|| W d    n1 sw   Y  |D ]@\}}t j|rt j||d }t j|st|| qt j|rt j||}t j|st|| qtd| |D ]\}}t j||}t j|rt||st|| qW d    |S 1 s+w   Y  |S )Nr8   z.lockTrG   rI   rJ   r   z.json)zoriginal file pathzlocal file pathr   r   zError with local import at )rM   rA   rO   r   r   ZFORCE_REDOWNLOADrP   shutilrmtreerN   r'   copyfilesplitext_jsondumpisfilerU   copytreeImportErrorfilecmpcmp)rF   rb   r]   rc   rd   re   download_modeZimportable_subdirectoryZimportable_fileZ	lock_pathZinit_fpZinit_fp2	meta_pathmetamfZimp_nameZimp_pathdest	file_nameZoriginal_path	dest_pathr#   r#   r$   !_compat_copy_script_and_resources   st   	

))rx   
local_pathc           	   	   C   sd   t j|||dd}t|jddd t|jd jdd t|	dd ||| |||d d S )	Nr:   r^   T)parentsrH   rI   rG   r_   )rF   rb   r]   rc   rd   re   rq   )
rM   rA   rO   r@   r   mkdirparenttouchrx   rB   )	ry   rd   re   rS   r\   r]   rF   rq   r`   r#   r#   r$   _compat_create_importable_file   s   



r~   c              	   C   s6   d tj| ||dd||dd g}||fS )N.r:   r^   r_   )rO   rM   rA   basenamer@   rB   )rS   r\   r]   rF   module_pathr#   r#   r$   _compat_load_importable_file   s   

r   repo_idc                 C   s   | d u rt d| d| S )NzThe repository for z contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at the Hub.
Please pass the argument `trust_remote_code=True` to allow custom code to be run.)r?   )Ztrust_remote_coder   r#   r#   r$   !_compat_resolve_trust_remote_code  s
   
r   )5__doc__rQ   rM   rL   pathlibr   typingr   r   r   Zdatasetsr   r   Zdatasets.loadr   r	   r
   r   r   r   r   r   Zdatasets.utils.py_utilsr   Z_HAS_SCRIPT_LOADINGrn   ro   hashlibjsonrj   r)   rf   urllib.parser   Zdatasets.packaged_modulesr   Zdatasets.utils.file_utilsr   Zdatasets.utils.hubr   filelockr   rK   r5   r.   ZMODULE_NAME_FOR_DYNAMIC_MODULESrT   r[   ra   rx   r~   r   r   r#   r#   r#   r$   <module>   s   	(

.


8	



 