o *j|3@sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z ddl mZmZddlmZmZddlZddlmZddlmZdd lmZmZmZmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&e"Z'd)deee(e)fde(fddZ*    d*de(de(de+de,de,de,dej-fddZ. d+dej-fdd Z/ d,deej-fd"d#Z0  $  % $  $   & $   d-de(fd'd(Z1dS).N)contextmanager)partial)Path)OptionalUnion)urljoinurlparse)tqdm)config)hash_url_to_filename"get_authentication_headers_for_url fsspec_head fsspec_get)FileLock)MS_DATASETS_CACHE) get_logger)HubApi) __version__ user_agentreturncCsdt}|dtj7}|dtj7}tjr |dtj7}tjr+|dtj7}tjr6|dtj 7}t |t rN|dd dd | D7}|St |trY|d|7}|S) Nz datasets/z ; python/z ; pyarrow/z; torch/z ; tensorflow/z; jax/z; css"|] \}}|d|VqdS)/N).0kvrrj/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/utils/hf_file_utils.py -s z-get_datasets_user_agent_ms..)rr Z PY_VERSIONZPYARROW_VERSIONZTORCH_AVAILABLEZ TORCH_VERSIONZ TF_AVAILABLEZ TF_VERSIONZ JAX_AVAILABLEZ JAX_VERSION isinstancedictjoinitemsstr)rZuarrrget_datasets_user_agent_ms"s  " r"?$@methodurl max_retriesbase_wait_time max_wait_timetimeoutc Ksd\}}d} |s_|d7}ztjd |||d|} d}Wn>tjjtjjfy\} z,||kr2| t|d|d||dt||d |d} t | WYd} ~ nd} ~ ww|r| S) aWrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. Note that if the environment variable HF_DATASETS_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. Args: method (str): HTTP method, such as 'GET' or 'HEAD'. url (str): The URL of the resource to fetch. max_retries (int): Maximum number of retries, defaults to 0 (no retries). base_wait_time (float): Duration (in seconds) to wait before retrying the first time. Wait time between retries then grows exponentially, capped by max_wait_time. max_wait_time (float): Maximum amount of time between two retries, in seconds. **params (additional keyword arguments): Params to pass to :obj:`requests.request`. )rFN)r&r'r+Tz request to z timed out, retrying... []r#r) requestsrequestupper exceptionsConnectTimeoutConnectionErrorloggerinfomintimesleep) r&r'r(r)r*r+paramstriessuccessresponseerrZ sleep_timerrr_request_with_retry_ms3s"  r>Tc Cs>t|pi}t|dd|d<td|||||||d}|S)N user-agentrHEAD)r&r'proxiesheaderscookiesallow_redirectsr+r()copydeepcopyr"getr>)r'rBrCrDrEr+r(r<rrr http_head_msZs rIY@c  Cs|durt|ni}t|dd|d<|dkr!d|dd|d<td|d |||||d } |dur3| S| jd kr:dS| jd } | durJ|t| nd} t| |d d |pTdd} | jddD]} | t | | | q]| dS)Nr?r@rzbytes=d-RangeGETT)r&r'streamrBrCrDr(r+izContent-LengthBZ Downloading)totalinitialZ unit_scaleunitdesci) chunk_size) rr"rHr> status_coderCintr iter_contentupdatelenwriteclose)r' temp_filerB resume_sizerCrDr+r(rTr<content_lengthrQprogresschunkrrr http_get_msls2     rbFd deprecatedc% s@| dkrtd| dt| } |durt}t|trt|}tj|dd| r1t |t |j }n|}d}d}d}d}d}d}t |dd}tj ||}|durVd |d }tj |rb|sb|sb|St|| d }|durp||d <|sTt |j}|d vrt|| d}|r|ddp|ddnd}d}ztj| d}t|d||| ||d}|jdkr|r|jdnd}|jD]\}}|drd|vr|d|7}|j}qd}d|vrd|vr|d7}nX|jdkrd|vs|jdkrd|vs|jdkr td|std|js|jdkr d|vr d}td |n|jd!kr9t j!|vr9| dur9t"d"|d#Wnt#t$j%j&fyS}z|}WYd}~nd}~ww|stj |rc|sc|S|rnt'd$|d%|dur|jd&krt'd'||durt"d(|d)t(|d*|durt"d(|d+|jd*t"d(|t ||}tj ||}tj |r|s|S|d,}t)|tj |r|s|WdS|d-t*d?fd/d0 }d1}|rt+|d2d3}tj rt,j-}|"} |d vrt.|| | |d4n t/|| ||||| |d5Wdn 1s,wYtd6|d7|t01| j2|t3d8}!t3|!t4|d8|!@td9|||d:}"|d;}#t5|#d}$t67|"|$Wdn1swYWd|SWd|S1swY|S)@a Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the path to the cached file. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk rdz'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. You can remove this warning by passing 'token=z ' instead.NT)exist_okF)etagz Downloading [r-)tokenr?)httphttps)storage_optionsETagrf)Z access_token)rErBr+r(rCrDZdownload_warningzdrive.google.comz &confirm=zconfirm=z &confirm=tizfirebasestorage.googleapis.comiiz7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url izUnauthorized for URL zU. Please use the parameter `token=True` after logging in with `huggingface-cli login`z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.izCouldn't find file at zCouldn't reach z ()z (error z.lockz .incompletew+bc3s8t| }|VWddS1swYdSN)open)modefZincomplete_pathrrtemp_file_manager0s "z,get_from_cache_ms..temp_file_managerrza+b)rq)rjrT)r]rBr^rCrDr(rTzstoring z in cache at izcreating metadata file for )r'rfz.jsonwzutf-8)encoding)rn)8warningswarn FutureWarningrrrr!osmakedirsrrpathr rexistsr schemer rHrZ get_cookiesrIrVrCrDr startswithrematchr'r4r5r Z HF_ENDPOINTr3OSErrorr.r1TimeoutFileNotFoundErrorreprrrrstatst_sizerrbshutilmovenameumaskchmodrpjsondump)%r' cache_dirZforce_downloadrBZ etag_timeoutZresume_downloadrZlocal_files_onlyZuse_etagr(rgZuse_auth_tokenZignore_url_paramsrjZ download_descZ disable_tqdmZ cached_url connectedr<rDrfZ head_errorr~filename cache_pathrCrreZ lock_pathrtr^r]rmeta meta_pathZ meta_filerrsrget_from_cache_mss"           "              1 11rro)r#r$r#r%)NNNTr%r)NrNNrJrN)NFNrcFNFTrNrdFNNN)2rrzrrFrr7rw contextlibr functoolsrpathlibrtypingrr urllib.parserrr.Z tqdm.autor Zdatasetsr Zdatasets.utils.file_utilsr r r rfilelockrZmodelscope.utils.config_dsrZmodelscope.utils.loggerrZmodelscope.hub.apirZ modelscoperr4r!rr"rWfloatResponser>rIrbrrrrrs            (  "