o
    )jF}                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZmZ d d
lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, d dl-m.Z.m/Z/ d dl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8 ddl9m:Z: ddl;m<Z<m=Z=m>Z> e1 Z?dZ@e(dddddddfdeAdeAdeeA deeA deeeAdf deeB dee deeA deeA deeA deeA fd d!ZCe'dddddddfd"eAdeAdeeA deeAedf deeA deeeeAf  deeB dee deeA deeA deAfd#d$ZDde(ddddddddd%
d&eAdeAd'eAdeeA deeA deeeAdf deeB dee deeA d(eBdeeA deeA deeA fd)d*ZEdeAdeAfd+d,ZFdde+fdeAdeAdeAd'eAfd-d.ZG	d@deAdeAdeAdeeA fd/d0ZHd1d2 ZI					dAd3eAdeAd4eAded5eeeAeAf  d6eJd(eBd7eee3  deAfd8d9ZK			dBd3eAdeAd4eAd6eJded5eeeAeAf  d(eBd7eee3  fd:d;ZL	d@d3eAdeAd4eAded5eeeAeAf  f
d<d=ZM		dCd7eee3  fd>d?ZNdS )D    N)ThreadPoolExecutor)partial)	CookieJar)Path)DictListOptionalTypeUnion)Retry)tqdm)HubApiModelScopeConfig)API_FILE_DOWNLOAD_CHUNK_SIZEAPI_FILE_DOWNLOAD_RETRY_TIMESAPI_FILE_DOWNLOAD_TIMEOUT	FILE_HASHMODELSCOPE_DOWNLOAD_PARALLELS)MODELSCOPE_PARALLEL_DOWNLOAD_THRESHOLD_MBTEMPORARY_FOLDER_NAME)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONINTRA_CLOUD_ACCELERATIONREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORT)get_dataset_cache_rootget_model_cache_root)
get_logger   )ProgressCallbackTqdmCallback)FileDownloadErrorInvalidParameterNotExistError)ModelFileSystemCache)file_integrity_validationget_endpointmodel_id_to_group_owner_name   Fmodel_id	file_pathrevision	cache_dir
user_agentlocal_files_onlycookies	local_dirtokenendpointreturnc
           
      C   s   t | |t||||||||	dS )a  Download from a given URL and cache it if it's not already present in the local cache.

    Given a URL, this function looks for the corresponding file in the local
    cache. If it's not there, download it. Then return the path to the cached
    file.

    Args:
        model_id (str): The model to whom the file to be downloaded belongs.
        file_path(str): Path of the file to be downloaded, relative to the root of model repo.
        revision(str, optional): revision of the model file to be downloaded.
            Can be any of a branch, tag or commit hash.
        cache_dir (str, Path, optional): Path to the folder where cached files are stored.
        user_agent (dict, str, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional):  If `True`, avoid downloading the file and return the path to the
            local cached file if it exists. if `False`, download the file anyway even it exists.
        cookies (CookieJar, optional): The cookie of download request.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.
        token (str, optional): The user token.
        endpoint (str, optional): The remote endpoint.

    Returns:
        string: string of local file or if networking is off, last version of
        file cached on disk.

    Raises:
        NotExistError: The file is not exist.
        ValueError: The request parameter error.

    Note:
        Raises the following errors:

            - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
            if `use_auth_token=True` and the token cannot be found.
            - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
            if ETag cannot be determined.
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            if some parameter value is invalid
    		repo_typer,   r-   r.   r/   r0   r1   r2   r3   )_repo_file_downloadr   )
r*   r+   r,   r-   r.   r/   r0   r1   r2   r3    r8   ]/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/hub/file_download.pymodel_file_download.      2r:   
dataset_idc
           
      C   s   t | |t||||||||	dS )aI  Download raw files of a dataset.
    Downloads all files at the specified revision. This
    is useful when you want all files from a dataset, because you don't know which
    ones you will need a priori. All files are nested inside a folder in order
    to keep their actual filename relative to that folder.

    An alternative would be to just clone a dataset but this would require that the
    user always has git and git-lfs installed, and properly configured.

    Args:
        dataset_id (str): A user or an organization name and a dataset name separated by a `/`.
        file_path (str): The relative path of the file to download.
        revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
            commit hash. NOTE: currently only branch and tag name is supported
        cache_dir (str, Path, optional): Path to the folder where cached files are stored, dataset file will
            be save as cache_dir/dataset_id/THE_DATASET_FILES.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.
        user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        cookies (CookieJar, optional): The cookie of the request, default None.
        token (str, optional): The user token.
        endpoint (str, optional): The remote endpoint.
    Raises:
        ValueError: the value details.

    Returns:
        str: Local folder path (string) of repo snapshot

    Note:
        Raises the following errors:
        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
        if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
        ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
        if some parameter value is invalid
    r5   )r7   r   )
r<   r+   r,   r-   r1   r.   r/   r0   r2   r3   r8   r8   r9   dataset_file_downloadn   r;   r=   )
r6   r,   r-   r.   r/   r0   r1   disable_tqdmr2   r3   repo_idr6   r>   c       
          C   s(  |st }|tvrtd|tf t| |||d\}}|r0||}|d ur,td |S tdt|
d}t	j
|dtt d}tdkr]td	pN| }|r]td
|   ||d< |d u re| }g }|d u rs|j| ||
d}d }|t kr|j| |||d}|j| |d|d u rdn||d}|D ],}|d dkrq|d |kr||r|d }td| d ||  S |} nqn|tkrXt| \}}|st}|j||||
d\}}d}d}	 z|j| |dd||||
|d	}W n ty } zt d|  d|  W Y d }~nMd }~ww d}|D ]3}|d dkrq|d |krF||r@|d }td| d ||  S |}d} nqt!||k sR|rSn|d7 }q|d u ret"d|| f |t krrt#| |||}n|tkr|j$|d ||||d}ntd | t%||||||S )!Nz'Invalid repo type: %s, only support: %s)r1   r-   r6   z>File exists in local cache, but we're not sure it's up to datezCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.)r2   )r.   )z
user-agentzsnapshot-identifiertrueZINTRA_CLOUD_ACCELERATION_REGIONz6Intra-cloud acceleration enabled for downloading from zx-aliyun-region-id)r?   r6   r2   )r,   r0   r3   TF)r*   r,   	recursiveZuse_cookiesr3   r	   treer   NamezFile z8 already in cache with identical hash, skip downloading!)dataset_name	namespacer3   r2   r   d   /)	r?   r,   	root_pathrA   page_number	page_sizer3   r2   Zdataset_hub_idzGet dataset: z file list failed, error: z"The file path: %s not exist in: %s)	file_namerD   rE   r,   r3   zInvalid repo type )&r   r   r#   $create_temporary_directory_and_cacheZget_file_by_pathloggerwarning
ValueErrorr   r   Zget_user_agentstruuiduuid4r   osgetenvZ!_get_internal_acceleration_domaininfoZget_cookiesZget_endpoint_for_readZget_valid_revisionZget_model_filesexistsdebugZget_file_by_infor   r(   r   Zget_dataset_id_and_typeZget_dataset_files	Exceptionerrorlenr$   get_file_download_urlZget_dataset_file_urldownload_file) r?   r+   r6   r,   r-   r.   r/   r0   r1   r>   r2   r3   temporary_cache_dircacheZcached_file_pathZ_apiheadersZ	region_idZ
repo_filesZfile_to_download_metaZ	repo_filerK   group_or_ownernameZ_hub_id_rI   rJ   Zdataset_fileseZis_existZurl_to_downloadr8   r8   r9   r7      s  














%

r7   c                 C   s   |  tjjr| tjj} tj| }tj| }|dkr d S | ds,tj|d}t|\}}|	dd}tj| ||}tj|||}tj
|rstj
|sutd| d|  z	t|| W d S  tyr   Y d S w d S d S )NZdatasetsZhub.___zLegacy cache dir exists: z
, move to )endswithrS   pathsepstripdirnamebasenamejoinr(   replacerV   rM   rU   shutilmoverX   )r-   r*   Zlegacy_cache_root	base_namer`   ra   r]   Zlegacy_cache_dirr8   r8   r9   !move_legacy_cache_to_standard_dirM  s4   


rq   c           	      C   s   |t krt }n|tkrt }ntd| t| \}}|d ur-tj|t	}t
|}n)|d u r8|}t||  t|trAt|}tj|t	||}|dd}t
|||}tj|dd ||fS )Nz7repo_type only support model and dataset, but now is : rd   re   Texist_ok)r   r   r   r   rO   r(   rS   rg   rl   r   r%   rq   
isinstancer   rP   rm   makedirs)	r*   r1   r-   r6   Zdefault_cache_rootr`   ra   r]   r^   r8   r8   r9   rL   p  s.   



rL   c                 C   s8   t j|}t j|}d}|st }|j|| ||dS )a  Format file download url according to `model_id`, `revision` and `file_path`.
    e.g., Given `model_id=john/bert`, `revision=master`, `file_path=README.md`,
    the resulted download url is: https://modelscope.cn/api/v1/models/john/bert/repo?Revision=master&FilePath=README.md

    Args:
        model_id (str): The model_id.
        file_path (str): File path
        revision (str): File revision.
        endpoint (str): The remote endpoint

    Returns:
        str: The file url.
    zQ{endpoint}/api/v1/models/{model_id}/repo?Revision={revision}&FilePath={file_path})r3   r*   r,   r+   )urllibparse
quote_plusr'   format)r*   r+   r,   r3   Zdownload_url_templater8   r8   r9   r[     s   r[   c              
   C   s  | \}}}}}}}}|d u ri nt |}	tt j|	d< ttddgd}
|d||f  }	 zd}tj	
|r^t|d}|dtj}|D ]}|| qGW d    n1 sYw   Y  || }||kriW d S d	||f |	d
< t|d4}tj|d|	|td}|  |jtdD ]}|r|| |D ]	}|t| qqW d    W d S 1 sw   Y  W d S  ty } z|
jd||d}
td||f  |
  W Y d }~nd }~ww q/)NX-Request-IDr   GETtotalbackoff_factorallowed_methods_%s_%sTr   rbbytes=%s-%sRangeab+streamr_   r0   timeout
chunk_sizerY   z-Downloading: %s failed, reason: %s will retry)copydeepcopyrP   rQ   rR   hexr   r   rS   rg   rV   openseekioSEEK_ENDupdaterequestsgetr   raise_for_statusiter_contentr   writerZ   rX   	incrementrM   rN   sleep)paramsZmodel_file_pathprogress_callbacksstartendurlrK   r0   r_   get_headersretrypart_file_namepartial_lengthfcallbackZdownload_startrchunkrc   r8   r8   r9   download_part_with_retry  sp   


r   r   rK   r_   	file_sizer   c	                    s   |d u rg n|  }|s|t  fdd|D }d}	g }
tj| }tjtj|dd tt	|	 D ]}||	 }|d |	 d }|
|||||  ||f q9|d k rl|
|||d d |  ||f t
td}t|dd	}t|t|
 W d    n1 sw   Y  |D ]}|  qt }ttj| d
N}|
D ]@}|d d|d |d f  }t|d}	 |dt }|sn|| || qW d    n1 sw   Y  t| qW d    | S 1 sw   Y  | S )Nc                       g | ]}| qS r8   r8   .0r   rK   r   r8   r9   
<listcomp>      
z%parallel_download.<locals>.<listcomp>i   
Trr   r      download)max_workersZthread_name_prefixwbr   r      r)   r   )r   appendr!   rS   rg   rl   ru   rj   rangeintminr   r   listmapr   r   hashlibsha256r   readr   r   r   remove	hexdigest)r   r1   rK   r0   r_   r   r>   r   r3   Z	PART_SIZEtasksr+   idxr   r   Z	parallelsexecutorr   hash_sha256output_filetaskr   Z	part_filer   r8   r   r9   parallel_download  sb   	





r   c              
      st  |du rg n|  }|s|t  fdd|D }|du r!i nt |}tt j|d< tj	
| }	tjtj	|	dd td| |	 d}
t }ttd	d
gd}	 zdkr}t|	d |D ]}|d	 qdW d   n1 svw   Y  W nd}tj	|	rd}
t|	d}|dtj}|D ]}|| qW d   n1 sw   Y  |krW ntd|d	 f |d< t|	d9}tj| d||td}|  |jtdD ]}|r|D ]	}|t| q| | |
s|| qW d   n1 sw   Y  W n" t!y% } zd}
|j"d
| |d}|#  W Y d}~nd}~ww qW|D ]}|$  q)|
r6dS |% S )aj  Download remote file, will retry 5 times before giving up on errors.

    Args:
        url(str):
            actual download url of the file
        local_dir(str):
            local directory where the downloaded file stores
        file_name(str):
            name of the file stored in `local_dir`
        file_size(int):
            The file size.
        cookies(CookieJar):
            cookies used to authentication the user, which is used for downloading private repos
        headers(Dict[str, str], optional):
            http headers to carry necessary info when requesting the remote file
        disable_tqdm(bool, optional): Disable the progress bar with tqdm.
        progress_callbacks(List[Type[ProgressCallback]], optional):
            progress callbacks to track the download progress.

    Raises:
        FileDownloadError: File download failed.

    Nc                    r   r8   r8   r   r   r8   r9   r   4  r   z'http_get_model_file.<locals>.<listcomp>rz   Trr   downloading %s to %sFr   r{   r|   r   zw+r   r   r   r   r   r   r   )&r   r   r!   r   rP   rQ   rR   r   rS   rg   rl   ru   rj   rM   rW   r   r   r   r   r   r   rV   r   r   r   r   r   r   r   r   r   rZ   r   rX   r   r   r   r   )r   r1   rK   r   r0   r_   r>   r   r   Ztemp_file_pathZ	has_retryr   r   r   r   r   r   r   rc   r8   r   r9   http_get_model_file  s   !




/r   c                 C   s  d}t tjd|dd}|du ri nt|}tt j|d< | }t	
d| |j ttdd	gd
}		 zS| }
d|
 |d< tj| d||td}|  |jd}|dur[t|nd}tddd||
d| d d}|jtdD ]}|r|t| || qq|  W n ty } z|	jd	| |d}	|	  W Y d}~nd}~ww q4W d   n1 sw   Y  t	
d| | tj !|j}||krt"|j d|||f }t	#| t$|t%|jtj &|| dS )aj  Download remote file, will retry 5 times before giving up on errors.

    Args:
        url(str):
            actual download url of the file
        local_dir(str):
            local directory where the downloaded file stores
        file_name(str):
            name of the file stored in `local_dir`
        cookies(CookieJar):
            cookies used to authentication the user, which is used for downloading private repos
        headers(Dict[str, str], optional):
            http headers to carry necessary info when requesting the remote file

    Raises:
        FileDownloadError: File download failed.

    r   F)modedirdeleteNrz   r   r   r{   r|   Tz	bytes=%d-r   r   zContent-LengthBi   zDownloading [])unitZ
unit_scaleZunit_divisorr}   initialdescr   r   zstoring %s in cache at %sz}File %s download incomplete, content_length: %s but the                     file downloaded length: %s, please download again)'r   tempfileNamedTemporaryFiler   r   rP   rQ   rR   r   rM   rW   ra   r   r   tellr   r   r   r   r_   r   r   r   r   r   rZ   r   closerX   r   r   rS   rg   getsizer   rY   r"   rm   rl   )r   r1   rK   r0   r_   r}   Ztemp_file_managerr   	temp_filer   Zdownloaded_sizer   content_lengthprogressr   rc   Zdownloaded_lengthmsgr8   r8   r9   http_get_filey  s   



(
r   c                 C   s@  t j||d }ttD ]}	td d |d k r6tdkr6t| ||d ||d u r*d n| |d ||d}
nt	| ||d |d ||||d}
t
|v r|t
 }d}|
d urh|
|krgtd|d  t||sgd	}nt||sod	}|s|	td k rtd
|d |	d t t j|rt | qtd|d tf  |||S )Nr   i  Sizer   )r_   r0   r   r>   r   )r   r_   r0   r>   r   TzCMismatched real-time digest for %s, falling back to full hash checkFz@Hash validation failed for %s, retrying download (attempt %d/%d)zLFile %s hash validation failed after %d attempts, the file may be corrupted.)rS   rg   rl   r   HASH_RETRY_TIMESr   r   r   get_dictr   r   rM   rN   r&   rV   r   r"   Zput_file)r   Z	file_metar]   r^   r_   r0   r>   r   r   Zhash_attemptZfile_digestZexpected_hashZ
hash_validr8   r8   r9   r\     sr   




r\   )N)NNFNN)NFN)FN)Or   r   r   rS   rn   r   rv   rQ   concurrent.futuresr   	functoolsr   http.cookiejarr   pathlibr   typingr   r   r   r	   r
   r   Zrequests.adaptersr   Z	tqdm.autor   Zmodelscope.hub.apir   r   Zmodelscope.hub.constantsr   r   r   r   r   r   r   Zmodelscope.utils.constantr   r   r   r   r   r   Zmodelscope.utils.file_utilsr   r   Zmodelscope.utils.loggerr   r   r    r!   errorsr"   r#   r$   Zutils.cachingr%   Zutils.utilsr&   r'   r(   rM   r   rP   boolr:   r=   r7   rq   rL   r[   r   r   r   r   r   r\   r8   r8   r8   r9   <module>   s  $ 	

C	

D	

  $
#
1

:

o
Y
