o )j~@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZddlmZddlmZddlmZddlmZddlmZddlmZmZmZmZmZm Z m!Z!m"Z"m#Z#dd l$m%Z%ddl&Z&ddl'Z'dd l'm(Z(dd l)m*Z*m+Z+dd l,m-Z-dd l.m/Z/ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^ddl_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZiddljmkZkddllmmZmmnZnddlompZpddlqmrZrddlsmtZtddlumvZvmwZwmxZxmyZyddlzm{Z{ddl|m}Z}m~Z~mZmZmZmZmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZmZddlmZddlmZmZmZmZmZmZmZmZeZdedefddZGd d!d!ZGd"d#d#ZGd$d%d%ZGd&d'd'ZdS)(N) defaultdict)ThreadPoolExecutor) HTTPStatus) CookieJar) expanduserPath) AnyBinaryIODictIterableListLiteralOptionalTupleUnion) urlencode)Session) HTTPAdapterRetry) HTTPErrortqdm).API_HTTP_CLIENT_MAX_RETRIESAPI_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_MAX_WORKERSDEFAULT_MODELSCOPE_INTL_DOMAINMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_CREDENTIALS_PATHMODELSCOPE_DOMAINMODELSCOPE_PREFER_AI_SITEMODELSCOPE_REQUEST_IDMODELSCOPE_URL_SCHEMEONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODTEMPORARY_FOLDER_NAMEUPLOAD_ADAPTIVE_BATCH_SIZEUPLOAD_BLOB_MAX_RETRIESUPLOAD_BLOB_RETRY_BACKOFFUPLOAD_BLOB_RETRY_MAX_WAITUPLOAD_BLOB_TIMEOUT"UPLOAD_BLOB_TQDM_DISABLE_THRESHOLDUPLOAD_COMMIT_BATCH_SIZEUPLOAD_FAILED_FILE_MAX_RETRIESUPLOAD_MAX_FILE_COUNTUPLOAD_MAX_FILE_COUNT_IN_DIRUPLOAD_MAX_FILE_SIZE#UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT!UPLOAD_REACT_BACKOFF_MAX_EXPONENTUPLOAD_REACT_ENABLEDUPLOAD_REACT_MAX_DELAYUPLOAD_REACT_ROUND2_BASE_DELAYUPLOAD_REACT_ROUND3_FILE_DELAYUPLOAD_RETRY_ALLOWED_METHODS$UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFSUPLOAD_USE_CACHEUPLOAD_VALIDATE_BLOB_BATCH_SIZEVALID_SORT_KEYSDatasetVisibilityLicensesModelVisibility Visibility VisibilityMap) InvalidParameter NotExistErrorNotLoginException RequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper) DatasetInfo ModelInfo Repository)UPLOAD_HASH_CACHE_FILE) BatchTracker)_LEGACY_PROGRESS_FILE NullTracker UploadTrackerclassify_error) AigcModel)add_content_to_file get_domain get_endpointget_readable_folder_sizeget_release_datetime is_env_truemodel_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORT ConfigFieldsDatasetFormationsDatasetMetaFormatsDownloadChannel DownloadMode Frameworks ModelFileTasksVirgoDatasetConfig)compute_file_hash get_file_sizeis_relative_path) get_logger)DATASET_LFS_SUFFIXDEFAULT_IGNORE_PATTERNSMODEL_LFS_SUFFIXCommitHistoryResponse CommitInfoCommitOperationCommitOperationAdd RepoUtils total_filesreturncCsN|dkrdS|dkr |S|dkrtd|dS|dkr%tdtd|dSd S) aCalculate optimal commit batch size based on total file count. Adaptive strategy ensures batch granularity scales with workload: - Very few files (1-10): no splitting, single batch - Few files (11-100): ~10 batches for failure isolation - Medium (101-10K): 64-256 files per batch - Large (>10K): 512 files per batch to limit commit frequency Args: total_files: Total number of files (including checkpoint-skipped). Returns: Recommended batch size (>= 1). r di'@Pi)maxmin)rrS/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/hub/api.py_calculate_adaptive_batch_sizevsrc@s6eZdZdZddZd ddZeddZd d Zd S) _CountedReadStreamzFile wrapper that counts bytes read and updates a progress bar. Unlike a generator, this is a file-like object that requests can use with Content-Length header for transfer integrity verification. cCs"||_||_||_||_d|_dSNr)_file_expected_size_pbar _chunk_size _bytes_read)selfZfile_objZ expected_sizepbar chunk_sizerrr__init__s  z_CountedReadStream.__init__cCsP|dkr|jnt||j}|j|}|r&t|}|j|7_|j||S)z-Read a chunk from the underlying file object.r)rrrreadlenrrupdate)rsize read_sizechunknrrrrs  z_CountedReadStream.readcCs|jS)zTotal bytes read so far.)rrrrr bytes_readsz_CountedReadStream.bytes_readcCs*|j|jkrtd|jd|jddS)z9Raise IOError if bytes read does not match expected size.zUpload data incomplete: read z bytes, expected z2 bytes. File may have been modified during upload.N)rrIOErrorrrrrverify_completes  z"_CountedReadStream.verify_completeN)r) __name__ __module__ __qualname____doc__rrpropertyrrrrrrrs   rc@sHeZdZUdZdeedfdeedeefddZdefdd Z d deed ee fd d Z  d deedeefddZ e jejdddddfdedeedeedeedeedeedeededdefddZ   ddededeedeededdef ddZd dedeedeefddZddedeefd d!Zeddfded"eedeedeedef d#d$Zddd%d&ed'eedeedefd(d)Zedd*d&ed"eedeedefd+d,Zddd*d&ed"eedeedefd-d.Zeedd/d&ed'eed"eedeedeeeff d0d1Z ddd dd2d&ed'eedeed3ee deede f d4d5Z!  d d&ed'edeedeefd6d7Z"e#d8d9Z$e jejdd:de%ddddf ded;edeedeedeedeee&eefd?eeee&efdeefd@dAZ' B C  ddDedEeedFeedeedeedef dGdHZ(dBdCdddddIdDedEeedFeedJeedKeedeedeedefdLdMZ)ddNee e*fde*fdOdPZ+  d dedQeedNee e*fde&efdRdSZ,  ddedQeedNee e*fdeede&ef dTdUZ-dVdWZ.   ddedXee*deefdYdZZ/   ddedXee*deefd[d\Z0 ddedNee e*fdeede1e&ee&effd]d^Z2 ddedNee e*fde1e&ee&effd_d`Z3edd d idfded"eedaeedbee dNee e*fdceedeede&efdddeZ4dddfd&edged"eedeefdhdiZ5deje6jdddfdjedkedeedeedeedleedeedeedefdmdnZ7  d doedeedeefdpdqZ8iZ9ee:dr<  d djedkedeedeefdsdtZ; u B v  ddjedked"edwedbe dEedFedeedeefdxdyZe%ddfdoed"eedeedeefddZ?  d djedkedoed"edeedeef ddZ@e#dedefddZA  d djedked"edededeBdeedeefddZCe#deDjEdfdeefddZFeGd dudfdedjedked"eedee dee deefddZHeGdfdedjedked"eedeef ddZIeGddfdjedked"eedeedeef ddZJeGddfdjedkede d"eedeedeef ddZKddoededeedefddZL  d djedked"ededeedeef ddZM  d deedeefddZN  d dedjedked"edeedeedefddZO  d dedjedked"edeedeedefddZPddeefddZQ  ddjedkede deedeeddf ddZRddZS  d deede deTeeffddZUe#dddeTeeVffddZWdd&edeedefddZXdeYjedejdd dudd d&edeee dfdeed'eedeedeedeedee dee deeZdefddZ[ddee%ddd&ede\e]dGet dataset information including commit history. Args: repo_id (str): The dataset id in the format of ``namespace/dataset_name``. revision (str, optional): Specific revision of the dataset. Defaults to ``None``. endpoint (str, optional): Hub endpoint to use. When ``None``, use the endpoint specified when initializing :class:`HubApi`. Returns: DatasetInfo: The dataset detailed information returned by ModelScope Hub with commit history. ) dataset_idrrr7T)r+rr8rr9Nr)rc get_datasetr=riget_dataset_filesrdrR) rr+rrr r?Z dataset_datar:r< dataset_inforrrrDs  zHubApi.dataset_info)r*rrcCsN|dus|tkr|j|||dS|tkr|j|||dStd|dtd)aRGet repository information for models or datasets. Args: repo_id (str): The repository id in the format of ``namespace/repo_name``. revision (str, optional): Specific revision of the repository. Currently only effective for model repositories. Defaults to ``DEFAULT_MODEL_REVISION``. repo_type (str, optional): Type of the repository. Supported values are ``"model"`` and ``"dataset"``. If not provided, ``"model"`` is assumed. endpoint (str, optional): Hub endpoint to use. When ``None``, use the endpoint specified when initializing :class:`HubApi`. Returns: Union[ModelInfo, DatasetInfo]: The repository detailed information returned by ModelScope Hub. N)r+rrArg repo_type z# not supported. Please choose from r)rjr@rirDrGrk)rr+r*rrrrr repo_infoszHubApi.repo_infor/r,c Cs|dur|j}|dur|tvrtd||dus"|ddkr(td||j|dd}t|\}}|durJ|tkrJ|d|d|} n |d |d|} |jj | || |j d } t | t ||d} | d krod S| d kr{|ryt| dSt d| dtd|)a Checks if a repository exists on ModelScope Args: repo_id (`str`): A namespace (user or an organization) and a repo name separated by a `/`. repo_type (`str`, *optional*): `None` or `"model"` if getting repository info from a model. Default is `None`. TODO: support studio endpoint(`str`): None or specific endpoint to use, when None, use the default endpoint set in HubApi class (self.endpoint) re_raise(`bool`): raise exception when error token (`str`, *optional*): access token to use for checking existence. Returns: True if the repository exists, False otherwise. NzNot support repo-type: %srrz5Invalid repo_id: %s, must be of format namespace/nameFr/api/v1/datasets/rrTiz%Check repo_exists return status code rzOFailed to check existence of repo: %s, make sure you have access authorization.)rrrkr2countrrcrirrrrrMr&rr ) rr+r*rr,rrr r rrcoderrrr1s6     zHubApi.repo_existscCsttjdtdd|s |j}|tkr|j|||dn|tkr'|j|||dntd|dt d|d d S) al @deprecated Delete a repository from ModelScope. Args: repo_id (`str`): A namespace (user or an organization) and a repo name separated by a `/`. repo_type (`str`): The type of the repository. Supported types are `model` and `dataset`. endpoint(`str`): The endpoint to use. If not provided, the default endpoint is `https://www.modelscope.cn` Could be set to `https://ai.modelscope.ai` for international version. token (str): Access token of the ModelScope. rrr)rArr)rrrrEz not supported.r-z deleted successfully.N) rr r!rridelete_datasetrjr#r2r&info)rr+r*rrrrr delete_repos*zHubApi.delete_repocCs`tj|tj}tjtjtj t j i}t |d}t ||WddS1s)wYdS)Nw)rrjoinrr CONFIGURATIONrl frameworkrqZtorchtaskrsotheropenrdump) model_dircfg_filecfgfilerrr_create_default_configMs "zHubApi._create_default_configz upload modelrVcommit_messagerignore_file_pattern lfs_suffixc  s^tjdtdd |durtd|durtdtj|r%tj|r)tdtj|t j } tj| sIt dt j d|d t ||j| d d }|durXtd t|}t|}| durgg} t| tro| g} |dusw|dur{td |j|| dst d|d|j||||| | |jdtj|t}t}t d|d|dt d|dzzt||| |jd}||}||vrt d|||||||t|}|D]$ddkr tj|}tj|rt|qt j!|d dq|D]9ddkrEt"fdd| Dr$q tj|}tj#|r?t $|tj|q t %||q |sXt&j&'(d}d||f}| durst| trf| gn| }|D]}|)|qj|j*|||d|dur|+||t d |d!|d"Wn t,ywWt j!|d ddSt j!|d dw)#NzThis function is deprecated and will be removed in future versions. Please use git command directly or use HubApi().upload_folder insteadrrzmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.zNo z file found in z, creating a default one.TrzMust login before upload!z5Visibility and License cannot be empty for new model.rzCreating new model [])rrrrrrrzPushing folder z as model rzTotal folder size z;, this may take a while depending on actual pushing size...)rVZ clone_from auth_tokenrzCreating new branch r) ignore_errorscsg|] }t|duqSN)research).0patternfrr z%HubApi.push_model..z%Y_%m_%d_%H_%M_%Sz$[automsg] push model %s to hub at %s)r[Z local_branchZ remote_branchzSuccessfully push folder z to remote repo [z].)-rr r!rGrrexistsisfilerOrrrPr&r4rrZrrIlistdirr` isinstancestrr1rLrrr+rQrUZget_remote_branchesZ new_branchcheckoutremoveshutilrmtreeanyisdircopytreecopydatetimenowstrftimeZ add_lfs_typepushZ tag_and_pushr2)rrrVrrrr[rrrr\r]rrWrZ files_to_saveZ folder_sizeZtmp_dirZ git_wrapperrepobranchesZ files_in_reposrcdateZlfs_suffix_listsuffixrrgr push_modelWs /             "zHubApi.push_modelrrr  page_number page_sizec Cs|j|dd}|s |j}|d}|jj|d|||f|||jd}t|t|||jt j krGt | r?| t } | St| tt|dS)aHList models in owner or group. Args: owner_or_group(str): owner or group. page_number(int, optional): The page number, default: 1 page_size(int, optional): The page size, default: 10 endpoint: the endpoint to use, default to None to use endpoint specified in the class token (str, optional): access token for authentication Raises: RequestError: The request error. Returns: dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group} Frrz.{"Path":"%s", "PageNumber":%s, "PageSize": %s})datarrN)rrrputrrrMr&r'rr(rNrrrJrrO) rr rrrrrrrrrrr list_modelss(     zHubApi.list_models)rrsortrdrrrrdc Cs|s|j}|d}||d} |r%|tvr!td|dtt|| d<|r+|| d<|r1|| d<|j|dd } |jj|| | d } t| || S) aqList datasets via OpenAPI with pagination, filtering and sorting. Args: owner_or_group (str): Search by dataset authors (including organizations and individuals). page_number (int, optional): The page number. Defaults to 1. page_size (int, optional): The page size. Defaults to 10. sort (str, optional): Sort key. If not provided, the server's default sorting is used. choose from ['default', 'downloads', 'likes', 'last_modified']. search (str, optional): Search by substring keywords in the dataset's Chinese name, English name, and authors (including organizations and individuals). endpoint (str, optional): Hub endpoint to use. When None, use the endpoint specified in the class. token (str, optional): Access token for authentication. Returns: dict: The OpenAPI data payload, e.g. { "datasets": [...], "total_count": int, "page_number": int, "page_size": int } z/openapi/v1/datasets)rrzInvalid sort key: z. Supported sort keys: rrdr;F)rtoken_required)paramsr) rrArGlist_build_bearer_headersrrrO_parse_openapi_response) rr rrrrdrrrrrrrrr list_datasetss(  zHubApi.list_datasets use_cookiescCs0d}t|tr |}|St|tr|j|d}|S)N)r)rnrboolr)rrrrrr _check_cookieLs  zHubApi._check_cookiecutoff_timestampcCs.|j|||d}|rdd|D}|Sg}|S)a3Get model branch and tags. Args: model_id (str): The model id cutoff_timestamp (int): Tags created before the cutoff will be included. The timestamp is represented by the seconds elapsed from the epoch time. use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. Returns: Tuple[List[str], List[str]]: Return list of branch name and tags )rrrcSg|]}|dqSRevisionrrexrrrrihz/HubApi.list_model_revisions..)list_model_revisions_detail)rrrrZ tags_detailstagsrrrlist_model_revisionsTszHubApi.list_model_revisionsc Cs||}|dur t}|s|j}|d|d|}|jj||||jd}t|t||| }t ||t } | ddS)aGet model branch and tags. Args: model_id (str): The model id cutoff_timestamp (int): Tags created before the cutoff will be included. The timestamp is represented by the seconds elapsed from the epoch time. use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. endpoint: the endpoint to use, default to None to use endpoint specified in the class Returns: Tuple[List[str], List[str]]: Return list of branch name and tags Nrz/revisions?EndTime=%sr RevisionMapTags) rrarrrrrrMr&rrPr) rrrrrrrrrrLrrrrls    z"HubApi.list_model_revisions_detailcCs"|D] }|d|kr|SqdS)Nrr)rdetailsr itemrrrget_branch_tag_details  zHubApi.get_branch_tag_detailrc sD|s|j}ttttj}|j||durdn||d\}}|r,dd|Dng}|r7dd|Dng} |tkrx|durNt }t d|d||vr^|| vr^t d||f| ||} | durn| ||} t d || S|dur||vr| ||} | St|d kr|dus|t krt }nt d ||f| ||} | S|dur|rfd d|Dng} t| d kr| d d }| d } nt }| ||} dd| } t d| t d|| S|| vr|t krt d| ||} ndd| } t d||| f| ||} t d|| S)NF)rrcSrrrrrrrrirz4HubApi.get_valid_revision_detail..cSrrrrrrrrirz-Model revision not specified, using default [z ] version.z$The model: %s has no revision : %s .zDevelopment mode use revision: rz#The model: %s has no revision: %s !csg|] }|dkr|qS)Z CreatedAtrrZrelease_timestamprrris rz[%s],z3Model revision should be specified from revisions: z,Model revision not specified, use revision: z?Using the master branch is fragile, please use it with caution!z0The model: %s has no revision: %s valid are: %s!z#Use user-specified model revision: )rraintroundrxry timestamp#get_model_branches_and_tags_detailsr)rgr&rLrHrdebugrrOr4) rrrrrZcurrent_timestampZall_branches_detailZall_tags_detailZ all_branchesZall_tagsZrevision_detailZrevisions_detailZvlrrrget_valid_revision_detailst     '           z HubApi.get_valid_revision_detailcCs|j||||ddS)N)rrrrr)r)rrrrrrrrget_valid_revisionszHubApi.get_valid_revisionc Cs|||}|s |j}|d|d}|jj||||jd}t|t|||}t ||t }|dd|ddfS)aGet model branch and tags. Args: model_id (str): The model id use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. endpoint: the endpoint to use, default to None to use endpoint specified in the class Returns: Tuple[List[str], List[str]]: Return list of branch name and tags rz /revisionsrrZBranchesr) rrrrrrrMr&rrPr) rrrrrrrrrLrrrrs   z*HubApi.get_model_branches_and_tags_detailscCsF|j||d\}}|rdd|Dng}|rdd|Dng}||fS)aqGet model branch and tags. Args: model_id (str): The model id use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. Returns: Tuple[List[str], List[str]]: Return list of branch name and tags )rrcSrrrrrrrrirz6HubApi.get_model_branches_and_tags..cSrrrrrrrrir)r)rrrZbranches_detailZ tags_detailr}rrrrget_model_branches_and_tagss z"HubApi.get_model_branches_and_tagsrootr8rcCs|s|j}|rd||||f}nd|||f}||} |dur'|d|}|dur.|jn|}ttj|d<|jj|| |d} t | t | || } t | g} | t dsgt d|d |d | S| t dD]} | d d ks{| d d kr|qm| | qm| S)aList the models files. Args: model_id (str): The model id revision (Optional[str], optional): The branch or tag name. root (Optional[str], optional): The root path. Defaults to None. recursive (Optional[bool], optional): Is recursive list files. Defaults to False. use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, will load cookie from local. Defaults to False. headers: request headers endpoint: the endpoint to use, default to None to use endpoint specified in the class Returns: List[dict]: Model file list. z7%s/api/v1/models/%s/repo/files?Revision=%s&Recursive=%sz+%s/api/v1/models/%s/repo/files?Recursive=%sNz&Root=z X-Request-IDrFileszNo files found in model z at revision rrz .gitignorez.gitattributes)rrrrouuiduuid4hexrrrMr&rrPrr4append)rrrrr8rrrrrrrfilesrYrrrr>s:   zHubApi.get_model_files)rrfilenamecCs@|j|d}|j|d||durdn|d}dd|D}||vS)aGet if the specified file exists Args: repo_id (`str`): The repo id to use filename (`str`): The queried filename, if the file exists in a sub folder, please pass / revision (`Optional[str]`): The repo revision token (`Optional[str]`): The access token Returns: The query result in bool value rTNF)r8rrcSrrr)rerYrrrrifrz&HubApi.file_exists..)rr>)rr+rrrrrrrr file_existsLs zHubApi.file_exists dataset_name namespacerc Cs|dus|dur td|j|dd} |s|j}|d} d|fd|fd|fd|fd|fd|fd} |jj| | | ||jd} t| | | t| |d|d |} t d | | S) aN Create a dataset in ModelScope. Args: dataset_name (str): The name of the dataset. namespace (str): The namespace (user or organization) for the dataset. chinese_name (str, optional): The Chinese name of the dataset. Defaults to ''. license (str, optional): The license of the dataset. Defaults to Licenses.APACHE_V2. visibility (int, optional): The visibility of the dataset. Defaults to DatasetVisibility.PUBLIC. description (str, optional): The description of the dataset. Defaults to ''. endpoint (str, optional): The endpoint to use. If not provided, the default endpoint is used. token (str, optional): The access token for authentication. Returns: str: The URL of the created dataset repository. Nz(dataset_name and namespace are required!Trz/api/v1/datasets)rrZOwnerrrE Description)rrrz /datasets/rzCreate dataset success: ) rGrrrrrrrLrPrr&rL)rrrrrrrrrrrrrZdataset_repo_urlrrrcreate_datasetis0     zHubApi.create_datasetrAcCr) ac @deprecated Delete a dataset from ModelScope. Args: dataset_id (str): The dataset id to delete. endpoint (str, optional): The endpoint to use. If not provided, the default endpoint is used. token (str, optional): The access token for authentication. Returns: None rrrTrNrrGrr)rrArrrrrrrrrKs" zHubApi.delete_dataset_dataset_id_type_cachec Cs|s|j}|||f}tj|}|dur|S|d|d|}|j|d}|jj||d} | } t|| | | dd} | dd} | | ftj|<| | fS) z Get the dataset id and type. NrGrrrDataZIdType)rrrrrrrrK) rrrrr cache_keycached datahub_urlrrresprA dataset_typerrrget_dataset_id_and_types      zHubApi.get_dataset_id_and_typeTr root_pathc Cstdt|j|||| d\} } |rdnd}|s|j}|d| d} |r(|nd|r-|nd|||d } |j| d }|jj| | |d }|}t | |||S) z? @deprecated: Use `get_dataset_files` instead. zMThe function `list_repo_tree` is deprecated, use `get_dataset_files` instead.rrrrTrueFalserG /repo/treerrrZRootZ Recursive PageNumberPageSizerrr) rr r!rrrrrrrK)rrrrrr8rrrrdataset_hub_idrrrrrrrrrlist_repo_trees&      zHubApi.list_repo_tree2)r*rrrrrc Cst|r|ddkr|d\}} ntd|d|s |j}|r-|d|d|dn|d|d} |p8t||d } |j|d } z+|jj| | | | |j d } t | | }t ||d tjkrlt|WSWdStjjy}z td |dt|d}~ww)au Get the commit history for a repository. Args: repo_id (str): The repository id, in the format of `namespace/repo_name`. repo_type (Optional[str]): The type of the repository. Supported types are `model` and `dataset`. revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. page_number (int): The page number for pagination. Defaults to 1. page_size (int): The number of commits per page. Defaults to 50. endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class. token (Optional[str]): The access token. Returns: CommitHistoryResponse: The commit history response. Examples: >>> from modelscope.hub.api import HubApi >>> api = HubApi() >>> commit_history = api.list_repo_commits('meituan/Meeseeks') >>> print(f"Total commits: {commit_history.total_count}") >>> for commit in commit_history.commits: ... print(f"{commit.short_id}: {commit.title}") rrInvalid repo_id:  !z/api/v1/s/z/commitsr)rrrrrrrCodez%Failed to get repository commits for : N)rwrIr rrrfrrrrrrOrrPrr(r|Zfrom_api_responserequests exceptionsRequestExceptionr2ro)rr+r*rrrrr_owner _dataset_nameZ commits_urlrrrrerrrr=s6!    zHubApi.list_repo_commitsr)rrr8rrrrrrcCs| dur*t|r|ddkr|d\} } ntd|d|j| | ||d\} } |s/|j}|d| d} |||r=d nd ||d }|j|d }|jj| ||d }| }t | |||d}|durggS|dpmgS)a Get the dataset files. Args: repo_id (str): The repository id, in the format of `namespace/dataset_name`. revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. root_path (str): The root path to list. Defaults to '/'. recursive (bool): Whether to list recursively. Defaults to True. page_number (int): The page number for pagination. Defaults to 1. page_size (int): The number of items per page. Defaults to 100. endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class. token (Optional[str]): The access token. dataset_hub_id (Optional[str]): Pre-fetched dataset hub id. When provided, skips the internal ``get_dataset_id_and_type`` lookup. Useful in pagination loops to avoid redundant API calls per page. Returns: List: The response containing the dataset repository tree information. e.g. [{'CommitId': None, 'CommitMessage': '...', 'Size': 0, 'Type': 'tree'}, ...] NrrrrrrGrrrrrrrr) rwrIr rrrrrrrrK)rr+rrr8rrrrrrrr?rrrrrrrrrrC=s0      zHubApi.get_dataset_filesc Cs||j|d}|s |j}|r|d|d|}n|d|}|jj||||jd}t||}t||||t S)a Get the dataset information. Args: dataset_id (str): The dataset id. revision (Optional[str]): The revision of the dataset. endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class. token (Optional[str]): The access token. Returns: dict: The dataset information. rrGr%r) rrrrrrrOrrKr) rrArrrrrrrrrrrB{s  zHubApi.get_datasetc Cs|s|j}|d|d|}|j|d}|jj||||jd} | } t|| | | d} | durBtd|d|d |d | d } | S) z( Get the meta file-list of the dataset. rGz/repo/tree?Revision=rrrNz'The modelscope dataset [dataset_name = z, namespace = z , version = z] dose not existr) rrrrrrrrKrH) rrrrArrrrrrr file_listrrrget_dataset_meta_file_lists&   z!HubApi.get_dataset_meta_file_listrmeta_cache_dircCsXtj|t|tjj}t|d}|dWddS1s%wYdS)z Dump the data_type as a local file, in order to get the dataset formation without calling the datahub. More details, please refer to the class `modelscope.utils.constant.DatasetFormations`. rNz3*** Automatically-generated file, do not modify ***N) rrrOrormZformation_mark_extvaluerTwrite)rrZdataset_type_file_pathfprrrdump_datatype_files   "zHubApi.dump_datatype_filerc  Cs,tt} t|} t| } |j|d} tj||d|s|j}|D]q} | d}tj |d}|| vr|d|d|d|d| }|j j || d }t |tj ||}tj |rotd |d |d | ||q t|d }||jWdn1swY| ||q | | fS)Nr)rrrrrGr/repo?Revision= &FilePath=rzReusing dataset z's python file ()wb)rrrmrnrrrrrrsplitextrrrOrOrkr&r4rrTrcontent)rrrrrrrrrZ local_pathsZdataset_formationZdataset_meta_formatr file_info file_path extensionrrZ local_pathrhrrr"get_dataset_meta_files_local_pathss>   z)HubApi.get_dataset_meta_files_local_pathscsddl}ddlm}ddl}tj|||jdd }|t j kr.tj |r.t |tj |r>td||Stj|d}tdtj||d d } t| jd d} || d d } fd d} d} t|dQ}| | D]D}| t||drdd|D}t|dkrqs| dkrd }nd}||}|j|d|dd| d7} qs|D] }||dqqsWdn1swY| |S)zO Fetch the meta-data files from the url, e.g. csv/jsonl files. rNrzUTF-8encodingzReusing cached meta-data file: rzLoading meta-data file ...T)rstreamzcontent-length)rZ dynamic_ncolsc3sHg}|D]}|d}||t|kr|Vg}q|VdS)Nutf-8) iter_linesdecoderr)rZ chunk_datarrrr get_chunks     z3HubApi.fetch_meta_files_from_url..get_chunkaZjsonlcSsg|] }|rt|qSr)r0rloads)relinerrrri sz4HubApi.fetch_meta_files_from_url..F\)indexheader escapecharr )hashlib tqdm.autorZpandasrrrOmd5encode hexdigestrpZFORCE_REDOWNLOADrkrqr&rLrrrrrrrTrrr Z DataFrameZto_csvrclose)urlZout_pathrmoderrrpdrresponse total_sizeprogressrZiter_numrhrZ with_headerZchunk_dfrrrrfetch_meta_files_from_urlsH             z HubApi.fetch_meta_files_from_url file_nameviewextension_filterc CsR|r|r|s tdd|||d}t|}|s|j}|d|d|d|} | S)Nz:Args (file_name, dataset_name, namespace) cannot be empty!ZSDK)SourcerFilePathZViewrGr/repo?)rrr) rrrrrr r rrfile_urlrrrget_dataset_file_urls zHubApi.get_dataset_file_urlc CsF|s|j}|r!tj|dtvr!|d|d|d|d| }|S)NrrGrrr)rrrrrh)rrrrrrrrrget_dataset_file_url_origin>sz"HubApi.get_dataset_file_url_origincCs2|s|j}|d|d|d|}|j||dS)NrGr/ststoken?Revision=r^)rdatahub_remote_call)rrrrrrrrrrget_dataset_access_configLs z HubApi.get_dataset_access_config check_cookiec Cst|s|j}|d|d|d|}|r|jdd}n|j|d}|jj||||jd} | } t| | dS) NrGrrT)rrrrrr) rrrrrrrrrP) rrrrrrrrrrrrrr!get_dataset_access_config_sessionYs    z(HubApi.get_dataset_access_config_sessionversionc Cstjtjd}|stdtj|d}tj|j |d}t ||d}t |d}|j j |||| |jdd} | } | d d krLtd | | d S) z. Get virgo dataset meta info. rz"Virgo endpoint is not set in env: z/data/set/downloadr)Z dataSetIdZdataSetVersion)ri)rrrrrrJrzFailed to get virgo dataset: r)rrrrtZenv_virgo_endpoint RuntimeErrorrutilsdict_from_cookiejarrdictrrrrr) rrArrZvirgo_endpointZvirgo_dataset_urlrrDrrrrrrget_virgo_metass,   zHubApi.get_virgo_meta zip_file_namecCs|s|j}|d|d|}|j|d}|jj||||jd} | } t| | d} t| d} |d|} |jj| |||jd}|}t||d}| dd|d |d |}||d <|S) NrGrrrrrErz -unzippedr?ZDir) rrrrrrrrPrF)rrrrrrrrrrrrrZdatahub_sts_urlZr_stsZresp_stsZdata_stsZfile_dirrrr&get_dataset_access_config_for_unzippeds*      z-HubApi.get_dataset_access_config_for_unzippedc Csp|s|j}|d|d|d|d|d|d| } |j|d} |jj| | dd } | } t| | d } | S) NrGrz/oss/tree/?MaxLimit= &Revision=z &Recursive=z &FilterDir=ri)rrrr)rrrrrrP) rrrZ max_limitZ is_recursiveZ is_filter_dirrrrrrrrrrlist_oss_dataset_objectss" zHubApi.list_oss_dataset_objects object_namec C||r|r|r|s td|s|j}|d|d|d|d| }|j|dd}|jj||d} | } t| | d } | S) NArgs cannot be empty!rGrz /oss?Path=rTrrrMessagerrrrr"rrP rr!rrrrrrrrrrrdelete_oss_dataset_objects z HubApi.delete_oss_dataset_objectc Cr") Nr#rGrz/oss/prefix?Prefix=z /&Revision=Trr$r%r&r'rrrdelete_oss_dataset_dirszHubApi.delete_oss_dataset_dircCsB|j|d}|jj||dtid}|}t||||dS)Nrrrr)rrrrrrrK)rrrrrrrrrrs   zHubApi.datahub_remote_call use_streamingc Cs2tddk}|s |j}|r|r|s|szd|j|d}|d|d|d}|jj||||jd} t| t j j } d} t tj vrGtj t } ttj vrQtj t} |d|d|d | d | } |jj| |||jd} | } t| WdSty}z t|WYd}~dSd}~wwdSdSdSdS) NZCI_TESTrrrGrz/download/increaserrz /download/uv/z?user=)rgetenvrrrrrrrOroZLOCALrr"rr#rrPr2r&r3)rrrr*rrZ is_ci_testrZdownload_count_urlZdownload_count_respZchannel user_nameZdownload_uv_urlZdownload_uv_resprrrrdataset_download_statisticss@         z"HubApi.dataset_download_statisticscCstttji|Srb)r'rorrr)rrrrrr szHubApi.builder_headersrcCsv||j}|p|jptjd}|s(|}|r(|D] }|jdkr'|j}nq|r3d||d<|S|r9t d|S)a Build HTTP headers with optional Bearer token for OpenAPI endpoints. Token resolution order: 1. Explicit token param 2. self.token (set at construction) 3. MODELSCOPE_API_TOKEN env var 4. Locally cached cookies (m_session_id from login()) Args: token: Optional access token for one-time authentication. token_required: If True, raise ValueError when no token is available. Returns: Headers dict with user-agent, request-id, and optionally Authorization. Raises: ValueError: If token_required is True but no token is available. rrzBearer AuthorizationzAuthentication required but no token found. You can pass the `token` argument, or set MODELSCOPE_API_TOKEN environment variable, or use HubApi(token=`your_sdk_token`). Your token is available at https://modelscope.cn/my/myaccesstoken) rrrrrrrr rr)rrrrZresolved_tokenrcookierrrr s(   zHubApi._build_bearer_headersrzrequests.Responsec Csz|}Wn tjjtfy&}ztd|td||d}~wwt|t rJd|vrJ| ddur?d|vr?|dS| dpEd}t|t|t rU| diSiS) a' Parse OpenAPI response with unified JSON parsing and data extraction. Handles the standard OpenAPI response envelope: {"success": bool, "data": {...}, "message": str} Also handles the simpler envelope where only "data" is present. Args: response: requests Response object (HTTP status already validated). Returns: Parsed 'data' dict from the response envelope. Raises: RequestError: If JSON parsing fails or business-level error is returned. zJSON parsing failed: zInvalid JSON response: NsuccessTrmessagezOpenAPI request failed) rrrJSONDecodeErrorrr&r3rJrnrr)rrrmsgrrrr>s zHubApi._parse_openapi_responsecCs.|d\}}|s |j}|d|d|dS)NrrGr )r r)rr+rZ _namespacerrrrget_file_base_pathaszHubApi.get_file_base_path) rrr*rrrexist_okcreate_default_configrr5r6c  Ks|std|s |j}|j||||d} | r7|r/|d|d|} td|d| | Std|d|d}t|dkrFtd |\}}|tkrd d tj D}| | }|d urktd |d|j ||||| ||d} | rt7}ddlm}|||||d}dddd}| d}|si}i||}t|dt|gddWd n1swYtd| ddd| S|tkrdd tj D}| | }|d urtd |d|j|||||||d} td| ddd| Std |d!t)"a% Create a repository on the ModelScope Hub. Args: repo_id (str): The repo id in the format of `owner_name/repo_name`. token (Union[str, bool, None]): The access token. visibility (Optional[str]): The visibility of the repo, could be `public`, `private`, `internal`, default to `public`. repo_type (Optional[str]): The repo type, default to `model`. chinese_name (Optional[str]): The Chinese name of the repo. license (Optional[str]): The license of the repo, default to `apache-2.0`. endpoint (Optional[str]): The endpoint to use. In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai' exist_ok (Optional[bool]): If the repo exists, whether to return the repo url directly. create_default_config (Optional[bool]): If True, create a default configuration file in the model repo. **kwargs: The additional arguments. Returns: str: The repo url. Repo id cannot be empty!r.rrr-z already exists, got repo url: z already exists!rzBInvalid repo id, should be in the format of `owner_name/repo_name`cS i|] \}}|ds||qS__ startswithrekvrrr  z&HubApi.create_repo..NInvalid visibility: z9, supported visibilities: `public`, `private`, `internal`)rrrrrrrrrT)r`rZpytorchztext-generationT)rQrRZ allow_remote config_jsonzconfiguration.json)Zignore_push_errorz"New model created successfully at rflushcSr8r9r;r=rrrr@rA)rrrrrrrz$New dataset created successfully at Invalid repo type: , supported repos: )rrr1r&r4r rrjrD__dict__itemsrupperrtempfileTemporaryDirectorymodelscope.hub.repositoryrUr]rdumpsprintrirBrrk)rr+rrr*rrrr5r6rkwargsr1repo_urlZ repo_id_listr repo_nameZ visibilitiesZtemp_cache_dirrUr|Zdefault_configrCconfigrrr create_repohs$         zHubApi.create_repo)commit_descriptionrr*rr operationsrUcCs.|std|s |j}|tvrtd|dt|d|d|d|} |p,d|}|p0d}|j|d d } |j||d } |jj| ||jt | | d } | j d kryz| } Wn t j yh| j } Ynwd| j d| d| }t|| }|didd}td| t| |||dS)a Create a commit on the ModelScope Hub. Args: repo_id (str): The repo id in the format of `owner_name/repo_name`. operations (Iterable[CommitOperation]): The commit operations. commit_message (str): The commit message. commit_description (Optional[str]): The commit description. token (str): The access token. If None, will use the cookies from the local cache. See `https://modelscope.cn/my/myaccesstoken` to get your token. repo_type (Optional[str]): The repo type, should be `model` or `dataset`. Defaults to `model`. revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. endpoint (Optional[str]): The endpoint to use. In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai' timeout (int): Timeout for each request in seconds (default: 180). Returns: CommitInfo: The commit info. Raises: ValueError: If the request fails with a 4xx client error. requests.exceptions.RequestException: If a network-level error occurs. r7rFrG/api/v1/repos/rz/commit/z Commit to rTr)rVr[rrrrHzHTTP z error from rroidzCommit succeeded: )Z commit_urlr[rUrY)rrrkr_prepare_commit_payloadrrrrrrNr'r2textrr&rLr})rr+rVr[rUrr*rrrrpayloadrZ error_detail error_msgrrYrrr create_commitsJ#    zHubApi.create_commitz [Uploading])rr*r[rUbuffer_size_mb tqdm_desc disable_tqdmrpath_or_fileobj path_in_repor`rarbc  Cs|tvrtd|dt|std|j|ddt|ttfr5tjtj |}|p3tj |}n|s;tdt|t j rE| }|j||jj|g|d|durZ|nd |d }|d krhtd t||d } | d} | d}|j||||jddd|j|||| || | |d}t||| d}|j||rdnd|_|d|_|g}td|ddd|j||||||| d}|S)a Upload a file to the ModelScope Hub. Args: path_or_fileobj (Union[str, Path, bytes, BinaryIO]): The local file path or file-like object (BinaryIO) or bytes to upload. path_in_repo (str): The path in the repo to upload to. repo_id (str): The repo id in the format of `owner_name/repo_name`. token (Union[str, None]): The access token. If None, will use the cookies from the local cache. See `https://modelscope.cn/my/myaccesstoken` to get your token. repo_type (Optional[str]): The repo type, default to `model`. commit_message (Optional[str]): The commit message. commit_description (Optional[str]): The commit description. buffer_size_mb (Optional[int]): The buffer size in MB for reading the file. Default to 1MB. tqdm_desc (Optional[str]): The description for the tqdm progress bar. Default to '[Uploading]'. disable_tqdm (Optional[bool]): Whether to disable the tqdm progress bar. Default to False. revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`. Returns: CommitInfo: The commit info. Examples: >>> from modelscope.hub.api import HubApi >>> api = HubApi() >>> commit_info = api.upload_file( ... path_or_fileobj='/path/to/your/file.txt', ... path_in_repo='optional/path/in/repo/file.txt', ... repo_id='your-namespace/your-repo-name', ... commit_message='Upload file.txt to ModelScope hub' ... ) >>> print(commit_info) rFrGz$Path or file object cannot be empty!Trz#Arg `path_in_repo` cannot be empty!file_path_listr*NzUpload z to ModelScope hubrz4Buffer size: `buffer_size_mb` must be greater than 0)file_path_or_objr` file_size file_hashFr+rr*rr5r6)r+r*sha256rrrbrarrdrcfile_hash_infolfsnormal is_uploadedzCommitting file to  ...rDr+rVr[rUrr*r)rkrrrnrorrrabspathrbasenameioBufferedIOBaserr check_filecheck_normal_filesrurTr _upload_blobris_lfs _upload_mode _is_uploadedrOr^)rrcrdr+rr*r[rUr`rarbr hash_info_drhri upload_resZ add_operationrV commit_inforrr upload_file( s~/     zHubApi.upload_filecCs0|D]}||d|d|dq|dS)z1Mark files as uploaded and persist tracker state.file_path_in_repo file_mtimefile_size_on_diskN)Z mark_uploadedsave)rtrackerresultsrrrr_track_uploaded_batch s   zHubApi._track_uploaded_batchcCs |dd|D|dS)z2Mark files as committed and persist tracker state.cSs"g|] }|d|d|dfqS)rrrrrerrrrri s  z1HubApi._track_committed_batch..N)Zmark_committed_batchr)rrrrrr_track_committed_batch s  zHubApi._track_committed_batch)r pre_validatedrrcCs|durt}d}d} t|ttjf} | r;zt|} ||| j| j} | dur.| }||d<Wn t y:d} Ynw|duret |d}| rez| durPt|} | || j| j|Wn t ydYnw| dur|| r|zt|} Wn t y{Ynw|d} |d} d}t t D]}z8t|ttjfrtjt|}|| krtd| d|d||j||| | || tkd |d ||d }Wnmttjjtjjtfy#}zJt|tjjrt|d r|jdur|jjd kr|}|t dkrtt|t}td|ddt d|d|d|d t |WYd}~qd}~wwt!dt d|d||||| r:| jnd| rA| jn|"dd|d|"dd|dS)z5Hash and upload a single file, returning result dict.Nrg)rgrhriz.File size changed since hash computation: was z, now z. File may have been modified: z [Uploading r_) r+r*rkrrrbrarrrrrzBlob upload attempt rz failed for r, retrying in s ...zBlob upload failed after z attempts for rrp is_reusedF)rrrrrprrm)#rYrnrorPathLikestatget_hashst_mtimest_sizeOSErrorruZput_hashranger-rgetsizerryr1ConnectionErrorrrrhasattrrr'rr.r/r&r4timesleeprr)rrrr+r*rrrr}Z file_statZ is_real_pathrrhri last_errorattemptZ current_sizer~rwaitrrr_upload_single_file s               zHubApi._upload_single_file)rUrr*rrrc sd} t|D]} z|j|||||||dWSttjjfy/} z| } WYd} ~ nsd} ~ wttjjfy]} zt| drQ| jdurQd| jjkrPdkrQ| } WYd} ~ nEd} ~ wt y} z"t | t dr~ddg} t fd d | Ds~| } WYd} ~ nd} ~ wty} z| } WYd} ~ nd} ~ wwtd | d } td | dd|d| d| d t| qtd|d| | )aCommit with application-level exponential backoff retry. Retries on transient errors (5xx, ConnectionError) and specific retryable 4xx errors (e.g. git ref conflicts). Raises immediately on non-retryable client errors (4xx). Nrrrirz HTTP 4\d{2}zCould not update refsz try againc3|]}|vVqdSrbrrepZ error_strrr O z,HubApi._commit_with_retry..r<zCommit attempt rrz failed: rrzCommit failed after z attempts: )rr^rrrrrrr'rrorcrdrtr2rr&r4rrr)rr+rVr[rUrr*rrrrrZretryable_patternsrrrr_commit_with_retry" s`     zHubApi._commit_with_retryrcCs\g}|D]'}t|d|d|dd}|j|d|rdnd|_|d|_||q|S)z2Build CommitOperationAdd list from upload results.rrrmrlrnrorp)rrrzr{r|r)rrr*rVZitem_doptrrr_build_batch_operations_ s  zHubApi._build_batch_operations) rdr[rUrr*allow_patternsignore_patterns max_workers use_cacher folder_pathrrrrc Ds t} s td|durtdtvr tddtjddr+nd| r1| nd} | dur:g} nt| trB| g} | t7} durSfdd | D} |durY|nd d }|pbd }ttg}| durn|} nt| try| g|} nt | |} t d j ||| d}t |dkrtd|dt dt |djjdd |Ddjjdddt|ddd}trtt |}t d|dt |dn tdkrtnt |}t|ttfrt|nt|dj}| r |t}t|dnttt ||g}tt|D]V\}\}}t|ttjfrkzt |}!||j"|j#rK$|%|WqWnt&yj}zt 'd|d |WYd}~nd}~ww|(|||ffqi}i}g}|D]B\}\}}t|ttjfrzt |})||j"|j#}|dur||f||<Wq~Wn t&yYnw|(|||ffq~|rd!d |*D}j+|d"}|}t,d#d$|*D}t d%t |d&|d't ||d(t } | dkr t | d)t d*t |d+| d,t |d-t d.t |d/j-d0|d1 dld2t.d3t/ffd4d5 }!g}"g}#g}$j-}%zbt0| d6M}&|D](\}}'d}(||vrs||dd7})|1|)}(|(dursd}(|&2|!||'|(qTt3t4|%d8|%d9D]}*|*|}+t5|+|t |},t6fd:d$t4|+|,Drt d;|*d<d=|%d>q7|*\}-}.|.r|$8|.|.D]\}/}0t 9d?|/dd@|0qƈ:|-;|-}1|1st 9d;|*d<d=|%dAq|dB|*d<d=|%dC}2z2j<|1|2|| dD}3|"(|3|#8|-t d;|*d<d=|%dEt |-dF=|-Wqt>y}zRt 9d;|*d<d=|%dG||-D]}4j?|4dH|4dI|4dJdKdLqR|-D]}4|$(|4dH|4dMf|fqgt 'd;|*d<d=|%dNt |-dOWYd}~qd}~wwWdn 1swYW@n@w|$rtArԈjB|$||| | dP \}$}5}6|"8|5|#8|6n|$rt4tCD]}7|$snt dQ|7d<d=tCdRt |$dSg}8g}9|$D]A\\}}}:zjD||dT};|9(|;Wqt>y<}zt 9dU|d@||8(||f|fWYd}~qd}~ww|9r:|9;|9}1|1rz6j<|1|dV|7d<dC|| dD}3|"(|3|#8|9=|9t dW|7d<dEt |9dFWn5t>y}z(t 9dW|7d<dG||9D]};|8(|;dH|;1dMdXf|fqWYd}~nd}~ww|8}$qۈ@t| }t,dYd$|#D}?t,dZd$|#D}@tEd[tEd\tEd]tEd^|=tEd_| tEd`|?tEda|@tEdb|>|?|@}AtEdc|AtEdd|}CtHdh|>di|Cdj|"sj| t |krht dkdSdSt |"d.z Upload to z on ModelScope hubzUploading fileszPreparing files to upload ...)folder_path_or_filesrdrrrz"No files to upload in the folder: rz Checking z files to upload ...cSsg|]\}}|qSrr)rer?rrrrri rreFrjcSs|dSrr)rrrr sz&HubApi.upload_folder..)keyzAdaptive batch size: z (for z files))r+zCannot stat file z, will re-upload: cSs"g|] \}}|d|ddqS)rirhrYrr)rerLr?rrrri" sr+r*objectsrcss|] }|durdVqdS)Nrr)rer?rrrr* sz'HubApi.upload_folder..zPre-validated z cached hash(es): z globally existing, z need upload.z% file(s) already committed, skipping.zScan complete: z total, z committed (skip), z to process.z Uploading z file(s) in z batch(es) of size z (pipeline mode).file_idxrc s|\}}z&td|dj|||d}td|||WdStyO}ztd|d||||WYd}~dSd}~ww)Nz Uploading: rq)r+r*rrrz Uploaded: zUpload failed:  - )r&rrZrecord_successr2r3Zrecord_failure)rrrrdrresultr) batch_trackerr+r*rrrrr_upload_worker> s z,HubApi.upload_folder.._upload_workerrriz[Committing batches])descrc3rrbr)rei)skipped_indicesrrre rzBatch rrz fully committed, skipping.z Failed: rz$: all files failed, skipping commit.z (batch rrr : committed file(s). commit failed: rrrZ commit_failedZ error_typerrz+ uploaded file(s) recovered to retry queue.) failed_filesrr+r*rr[rUrrz Retry round z: re-uploading z failed file(s) ...r+r*rrz Retry failed: z (retry round z Retry round rcss|] }|drdVqdSrrNrrrrrr   css|] }|dsdVqdSrrrrrrr rz<============================================================z Upload Reportz<------------------------------------------------------------z Total files : z Skipped (cached) : z Existed (server) : z Uploaded (PUT) : z Failed : z Committed : z Elapsed : z.1fr5z - zERROR - zL file(s) failed to upload. Please manually try again. Successfully uploaded z8 file(s) will be automatically skipped during the retry.z!All files were already committed.rb)IrrrkrrnrorzrVrXrr&rL_prepare_upload_folderrrrxrTrsortedr,rr2rresolveparentrZrYrWr enumeraterrrZ is_committedrraddZmark_file_skippedrr4rrvalues_validate_blobsum num_batchesrtuplerrsubmitrrrallZwait_for_batchextendr3rrrrr2 mark_failedrr9_retry_failed_files_reactr3rrOtyperr)Drr+rrdr[rUrr*rrrrr start_timeZ_internal_ignoreprepared_repo_objectsZ sorted_filesZcommit_batch_sizeZfolder_path_resolved cache_pathZfiles_to_uploadrrstrZpre_validated_mapZ hash_info_mapZfiles_need_hashrr validatedZreusedZ skipped_countr commit_infosZ all_resultsZtotal_failed_filesrexecutorrpv cached_hashZ batch_idx batch_startZ batch_endrZfailuresrerrrVZbatch_commit_messagerrZ react_commitsZ react_resultsZ retry_roundZretry_failuresZretry_successes_errrelapsedrZ failed_countZ reused_countZuploaded_countZcommitted_countr?Z succeededr)rrr+r*rrrrr upload_folderr s/                       "           H            & zHubApi.upload_folderc / Csg} g} i} g} t|}g}|D][}|\\}}}t|}|jr%||q| |zt|ttjfr8t|nd}Wn t yFd}Ynw|j ||rO|j nd|rU|j nd|j dtd|d|j d|dq|}dd td | d d dd ddd dtd ddd d td g}t|D]\}}|sn|d}td|dt|dg}g}|dr-t|d kr-ddlm}m}||ddZ}i}|D]\\}}} |j|j||||||d}!||f||!<q||D]/}!||!\}}z |!}"||"Wqty}#z|||f|#fWYd}#~#qd}#~#wwWdn 1s'wYnt|D]|\}$\\}}} |ddkro|$dkro|d krU|dd t|$tn|d}%t|%t}%td|%d|dt !|%z|j||||||d}"||"Wq1ty}#ztd|d|d |#|||f|#fWYd}#~#q1d}#~#ww| "|t|d!td t|}&t#dt||&D]{}'||'|'|&}(|$||(|%|(|})|)sqz.|j&||)|d|d||||d"}*| |*|'||(td|d#t|(d$WqtyB}#z%td|d%|#|(D]}+||+d&|+d'f|#fq&WYd}#~#qd}#~#wwg},|D]}|\\}}}| (|dd | |<| |d(kr| |zt|ttjfrtt|nd}Wn t yd}Ynw|j ||r|j nd|r|j ndd)dtd*|qGt|}|jr|,|qG| |zt|ttjfrt|nd}Wn t yd}Ynw|j ||r|j nd|r|j nd|j dtd|d|j dqGt|t|,}-|-dkrtd|d+|-d,t|,d-n |,r&t)d|d.|,}q| |}.|r;tdt|d/|.| | fS)0aReAct-style progressive retry for failed files. Implements Reason-Act-Observe loop with three escalating rounds: Round 1: Parallel retry with reduced concurrency (workers//2, batch=16) Round 2: Serial retry with exponential backoff (delay * 2^min(i, max_exp)) Round 3: Single-file commit with long delays (one file per commit) Files that exceed the per-file retry limit are classified as permanent failures and will not be retried further. Args: failed_files: List of ((path_in_repo, file_path), error) tuples. tracker: UploadTracker or NullTracker instance. repo_id: Repository identifier. repo_type: Repository type. token: Authentication token. commit_message: Base commit message. commit_description: Commit description. revision: Branch or tag name. max_workers: Max upload concurrency from caller. Returns: Tuple of (all_failures, commit_infos, all_successes) where: - all_failures: list of ((path_in_repo, file_path), error) for files that could not be resolved (permanent + exhausted retries). - commit_infos: list of CommitInfo for successful retry commits. - all_successes: list of upload result dicts for successfully retried files (to be merged into the upload report). NrrzeAct] Permanent failure: z (rrzRound 1 (parallel)Trrr_)r parallelworkers batch_sizedelayzRound 2 (serial+backoff)FzRound 3 (single-file)r z[ReAct] z : retrying z file(s) ...r)r as_completedrrrrz[ReAct] Waiting zs before retrying rqz : failed rrrrrrrrrZmax_retries_exceededz![ReAct] Max retries exceeded for u: made progress — z file(s) resolved, z remaining.z(: no progress, escalating to next round.z. file(s) still failing after all retry rounds.)*rr[Z is_retryablerrnrorrrrrrrrr&r3rr;r<rrLrconcurrent.futuresrrrrrr2rr8r:rrrrrrrrrr4)/rrrr+r*rr[rUrrrZ all_successes retry_countsZpermanent_failuresZ retryable remainingZitem_errrdrrcategoryrZ round_configsZ round_idxrXZ round_nameZround_successesZround_failuresrrrZ future_maprfuturerrrrrrbatchrVrrZ new_retryablerZ all_failuresrrrr s)                             z HubApi._retry_failed_files_react)rbrar`rrrkrrc  Cs|tdddddd} | dur$td|dddd| d<d| d<| St| tr,| } n-|j||||d g| d } | |} | durYtd|dddd| d<d| d<| S|j| dd }|rft|nd}|durptd |j d d|di| |j }|dd}t||d<t |dd||dy}t|tt frt|d}t||||}|jj| ||td}Wdn1swY|n?t|trtt||||}|jj| ||td}|n!t|tjrt||||}|jj| ||td}|ntdWdn 1swYt|d|}t|d| | d<d| d<|d| d<|d| d<| S)NF)rrprr' status_msgTzBlob rz already exists globally, reuse.rprrrrrCookiez m_session_id=rrzContent-LengthB)runitZ unit_scalerdisablerb)rrrzInvalid data type to uploadZrsprrr'r%r)rr&rLrnrorrrrrrrrrrTrrrr0rbytesruBytesIOrvrOrrP)rr+r*rkrrrbrar`rrZres_dZ upload_urlrrrrrrhrrrrrrry s           ,   zHubApi._upload_blob)rrrcCs|s|j}i}t}tdt||D]{}||||} |d|d|d} d| d} |j|dd} |jj| ||jt | | d } t | d | }t |d |d d }t }|D]}|d i}|d}|rz|d||d<||dq\| D]}|d|vrd||d<q}q|S)aValidate whether blobs need uploading. Queries the LFS batch API in chunks of UPLOAD_VALIDATE_BLOB_BATCH_SIZE. Args: repo_id: The repo id on ModelScope. repo_type: The repo type ('dataset', 'model', etc.). objects: Objects to check, each with 'oid' (sha256) and 'size'. endpoint: API endpoint override. token: Access token. Returns: Dict mapping oid -> upload_url (needs upload) or None (already exists). rrWrz/info/lfs/objects/batchZupload) operationrTrrXrrractionshrefrYN)rr@rrrrrrrrrNrOrPrrr)rr+r*rrrrrrrrr\rrrZ resp_objectsZ needs_uploadobjrZ upload_actionorrrrr sF        zHubApi._validate_blobrc s6dd}t|trtj|dr|}ntdtj|r"|g}n||durQ|jt sBtddfddt dDni|D]}tj|rk|j||tj|<qUttj||d}|r|d d nd fd d |D}td t|d|S)Nrz0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directorycs$i|]}|r||qSr)is_file relative_toas_posix)rer)rrrr@ s z1HubApi._prepare_upload_folder..z**/*)rrrrcs g|] }|t|fqSr)ro)rerelpath)prefixrelpath_to_abspathrrri sz1HubApi._prepare_upload_folder..z Prepared z files for upload.)rnrrrrlrr check_folderrrris_dirrglobrwrtrZfilter_repo_objectskeysr0r&rLr) rrrdrrZ files_pathrZfiltered_repo_objectsrr)rrrrr sD         zHubApi._prepare_upload_folderc Cs|gd}d}|D]j}t|tr"|jr"td|jd|d7}q t|trE|jdkrEd|jd|jjd| d d }|d  |q t|trf|jd krfd|jd |jj|jj ddd }|d  |q t d |dt|dd|dkrtd|d|S)zN Prepare the commit payload to be sent to the ModelScope hub. )r[rrzSkipping file 'z(' in commit (ignored by gitignore file).rrocreaterbase64)actionrrrrkrrrrnz(Unknown operation to commit. Operation: z. Upload mode: r{NzSkipped z/ file(s) in commit (ignored by gitignore file).)rnrZ_should_ignorer&rrdr{Z upload_inforZ b64contentrrrkrrrL)rVr[r\Znb_ignored_filesrZ commit_actionrrrrZ sJ     zHubApi._prepare_commit_payload皙?internal_timeoutcCs~dtdtfdd}|jd}|||d}d}|dur=|}d |vr*|d d }nd}|r=|||d }|dur=|j}|S) a Get the internal acceleration domain. Args: internal_timeout (float): The timeout for the request. Default to 0.2s Returns: str: The internal acceleration domain. e.g. `cn-hangzhou`, `cn-zhangjiakou` rrcSs:ztj||d}|W|Stjjyd}Y|Sw)Nr)rrraise_for_statusrr)rrrrrr send_request8s z>HubApi._get_internal_acceleration_domain..send_requestz&/api/v1/repos/internalAccelerationInfo)rrrNrZInternalRegionQueryAddressr)rofloatrrr[r0)rrrZ internal_urlZinternal_info_responseZ region_idZ query_addrZdomain_responserrr!_get_internal_acceleration_domain-s   z(HubApi._get_internal_acceleration_domain)rrrdelete_patternsc Cs|tvr td||stdt|tr|g}|j|dd}|s%|j}|dur-td||j}|tkrK|j ||p=t d||d} dd | D} nw|t krg} | d \} } |j | | ||d \} }d }d } z|j||potd||||| d}Wnty}ztd|dt|WYd}~n#d}~ww|D]}|ddkr| |dqt||krn|d 7}qhn td|dtg}| D]}|D]}t||r||nqqgg}}|D]}zb|tkr| d \}}|d|d |d}|pt |d}n&|t kr | d \}}|d|d |d}d|i}n td|dt|jj||||d}t||}t|||Wqtyk}z||td|dt|WYd}~qd}~ww||t|dS)am Delete files in batch using glob (wildcard) patterns, e.g. '*.py', 'data/*.csv', 'foo*', etc. Example: # Delete all Python and Markdown files in a model repo api.delete_files( repo_id='your_username/your_model', repo_type=REPO_TYPE_MODEL, delete_patterns=['*.py', '*.md'] ) # Delete all CSV files in the data/ directory of a dataset repo api.delete_files( repo_id='your_username/your_dataset', repo_type=REPO_TYPE_DATASET, delete_patterns='data/*.csv' ) Args: repo_id (str): 'owner/repo_name' or 'owner/dataset_name', e.g. 'Koko/my_model' repo_type (str): REPO_TYPE_MODEL or REPO_TYPE_DATASET delete_patterns (str or List[str]): List of glob patterns, e.g. '*.py', 'data/*.csv', 'foo*' revision (str, optional): Branch or tag name endpoint (str, optional): API endpoint token (str, optional): Access token Returns: dict: Deletion result zUnsupported repo_type: zdelete_patterns cannot be emptyTrNr)rr8rrcSrrr)rerhrrrrirz'HubApi.delete_files..rrrr)r+rr8rrrrrz Get dataset: z file list failed, message: rtreerrGrz/file)rr rGz/repor rzFailed to delete r) deleted_filesrr)rkrrnrorrrrrjr>rerir rrCrdr2r&r3rrfnmatchrr"rOrrP) rr+r*rrrrrrr file_pathsrrZ_hub_idr?rrZ dataset_filesrZ file_info_dZ to_deleterZdelete_patternrrownerrRrrrrrrrr delete_filesTs$              &zHubApi.delete_files)modelZdatasetprivatepubliccCs|std|dvrtd|dddtD}||d}|j|dd }|tkr|j||d }|jd |} |d } d } t| t rY| rY| d} t| t rY| rY| d} |dd |dd|d|dd|dd |dd |dd |dd |dd|didd |didd | |dd d} nA|t kr| d }t |dkst|std!|d"|j|d#|d|d$\}}|jd%|} |dd&} n td'|d(t|jj| | |||jd)}t||}t||S)*aK Set the visibility of a repo. Args: repo_id (str): The repo id in the format of `owner_name/repo_name`. repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`. visibility (Literal['private', 'public']): The visibility to set, `private` or `public`. token (Union[str, None]): The access token. If None, will use the cookies from the local cache. See `https://modelscope.cn/my/myaccesstoken` to get your token. Returns: dict: The response from the server. rrrBz-, supported visibilities: `private`, `public`cSsi|]\}}||qSrrr=rrrr@sz.HubApi.set_repo_visibility..rTr)rrrrsrrr rModelFrameworkZPytorchr ApprovalModerrr ModelCoverSubScientificFieldNZNEXAScientificFieldr r)rrrE ProtectedModer rrrr!r"r#r Z ModelTaskrrzInvalid dataset repo_id: z-, should be in format of `owner/dataset_name`r)rrrrG)rEr$rFrGr)rrFrIrrrjr)rrnrrrir rrrrkrrrrrOrrP)rr+r*rrZvisibility_mapZvisibility_coderr@rtasksZ model_tasksfirstr\Z repo_id_partsZ dataset_idxr?rrrrrset_repo_visibilitysn               zHubApi.set_repo_visibilityskill collection_idc Cs~|s|j}|dkrtd|d|}|d}||||d}|jj|||||jd} t| | } t | | t S)aGet collection details and its elements. Args: collection_id (str): The collection ID (Fid). repo_type (str): Element type filter, only 'skill' is supported currently. page_number (int): Page number for pagination. page_size (int): Page size for pagination. Returns: dict: Collection details including elements. Raises: ValueError: If repo_type is not 'skill'. RequestError: If the API request fails. r(z repo_type=z7 is not supported, only "skill" is currently supported.z/api/v1/collections)ZFidZ ElementTyperrr) rrrrrrrrOrrPr) rr)r*rrrrrrrrrrrget_collection4s(    zHubApi.get_collectionskill_id local_dirc Cs|s|j}t|\}}|}|d|d|d}|dur#t}tj|ddtj||}|j j |d|| |j d} t | tj||d} zt| d } | jd d D] } | rc| | qZWdn1snwYtj|r~t|tj|ddt| d  } | |Wdn1swYt|}t|d krtj||d}tj|rt|D]}ttj||tj||qt|Wtj| rt| n tj| rt| wwtd|d|d||S)aDownload a single skill archive and extract it. Args: skill_id (str): The skill identifier in format '/'. local_dir (Optional[str]): Target directory for extraction. Defaults to current directory. Returns: str: Path to the extracted skill directory. Raises: ValueError: If skill_id format is invalid. RequestError: If the download request fails. z/api/v1/skills/rz/archive/zip/masterNTr5)rrrz.zipri rrrrzSkill z downloaded to )rrZvalidate_repo_idrrgetcwdmakedirsrrOrrrrrOrT iter_contentrrkrrrszipfileZipFile extractallrmrrumovermdirrqr&rL)rr+r,rZ element_pathZ element_namerrZ skill_dirrzip_pathrhrzfentriesZ nested_dirrrrrdownload_skill^s^                zHubApi.download_skill)NFr)NNNrb)rrNN)F)NFN)FN)TrrNN)rN)FNN)r)r(rrN)yrrrrrrrrorrrrrrDZPUBLICrCZ APACHE_V2rrrr#r$rerr)r6rSr@rRrDrjrrFr1rM staticmethodrZrfr rrrrrrrrrrrrrr>rrBrrKr__annotations__rrr=rCrBrrrrrpZREUSE_DATASET_IF_EXISTSrrdrrrrrrr r(r)rr-rr rr rr4rEr\rTr r~r}r^rrr rrrrrrr r?rrryrrrrZrrrrr'r*r9rrrrrsH  $  1 l "W# 0 . # $  ' = /       - 8     " F         ;   : #  % B  A $   $ = %   ""      "  1"       u   W        n   =            u F 9  ?,   Y+rc@seZdZeeZdZdZdZdZ dZ e ddZ e de fdd Ze d d Ze d d Ze defddZe dedefddZe deeeffddZe deefddZe ddeeedfdefddZdS)rrZ git_tokenuserrFcCstjtjdddS)NTr-)rr/rpath_credentialrrrrmake_sure_credential_path_existsz0ModelScopeConfig.make_sure_credential_path_existcCsRtttjtjtjd}t ||WddS1s"wYdS)Nzwb+) rr>rTrrrOr=COOKIES_FILE_NAMEpicklerU)rrhrrrrs "zModelScopeConfig.save_cookiescCstjtjtj}tj|rXt|d<}t |}|s% WddS|D]}|j dkrF| rFtj sFdt_ t dWddSq'|WdS1sSwYdS)NrrTzKNot logged-in, you can login for uploadingor accessing controlled entities.)rrrOrr=r?rkrTr@loadr  is_expiredcookie_expired_warningr&rL)Z cookies_pathrhrr/rrrrs*       zModelScopeConfig.get_cookiescCstjtjtj}d}tj|r2t|d}t| dd}|WdS1s-wY|dksr)Z session_pathZ session_idrhZwfrrrget_user_session_ids$       z$ModelScopeConfig.get_user_session_idrcCsPtttjtjtjd}||WddS1s!wYdS)NrE) rr>rTrrrOr=GIT_TOKEN_FILE_NAMErrrhrrrrs  "zModelScopeConfig.save_tokenr, user_emailcCsXtttjtjtjd}|d||fWddS1s%wYdS)NrEz%s:%s) rr>rTrrrOr=USER_INFO_FILE_NAMEr)r,rKrhrrrrs "zModelScopeConfig.save_user_inforcCs~z4ttjtjtjddd}|}|dd|ddfWdWS1s-wYWdSt y>YdSw)Nrrr:rrr) rTrrrOrr=rLrr FileNotFoundError)rhrLrrr get_user_infos" $ zModelScopeConfig.get_user_infocCshd}z'ttjtjtjddd}|}WdW|S1s"wYW|Sty3Y|Sw)z Get token or None if not existent. Returns: `str` or `None`: The token, `None` if it doesn't exist. Nrrr) rTrrrOrr=rIrrNrJrrr get_tokens&     zModelScopeConfig.get_tokenN user_agentcCsd}ttjvr tjt}d}ttjvrtjt}ddlm}d|tt tt ||f}t |t rH|dd dd|D7}|St |trS|d|7}|S) aFormats a user-agent string with basic info about a request. Args: user_agent (`str`, `dict`, *optional*): The user agent info in the form of a dictionary or a single string. Returns: The formatted user-agent string. Zcustomunknownr) __version__zSmodelscope/%s; python/%s; session_id/%s; platform/%s; processor/%s; env/%s; user/%sz; css"|] \}}|d|VqdS)rNrr=rrrr-s z2ModelScopeConfig.get_user_agent..)r"rrr#Z modelscoperSplatformpython_versionrrH processorrnrrOrIro)rQenvr,rSZuarrrr s,       zModelScopeConfig.get_user_agentrb)rrrrr$r=r?rIrLrFrCr:r>rrrrHrorrrrOrrPrr rrrrrrs2    $rc @seZdZdZeeeeefde de de de de f ddZ dd d Z d e e effddZde e eeefde d efddZdee e efde d d fddZd S)ra Check the files and folders to be uploaded. Args: max_file_count (int): The maximum number of files to be uploaded. Default to `UPLOAD_MAX_FILE_COUNT`. max_file_count_in_dir (int): The maximum number of files in a directory. Default to `UPLOAD_MAX_FILE_COUNT_IN_DIR`. max_file_size (int): The maximum size of a single file in bytes. Default to `UPLOAD_MAX_FILE_SIZE`. size_threshold_to_enforce_lfs (int): The size threshold to enforce LFS in bytes. Files larger than this size will be enforced to be uploaded via LFS. Default to `UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS`. normal_file_size_total_limit (int): The total size limit of normal files in bytes. Default to `UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT`. Examples: >>> from modelscope.hub.api import UploadingCheck >>> upload_checker = UploadingCheck() >>> upload_checker.check_file('/path/to/your/file.txt') >>> upload_checker.check_folder('/path/to/your/folder') >>> is_lfs = upload_checker.is_lfs('/path/to/your/file.txt', repo_type='model') >>> print(f'Is LFS: {is_lfs}') max_file_countmax_file_count_in_dir max_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitcCs"||_||_||_||_||_dSrb)rXrYrZr[r\)rrXrYrZr[r\rrrrJs  zUploadingCheck.__init__rNcCslt|ttfrtj|std|dt|}||jkr4t d|jddt |ddddSdS) a Check a single file to be uploaded. Args: file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked. Raises: ValueError: If the file does not exist or exceeds the size limit. File  does not existzFile exceeds size limit: @z GB, got  GBN) rnrorrrrkrrvrZr&r4r)rrgrhrrrrwXs   zUploadingCheck.check_filercCsd}d}t|tr t|}|D][}|r>|d7}t|}||jkr=td|d|jdddt |dddq| rl|d7}| |\}}|||j krdt d |d ||d |j ||7}||7}q||jkr}t d |d |j||fS)z Check a folder to be uploaded. Args: folder_path (Union[str, Path]): The folder path to be checked. Raises: ValueError: If the folder does not exist or exceeds the file count limit. rrr]z exceeds size limit: r_razgot r`z Directory z contains z items and exceeds limit: zTotal file count z and exceeds limit: )rnroriterdirrrvrZr&r4rrrrYrrX)rrZ file_countZ dir_countrZ item_sizeZsub_file_countZ sub_dir_countrrrrks4     zUploadingCheck.check_folderrgr*cCsd}t|ttfr;t|}|std|d|tkr%|jtvr$d}n|tkr1|jt vr0d}n td|dt t |}||j kpE|S)aj Check if a file should be uploaded via LFS. Args: file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked. repo_type (str): The repo type, either `model` or `dataset`. Returns: bool: True if the file should be uploaded via LFS, False otherwise. Tr]r^FrFrG) rnrorrkrrjrr{riryrkrvr[)rrgr*Zhit_lfs_suffixrhrrrrzs    zUploadingCheck.is_lfsrfcsTfdd|D}tdd|D}|jkr(td|ddjdddS) af Check a list of normal files to be uploaded. Args: file_path_list (List[Union[str, Path]]): The list of file paths to be checked. repo_type (str): The repo type, either `model` or `dataset`. Raises: ValueError: If the total size of normal files exceeds the limit. Returns: None csg|] }|s|qSr)rzrerr*rrrrirjz5UploadingCheck.check_normal_files..cSsg|]}t|qSr)rvrcrrrrirzTotal size of non-lfs files izMB and exceeds limit: MBN)rr\r)rrfr*Znormal_file_listrrrdrrxs   z!UploadingCheck.check_normal_files)rN)rrrrr4r5r6r>r7rrrwrrorrrr rrzr rxrrrrr3s, "&&r)rxrrrurr@rTrcrrrKrrrr1 collectionsrrrhttprhttp.cookiejarrZos.pathrpathlibrtypingr r r r r rrrrrrrrrZrequests.adaptersrrZrequests.exceptionsrrrZmodelscope.hub.constantsrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFZmodelscope.hub.errorsrGrHrIrJrKrLrMrNrOrPZmodelscope.hub.gitrQZmodelscope.hub.inforRrSrMrUZmodelscope.hub.upload_cacherVZmodelscope.hub.upload_pipelinerWZmodelscope.hub.upload_trackerrXrYrZr[Zmodelscope.hub.utils.aigcr\Zmodelscope.hub.utils.utilsr]r^r_r`rarbrcZmodelscope.utils.constantrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtZmodelscope.utils.file_utilsrurvrwZmodelscope.utils.loggerrxZmodelscope.utils.repo_utilsryrzr{r|r}r~rrr&rrrrrrrrrrs      ,    0*     $L (&