o *j@snUdZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZmZmZddlmZddlZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%mZ&zddlm Z'Wn e(ytdZ'Ynwdd l)m*Z*dd l+m,Z,dd l-m.Z.m/Z/dd l0m1Z1m2Z2dd l3m4Z4m5Z5m6Z6m7Z7m8Z8ddl9m:Z:m;Z;ddlm?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZLddlJmMZMmNZNmOZOddlPmQZQddlRmSZSddlTmUZUddlVmWZWddlXmYZYddlZm[Z[m\Z\ddl]m^Z^ddl_m`Z`ddlambZbdd lcmdZdmeZemfZfdd!lgmhZhmiZimjZjmkZkmlZlmmZmmnZnz dd"l3moZpmqZrWne(yOdd#l3mpZpmrZrYnwebZse d$Zte'dur^e'ne&Zud%e fd&d'Zvd(ewd)ed*ewfd+d,Zxddd-ddd.d/ewd0eewd1eeyd2ezd3eeezewfd4ee etd*eLfd5d6Z{iZ|e e}e eeNeOffe~d7<d/ewd0ewd8ewd9ezd*ee eeNeOff d:d;Z dLdeewd3eeezewfd*e eeNeOffd?d@Zd-ddddAd/ewdBee ewewfd4ezd0eewd>eewd3eeezewfd*e eeNeOffdCdDZdadMdFdGZGdHdIdIZejdJdKZdS)NuModelScope dataset loading orchestration. This module provides :class:`DatasetsWrapperHF` and the :func:`load_dataset_with_ctx` context manager that monkey-patch the HuggingFace ``datasets`` library to work with the ModelScope Hub. Sub-modules: _compat – backward-compat shims for datasets>=4.0 script loading _module_factories – dataset module factory functions & data-file resolution N)fields)Path) AnyDictIterableListLiteralMappingOptionalSequenceTupleUnion) urlencode)DatasetDatasetBuilder DatasetDictDownloadConfigDownloadManager DownloadModeFeaturesIterableDatasetIterableDatasetDictSplitVerificationModeVersionconfig data_files LargeListr )r)features)_FEATURE_TYPES) DataFilesDictEmptyDatasetError)DataFilesNotFoundErrorDatasetNotFoundError)CachedDatasetModuleFactory DatasetModule(HubDatasetModuleFactoryWithParquetExportPackagedDatasetModuleFactoryget_dataset_builder_class)_EXTENSION_TO_MODULE_PACKAGED_DATASETS_MODULES) file_utils)!_raise_if_offline_mode_is_enabled cached_pathrelative_to_absolute_path)is_small_dataset) tracked_str) hf_hub_url)OfflineModeIsEnabled) DatasetInfo)HfApiRepoFile RepoFolder) HfFileSystem)HubApi) get_endpoint)get_from_cache_ms)MS_DATASETS_CACHE)DEFAULT_DATASET_REVISIONREPO_TYPE_DATASET)has_attr_in_class)is_relative_path) get_logger)_HAS_SCRIPT_LOADING!HubDatasetModuleFactoryWithScript#LocalDatasetModuleFactoryWithScript)_resolve_pattern_download_repo_fileget_module_without_scriptget_module_with_script_compat_local_script_module_compat_hub_script_module _get_hub_api)HubDatasetModuleFactoryLocalDatasetModuleFactory)$HubDatasetModuleFactoryWithoutScript&LocalDatasetModuleFactoryWithoutScript)authorcardDataZcitation createdAtdisabled description downloadsZdownloadsAllTimegated lastModifiedlikesZpaperswithcode_idprivatesiblingsshatagsobjcsNt|tr dd|DSd|vst|dtr dd|DSt|}|d}|dkr@|d}|dd }tt||d St|d pLt |d }|d ur_t d |d tt |t krr|d}t t|fi|St d ur|t ust|t r|d}t t|fi|Sddt|D|difdd|DS)a&Regenerate the nested feature object from a deserialized dict. This is a ModelScope-patched version of ``features.generate_from_dict`` that handles backward compatibility for legacy ``Sequence`` types in datasets 4.0+ where ``Sequence`` is no longer a registered feature type. cSg|]}t|qSgenerate_from_dict_ms).0valuer^r^m/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/utils/hf_datasets_util.py z)generate_from_dict_ms.._typecSsi|] \}}|t|qSr^r_)rakeyrbr^r^rc z)generate_from_dict_ms..r featurelength)rjrkNzFeature type 'z&' not found. Available feature types: cSsh|]}|jqSr^)name)rafr^r^rc z(generate_from_dict_ms..csi|] \}}|vr||qSr^r^)rakv field_namesr^rcrhsr^) isinstancelistdictitemspopget SequenceHfr`rglobals ValueErrorkeysr _NativeList issubclassr)r\rfrjrkZ class_typer^rsrcr`s*       r`url_or_filenamedownload_configreturnc Cs2t|}|dru|tdd}dD]}||r$|t|d}nqd|vrS|d}||dd}|d}|dkrD|}d} n#|d|}||dd} n|dd } t}t| d kre| d nd} td || d } |j| }nt |rt}td ||d } |j| }t ||d } t | } | || S) zModelScope replacement for ``DownloadManager._download``. Rewrites relative paths and ``hf://`` URLs to ModelScope API endpoints. zhf://N)z datasets/zmodels/@/rlZSDK)ZSourceZRevisionZFilePath)r) str startswithlenindexfindsplitr<rZ _base_pathr?r-r0Z set_origin) selfrrZhf_path_prefixZat_idxZafter_atZ slash_idxrevision file_pathparts params_stroutr^r^rc _download_mss<          rF)rtimeoutfiles_metadatatokenexpandrepo_idrrrrrc Cs|j|d|d||td}|d|r|ddnd|dddddggdd }g} |D]} t| tr<| t| j| j| j dq(| |d<t d i|S) z2ModelScope replacement for ``HfApi.dataset_info``.rF)r path_in_repor recursiverr repo_typerN) idrXrOrZrVrUrRrTrWr[rPrQ) rfilenameZblobIdsizerYr^) list_repo_treer=rrur5appendrwrZblob_idr HfDatasetInfo) rrrrrrrrepo_info_iter data_infoZ data_siblingsZ info_itemr^r^rc _dataset_infosF   r_repo_tree_cacherrc Cs||ddf}t|}|durdS|r|dkr|dnd}g}|D]5}|j} |rI| |ds5| |kr5q"| |drF| t|ddnd} n| } |sRd| vrRq"||q"|S)z;Try to derive results from a cached recursive root listing.rTNrr)rrzstrippathrrr) rrrrZroot_keyZ root_cachedprefixresultsitemZ item_pathrel_pathr^r^rc_derive_from_recursive_cache s"  (  rT)rrrrrrc cs|pt}|pd}||||f} t| } | dur| EdHdSt||||} | dur5| t| <| EdHdSt} | j|td} |d\}}| j||| d\}}g}d}d}d}||krz| j ||||||| |d}Wnt y}zt d |d |WYd}~nMd}~ww|snE|D]3}|d d krd nd|d|d|dd}|ddkrt di|ntdi|}|||Vqt||krn|d7}||ks\|t| <dS)z4ModelScope replacement for ``HfApi.list_repo_tree``.rNrr dataset_name namespaceendpointrii')rr root_pathr page_number page_sizerdataset_hub_idz Get dataset: z file list failed, message: Typetree directoryfilerSizeZSha256)typerroidrr^)r<rrzrrJget_endpoint_for_readr=rZget_dataset_id_and_typeZget_dataset_files Exceptionloggererrorr5r6rr)rrrrrrrrZnormalized_path cache_keycachedZderivedapir_owner _dataset_namer_rrrZ max_pagesZ dataset_fileseZ file_info_dZ path_inforr^r^rc_list_repo_tree&st         (   "r)rrrrpathsc s|pt}t|tr |g}t|tD],\}}|d|ks$|d|kr%qfdd|D} | r4| S|||ddfkr@gSq|j|d||||d} fd d| DS) z4ModelScope replacement for ``HfApi.get_paths_info``.rrcg|] }|jvr|qSr^rrarZ paths_setr^rcrdriz#_get_paths_info..rTF)rrrrrrcrr^rrrr^rcrdri)r<rursetrrxr) rrrrrrrrZ cached_itemsZmatchedrr^rrc_get_paths_infors,  rrbcKs8|dkrd|vrz||}|}||}d}||jvr;|j|D]}|d|kr:|ddkr:|dd}nq#|dkrt|j|j|j|j |j d } |j } t j| | d d d } | jd kr| jd} | rt| } | |d<|j|gD]}|d|kr| |d<nqwWn tyYnwt||fd|i|S)asWrapper for HfFileSystem._open that fixes size=0 from ModelScope API. The ModelScope tree API may report Size=0 for files. When HfFileSystem caches this, AbstractBufferedFile treats the file as empty (0 bytes). This wrapper detects size=0 for files opened in read mode and resolves the actual size via a HEAD request before creating the file object. rrNrmrrrlr)rrfilenamerrT)headersallow_redirectsrzContent-Lengthmode)Z resolve_pathZ unresolveZ_parentZdircacherzr1rrrrrZ_apiZ_build_hf_headersrequestshead status_coderintr_hf_fs_open_original)rrrkwargsresolvedZ resolved_nameparentZ cached_sizeentryurlrrespclZ actual_sizer^r^rc _hf_fs_opensJ         rc/@steZdZe                    d dedeedeedeeeeeeeeeeefffdeeee fd eed ee d ee d eee efd eee efdeededeeeefdeeeefdedeedeededeedeeeeeeff(ddZe             d!dedeedeedeeeeeeeeeeefffd eed ee d ee d eee efdeeeefdeeeefdeedeedefddZe          d"dedeeeefd ee d eee efdeedeedeeeeeefd eedeedefddZdS)#DatasetsWrapperHFNF deprecatedrrmdata_dirrr cache_dirrr download_modeverification_modekeep_in_memory save_infosrr streamingnum_procstorage_optionstrust_remote_codedataset_info_onlyrc#Ks|dkr tdt|} |dkrtdtnd}|dur'|s'td|dt|tjr3td|r=|dur=tdt |pBt j }t | sL| pKt j nt j } |r[td|d tjd||||||||| | |||dud |}|ri}t|tr|d rtj|rd d lm}||}dd|D}|S|dust|dstd|d|S|j}|D]%\}}t|}t|dr|jdurddt|jD||<qg||<q|S|r|j |dS|j!||| ||d| dur| nt"|j#j$} |j%|| | d}|dur(t&t'dt|(|}Wdn 1s#wY| r/|)z/t*}t+|rY|,ddkr\|-d\}} |j.|t/d}!|j0| ||!dW|SW|SW|St1yz}"ztd|"WYd}"~"|Sd}"~"ww) Nr'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. You can remove this warning by passing 'token=' instead.zF'task' was deprecated in version 2.13.0 and will be removed in 3.0.0. zEmpty 'data_files': 'z3'. It should be either non-empty or None (default).zjYou are trying to load a dataset that was saved using `save_to_disk`. Please use `load_from_disk` instead.zLoading a streaming dataset in parallel with `num_proc` is not implemented. To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead.z3Use trust_remote_code=True. Will invoke codes from z9. Please make sure that you can trust the external codes.) rrmrrrrrrrrrr_require_default_config_name.pyr)get_dataset_config_namescSsi|]}|gqSr^r^)raZ_subsetr^r^rcrh+rpz2DatasetsWrapperHF.load_dataset..builder_configszNo builder_configs found for z dataset.rcSr]r^)rrr^r^rcrd7rez2DatasetsWrapperHF.load_dataset..)r)rrrrr)rrZ in_memoryignorerrrrz&Could not record download statistics: r^)2warningswarn FutureWarningr}rrZDATASET_STATE_JSON_FILENAMEexistsNotImplementedErrorrREUSE_DATASET_IF_EXISTSrZ BASIC_CHECKSZ ALL_CHECKSrwarningrload_dataset_builderrurendswithosrdatasetsrhasattrrrrxrrvr~Zas_streaming_datasetZdownload_and_preparer/infoZ dataset_sizeZ as_datasetcatch_warnings simplefilterZprepare_for_taskZ _save_infosrJr?countrrr=Zdataset_download_statisticsr)#rrmrrrrrrrrrrrruse_auth_tokentaskrrrrr config_kwargsbuilder_instanceZret_dictrZ subset_listZ_tmp_builder_configsZtmp_config_nameZtmp_builder_configZdsrZ _namespacerrrr^r^rc load_datasets             zDatasetsWrapperHF.load_datasetTc s| dkr tdt| } t|ptj}| dur#|r|nt}| |_| dur6|r-|nt}|j | t j ||||||| | t ||d }|j }|d|}|d|}|d|p_|jj}|dd}|jrp|j|nd}tvr|dur|jjdjdurd d }fd d tD}|r|d |dd7}t|t||d}|d||||||j||| | d ||}|||S)Nrr) rrrrrrrr_require_custom_configsrmrr config_namerrz@Please specify the data files or data directory to load for the z dataset builder.csg|] }t|kr|qSr^)r))ra extensionrr^rcrds  z:DatasetsWrapperHF.load_dataset_builder..z9 For example `data_files={"train": "path/to/data/train/*.z"}`)r) rrrrrhashrrrrr^)rrrrrcopyrrrupdaterdataset_module_factoryboolbuilder_kwargsryZbuilder_configs_parametersZdefault_config_nameZ dataset_infosrzr*rrr)r}r(rZ!_use_legacy_cache_dir_if_possible)rrmrrrrrrrrr rrrr Zdataset_modulerrrr error_msgZexample_extensionsZ builder_clsr r^rrcras           z&DatasetsWrapperHF.load_dataset_builderdynamic_modules_pathc Ks| dd} |p t}|durtd.i| }|jd| i|jd|i|r/|jdur/t|_t|p4tj}d|_ d|_ |tj k|_ t tdd|tjddd} | ds^| d} tj|| }|tvrtt|||||d S|| rtj|rtrt||||d St||||d Std t|tj|rtrt||||d St||||d Stj|rt ||||d St!|ro|"dd kroz!t#z t$j%|||j&dd}Wngt'yL}zZt(|t)t*j+j,t*j+j-fr t-d|dt.|j/ddt0|vr&d|d}t1|r#|d|d|dt0|vrFd|d}|r>|d|dn|}t1|d|d}~wwt2|d||d}tj3tj4|}| dd|j5Dvrd}t6j7r|rzt8t9drt9|||dWWSt9|||d WWSt'y}z t:;|WYd}~nd}~wwtrt8tdrt>||||||d#WSt>||||||d$WSt'yn}zlt:;d%|d&|zd'|i}tr||d(<t?|fi|WWYd}~St'yit(|t)r>t-d)|d*|dt(|t@t1tAfrJ|dt(|trftd t|d+|d,t.|j/d&|d|dwd}~wwtd t|d-)/NrmrTcSs|SNr^)xr^r^rcsz:DatasetsWrapperHF.dataset_module_factory..rrlr)rrrr)rrrz"Couldn't find a dataset script at )rrrrgY@)rrrrzCouldn't reach 'z' on the Hub ()Z404z Dataset 'z' doesn't exist on the Hubz at revision ''Z401zT. If the repo is private or gated, make sure to log in with `huggingface-cli login`.z README.md)rrrrcSsg|]}|jqSr^)r)raZsiblingr^r^rcrdHsz.F)rr) commit_hashr)rrrrr)rrrrr)rrrrr)rrrrrz>> Error loading z: rrz1Couldn't reach the Hugging Face Hub for dataset 'z': z8 or any data file in the same directory. Couldn't find 'z"' on the Hugging Face Hub either: z( or any data file in the same directory.r^)Bryr<rrrrr;rrZextract_compressed_fileZ force_extractZFORCE_REDOWNLOADZforce_downloadrvfilterreplacerseprrrjoinr*r' get_moduleisfilerArCrHFileNotFoundErrorr.isdirrNr?r r,r4 dataset_inforrrur2r exceptionsConnectTimeoutConnectionErrorr__name__rr#rEbasenamedirnamerYrZUSE_PARQUET_EXPORTr>r&rrrBrIrMr$r"r!)rrrrrrrrrrrZdownload_kwargsZ subset_namerZ combined_pathr(rmsgZdataset_readme_pathrZ#can_load_config_from_parquet_exporte1Z_cached_factory_kwargsr^r^rcrs                      z(DatasetsWrapperHF.dataset_module_factory)NNNNNNNNNNFNNrrFNNFF) NNNNNNNNNrNNT) NNNNNNNNTF)r, __module__ __qualname__ staticmethodrr r r r rrrrrrrrrrrrrrwrrrrr r%rr^r^r^rcrs&              ^  rc ostj}tj}ttdrtjntj}tj }tj }tj }t j }tj} tr&tjnd} tj} tj} tt_tt_ttdr>tt_ntt_tt_ tt_ tt_ tt _ tt_trUtt_tt_| a t!t_|"dd} zKt#j$|i|}|VWt%&t'j(&| t_da | s|t_|t_| t_ttdr|t_n|t_|t_ |t_ |t_ |t _ | t_tr| t_dSdSdSt%&t'j(&| t_da | s|t_|t_| t_ttdr|t_n|t_|t_ |t_ |t_ |t _ | t_tr| t_w)aContext manager that monkey-patches ``datasets`` to use ModelScope. All monkey-patches are applied on entry and restored on exit (for non-streaming mode) or kept alive (for streaming mode, where lazy iteration needs the patches to remain active). _downloadNrF))rZ HF_ENDPOINTr+Zget_from_cacherrr4Z_download_singler4r(rZget_paths_inforZresolve_patternrMr$rArBrZgenerate_from_dictr7_openr9r:rrrrrDrFrGr`rrrzrrrclearr8Z_dataset_id_type_cache)argsrZhf_endpoint_originZget_from_cache_originZ_download_originZdataset_info_originZlist_repo_tree_originZget_paths_info_originZresolve_pattern_originZ get_module_without_script_originZget_module_with_script_originZgenerate_from_dict_originZhf_fs_open_originrZ dataset_resr^r^rcload_dataset_with_ctxs         r8r)r)__doc__ contextlibrr dataclassesrpathlibrtypingrrrrrr r r r r urllib.parserrrrrrrrrrrrrrrrrrr{Z DatasetList ImportErrorZdatasets.featuresrZdatasets.features.featuresrZdatasets.data_filesr r!Zdatasets.exceptionsr"r#Z datasets.loadr$r%r&r'r(Zdatasets.packaged_modulesr)r*Zdatasets.utilsr+Zdatasets.utils.file_utilsr,r-r.Zdatasets.utils.info_utilsr/Zdatasets.utils.trackr0Zhuggingface_hubr1Zhuggingface_hub.errorsr2Zhuggingface_hub.hf_apir3rr4r5r6Zhuggingface_hub.hf_file_systemr7Z modelscoper8Zmodelscope.hub.utils.utilsr9Z)modelscope.msdatasets.utils.hf_file_utilsr:Zmodelscope.utils.config_dsr;Zmodelscope.utils.constantr<r=Zmodelscope.utils.import_utilsr>Zmodelscope.utils.file_utilsr?Zmodelscope.utils.loggerr@Z#modelscope.msdatasets.utils._compatrArBrCZ-modelscope.msdatasets.utils._module_factoriesrDrErFrGrHrIrJrKrMrLrNrZExpandDatasetProperty_Trr`rrfloatrrrtuple__annotations__rrrrrrcontextmanagerr8r^r^r^rcs   0 H                 $ '/  5   Q + /a