o *j@sddlZddlZddlmZmZmZmZmZmZm Z m Z m Z ddl Z ddlmZmZmZmZmZddlmZddlmZddlmZddlmZmZmZmZddlm Z m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+ddl,m-Z-m.Z.ddl/m0Z0ddl1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlm?Z?m@Z@ddlAmBZBeBZCdefddZDGdddZEdS)N) AnyCallableDictIterableListMappingOptionalSequenceUnion)Dataset DatasetDictFeaturesIterableDatasetIterableDatasetDict)_PACKAGED_DATASETS_MODULES)DatasetRepository)DatasetContextConfig)LocalDataLoaderManagerLocalDataLoaderTypeRemoteDataLoaderManagerRemoteDataLoaderType)ExternalDatasetNativeIterableDataset)build_custom_dataset)DatasetDeleteManager)load_dataset_with_ctx)DatasetUploadManager)build_preprocessor)Config ConfigDict)MS_DATASETS_CACHE) DEFAULT_DATASET_NAMESPACEDEFAULT_DATASET_REVISIONREPO_TYPE_DATASET ConfigFieldsDatasetFormations DownloadModeHubsModeKeysTasks UploadMode)is_relative_path)is_tf_availableis_torch_available) get_loggerreturncCsJ|durg}|St|tr|g}|Stt|t|kr#td||S)Nz"List columns contains duplicates: ) isinstancestrlenset ValueError)parar6a/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/msdatasets/ms_dataset.py format_list)s r8c+@sReZdZUdZdZdZeed< d^dee e e e fde efddZdd Zd d Zd d ZeddZeddZe d^dee ee fdedeedffddZe d^dee ee e e efdedeedffddZeedeej dddde!j"e#ddde$dddfdeee%fde ede ede ede ede ede ed e ed!e eee&ee'eeee&efffd"e e!d#e ed$e e(d%e e)d&e e*d'e e$d(e ed)e e)d*e e)deede ff&d+d,Z+eeeddd-e,j-fd.ed/edede ede ed0e e*d1e e*d2e e)d3e e,ddfd4d5Z.eeddfd6ed7ed8e ed9e ed:e eddf d;d<Z/eedddfd6ed=ed8e ed9e ed:e ed>e)ddfd?d@Z0eeefd.edede ede edef dAdBZ1     -d_dCeee2efdDee3e2e3fdEedFe4dGe)f dHdIZ5       -d`dJe*dKe)dDee3e2e3fdCeee2efdLe3dMe)dNe6ee7fdOeee2efdPe)fdQdRZ8de fdSdTZ9dUe6eefde fdVdWZ:  -dadDee3e2e3fdCeee2efdGe)fdXdYZ;  -  dbdJe*dKe)dDee3e2e3fdMe)dPe)dOeee2efdCeee2effdZd[Z<  dcd'e$fd\d]Z=dS)d MsDataseta ModelScope Dataset (aka, MsDataset) is backed by a huggingface Dataset to provide efficient data access and local storage managements. On top of that, MsDataset supports the data integration and interactions with multiple remote hubs, particularly, ModelScope's own Dataset-hub. MsDataset also abstracts away data-access details with other remote storage, including both general external web-hosted data and cloud storage such as OSS. N_dataset_context_config ds_instancetargetcCsJ||_|dur||jjvrtdt|jjd|||_d|_dS)Nz)"target" must be a column of the dataset(z , but got F)_hf_dsfeatures TypeErrorlistkeysr< is_custom)selfr;r<r6r6r7__init__@s zMsDataset.__init__ccs0|jD]}|jdur||jVq|VqdSN)r=r<)rCitemr6r6r7__iter__Ls   zMsDataset.__iter__cCs |j|SrEr=)rCkeyr6r6r7 __getitem__S zMsDataset.__getitem__cC t|jSrE)r2r=rCr6r6r7__len__VrKzMsDataset.__len__cCs|jSrErHrMr6r6r7r;YszMsDataset.ds_instancecCst|jtr |jjSdSrE)r0r=r config_kwargsrMr6r6r7rO]s zMsDataset.config_kwargshf_dsr/cstdtt|tr|St|tr4t|dkr(tt | Sfdd| DSt|t r=|St dt|)z @deprecated This method is deprecated and may be removed in future releases, please use `to_ms_dataset()` instead. z@from_hf_dataset is deprecated, please use to_ms_dataset instead.ci|] \}}||qSr6r6.0kvclsr<r6r7 tz-MsDataset.from_hf_dataset..z2"hf_ds" must be a Dataset or DatasetDict, but got )warningswarnDeprecationWarningr0r r r2rAnextitervaluesitemsrr?type)rXrPr<r6rWr7from_hf_datasetds     zMsDataset.from_hf_datasetcst|tr |St|tr.t|dkr"tt|Sfdd|DSt|t r7|St|t r@|St|t rI|St|t rmt|dkratt|Sfdd|DSt dt|)z&Convert input to `MsDataset` instance.rQcrRr6r6rSrWr6r7rYrZz+MsDataset.to_ms_dataset..crRr6r6rSrWr6r7rYrZz8"ds_instance" must be a Dataset or DatasetDict, but got )r0r r r2rAr^r_r`rarrrrr?rb)rXr;r<r6rWr7 to_ms_dataset|s&        zMsDataset.to_ms_datasetFrQ dataset_name namespaceversionhub subset_namesplitdata_dir data_files download_mode cache_dirr> use_streamingstream_batch_size custom_cfgtokendataset_info_onlytrust_remote_codec'Ks|rddlm}|}||t| ptj} t|ptj}|tjk}t|t s6t|t s6t dt |t|t rO|durAd}t ||i}tj||dStj|}tj|}t|r|ddkr|s|s|d}|d}|d}|r|sd |rtd |d |d d krddl}ddl}z||jWnt y|dYnwt!d|||||||||| | | | |d|}|t"vstj#|stj$|rt%|&t'j(}tj||d}t|tr||_)|r|j*dd|i|d|_+|S|tjkr!ddl,m&}||f||||| | | j-||| d |S|tjkrddlm}|}|j.|d|t/d}|j0|||d\}} t | t t1j2j-krt3d|d|||||| | d| j-||| ||d| }!|!WdS1sywYdSt4|}"|"&t5j6}tj||d}t|tr|"j7|_)|r|j*dd|i|d|_+|S|tj8krt9:dt;ddlm?}$|t@kr|$jA|_B|tCkr|$jD|_E| tFkrddlGmH}%tjI|%ddd} | |_J|#|}&|&K|&jLSd) a' Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. Args: dataset_name (str): Path or name of the dataset. The form of `namespace/dataset_name` is also supported. namespace(str, optional): Namespace of the dataset. It should not be None if you load a remote dataset from Hubs.modelscope, namespace (str, optional): Namespace of the dataset. It should not be None if you load a remote dataset from Hubs.modelscope, target (str, optional): Name of the column to output. version (str, optional): Version of the dataset script to load: subset_name (str, optional): Defining the subset_name of the dataset. data_dir (str, optional): Defining the data_dir of the dataset configuration. I data_files (str or Sequence or Mapping, optional): Path(s) to source data file(s). split (str, optional): Which split of the data to load. hub (Hubs or str, optional): When loading from a remote hub, where it is from. default Hubs.modelscope download_mode (DownloadMode or str, optional): How to treat existing datasets. default DownloadMode.REUSE_DATASET_IF_EXISTS cache_dir (str, Optional): User-define local cache directory. use_streaming (bool, Optional): If set to True, no need to download all data files. Instead, it streams the data progressively, and returns NativeIterableDataset or a dict of NativeIterableDataset. stream_batch_size (int, Optional): The batch size of the streaming data. custom_cfg (str, Optional): Model configuration, this can be used for custom datasets. see https://modelscope.cn/docs/Configuration%E8%AF%A6%E8%A7%A3 token (str, Optional): SDK token of ModelScope. dataset_info_only (bool, Optional): If set to True, only return the dataset config and info (dict). trust_remote_code (bool, Optional): If set to True, trust the remote code. Default to `False`. **config_kwargs (additional keyword arguments): Keyword arguments to be passed Returns: MsDataset (MsDataset): MsDataset object for a certain dataset. r)HubApiz.dataset_name must be `str` or `list`, but got Nr<)r</rQzUThe dataset_name should be in the form of `namespace/dataset_name` or `dataset_name`.z3Use trust_remote_code=True. Will invoke codes from z9. Please make sure that you can trust the external codes.Zenginepythoni)rerfrgrirjr<rhrkrlrmcache_root_dirrorprtrqT) load_dataset) namerkrlrjrnr>rmrevisionrr streaming)Zrepo_idZ repo_type)rerfendpoint)pathrzrkrlrjrnr>Zdownload_configrmr{rrr|rsrtzMThe option `Hubs.virgo` is deprecated, will be removed in the future version.)VirgoDownloader)VirgoDatasetConfig) CACHE_HOMEvirgorhdatasetszPlease adjust input args to specify a loading mode, we support following scenes: loading from local disk, huggingface hub and modelscope hub.r6)MZmodelscope.hub.apiruloginr&REUSE_DATASET_IF_EXISTSr' modelscopeZ huggingfacer0r1r@r?rbr from_dictr9rdosr~ expanduserexistsr+countrjstriploggerwarninggetcsvsysfield_size_limitmaxsize OverflowErrorrrisdirisfilerryrZHF_DATA_LOADERr:to_custom_datasetrBrvalueZget_endpoint_for_readr#Zget_dataset_id_and_typer%ZgeneralrrrZMS_DATA_LOADERdataset_context_configrr[r\r]Z-modelscope.msdatasets.data_loader.data_loaderrmodelscope.utils.constantrr!Zdefault_virgo_namespacerfr"Zdefault_dataset_versionrgr modelscope.utils.config_dsrjoinrxprocessdataset)'rerfr<rgrhrirjrkrlrmrnr>rorprqrrrsrtrOruapiZis_huggingface_hubZ dataset_inst is_local_pathZdataset_name_splitZ csv_modulerrryZ_apir}Zdataset_id_on_hubZ dataset_typeZ dataset_resZremote_dataloader_managerrrrZvirgo_downloaderr6r6r7loadsj<                        &         zMsDataset.loadT object_namelocal_file_path num_processes chunksizefilter_hidden_files upload_modec Cs tdt|s tdt|||d} t|ptj}tj |r+| j |||ddStj |r>| j ||||||ddSt|d)z @deprecated This method is deprecated and may be removed in future releases, please use git command line instead. zThe function `upload` is deprecated, please use git command or modelscope.hub.api.HubApi.upload_folder or modelscope.hub.api.HubApi.upload_file.zobject_name cannot be empty!rerfrg)rrr)Zobject_dir_nameZlocal_dir_pathrrrrz& is not a valid file path or directoryN) r[r\r]r4rr* OVERWRITErr~ruploadrZ upload_dir) rrrerfrgrrrrZ_upload_managerr6r6r7rs:   zMsDataset.uploaddataset_work_dir dataset_idr{ auth_tokengit_pathcCsRtdtt|||||d}|}|rtd|dStd|dS)aClone meta-file of dataset from the ModelScope Hub. Args: dataset_work_dir (str): Current git working directory. dataset_id (str): Dataset id, in the form of your-namespace/your-dataset-name . revision (str, optional): revision of the model you want to clone from. Can be any of a branch, tag or commit hash auth_token (str, optional): token obtained when calling `HubApi.login()`. Usually you can safely ignore the parameter as the token is already saved when you login the first time, if None, we will use saved token. git_path (str, optional): The git command line path, if None, we use 'git' Returns: None zWThe function `clone_meta` is deprecated, please use git command line to clone the repo.Z repo_work_dirrr{rrzAlready cloned repo to: {}zRepo dir already exists: {}N) r[r\r]rclonerinfoformatr)rrr{rr_repoZclone_work_dirr6r6r7 clone_metas"zMsDataset.clone_metacommit_messageforcecCs2tdtt|d|||d}|j|||ddS)aUUpload meta-file of dataset to the ModelScope Hub. Please clone the meta-data from the ModelScope Hub first. Args: dataset_work_dir (str): Current working directory. commit_message (str): Commit message. revision(`Optional[str]`): revision of the model you want to clone from. Can be any of a branch, tag or commit hash auth_token(`Optional[str]`): token obtained when calling `HubApi.login()`. Usually you can safely ignore the parameter as the token is already saved when you log in the first time, if None, we will use saved token. git_path:(`Optional[str]`): The git command line path, if None, we use 'git' force (Optional[bool]): whether to use forced-push. Returns: None zuThe function `upload_meta` is deprecated, please use git command or CLI `modelscope upload owner_name/repo_name ...`.r)rbranchrN)r[r\r]rpush)rrr{rrrrr6r6r7 upload_metaszMsDataset.upload_metacCs0t|||d}|j|d}td|d|S)as Delete object of dataset. Please log in first and make sure you have permission to manage the dataset. Args: object_name (str): The object name of dataset to be deleted. Could be a name of file or directory. If it's directory, then ends with `/`. For example: your-data-name.zip, train/001/img_001.png, train/, ... dataset_name (str): Path or name of the dataset. namespace(str, optional): Namespace of the dataset. version (str, optional): Version of the dataset. Returns: res_msg (str): Response message. r)rzObject z successfully removed!)rdeleterr)rrerfrgZ_delete_managerZresp_msgr6r6r7rs  zMsDataset.deletecolumns preprocessors task_name data_config to_tensorcKsztstdt|jtr |d|i||jjt||S|dur,|j|||dS|j |jj d||d|jS)aFCreate a torch.utils.data.Dataset from the MS Dataset. The torch.utils.data.Dataset can be passed to torch.utils.data.DataLoader. Args: preprocessors (Callable or List[Callable], default None): (list of) Preprocessor object used to process every sample of the dataset. The output type of processors is dict, and each (numeric) field of the dict will be used as a field of torch.utils.data.Dataset. columns (str or List[str], default None): Dataset column(s) to be loaded (numeric data only if `to_tensor` is True). If the preprocessor is None, the arg columns must have at least one column. If the `preprocessors` is not None, the output fields of processors will also be added. task_name (str, default None): task name, refer to :obj:`Tasks` for more details data_config (ConfigDict, default None): config dict for model object. Attributes of ConfigDict: `preprocessor` (Callable, List[Callable], optional): preprocessors to deal with dataset `type` (str): the type of task `split_config` (dict, optional): get the split config for ExternalDataset `test_mode` (bool, optional): is test mode or not to_tensor (bool, default None): whether convert the data types of dataset column(s) to torch.tensor or not. format_kwargs: A `dict` of arguments to be passed to the `torch.tensor`. Returns: :class:`torch.utils.data.Dataset` z>The function to_torch_dataset requires pytorch to be installed preprocessorN)rrtorch)rbr format_kwargs) r- ImportErrorr0r=rupdaterOr!_to_torch_dataset_with_processors reset_format set_format)rCrrrrrrr6r6r7to_torch_dataset+s"!   zMsDataset.to_torch_dataset batch_sizeshuffle collate_fndrop_remaindercollate_fn_args label_colsprefetchc Csjtstd|dur|j||||| ||dS|dur"tddS|j|jj|||||||| dS)aCreate a tf.data.Dataset from the MS Dataset. This tf.data.Dataset can be passed to tf methods like model.fit() or model.predict(). Args: batch_size (int): Number of samples in a single batch. shuffle(bool): Shuffle the dataset order. preprocessors (Callable or List[Callable], default None): (list of) Preprocessor object used to process every sample of the dataset. The output type of processors is dict, and each field of the dict will be used as a field of the tf.data. Dataset. If the `preprocessors` is None, the `collate_fn` shouldn't be None. columns (str or List[str], default None): Dataset column(s) to be loaded. If the preprocessor is None, the arg columns must have at least one column. If the `preprocessors` is not None, the output fields of processors will also be added. collate_fn(Callable, default None): A callable object used to collect lists of samples into a batch. If the `preprocessors` is None, the `collate_fn` shouldn't be None. drop_remainder(bool, default None): Drop the last incomplete batch when loading. collate_fn_args (Dict, optional): A `dict` of arguments to be passed to the`collate_fn`. label_cols (str or List[str], default None): Dataset column(s) to load as labels. prefetch (bool, default True): Prefetch data. Returns: :class:`tf.data.Dataset` z?The function to_tf_dataset requires Tensorflow to be installed.N)rrrrz?The `preprocessors` and the `collate_fn` should`t be both None.)rrrr)r,r_to_tf_dataset_with_processorsrerrorr=r to_tf_dataset) rCrrrrrrrrrr6r6r7r]s<$  zMsDataset.to_tf_datasetcCs|j|jSrE)r=rrMr6r6r7 to_hf_datasets zMsDataset.to_hf_datasetcolumn_mappingcCs|j|j|S)a Rename columns and return the underlying hf dataset directly TODO: support native MsDataset column rename. Args: column_mapping: the mapping of the original and new column names Returns: underlying hf dataset )r=rZrename_columns)rCrr6r6r7 remap_columnss zMsDataset.remap_columnsc st|tr|n|g}tfdd|jjDg}g}|ritt|jfddD}|D]}|dd| Dq3dd} |D]} | || sct d| d | | qL| | qLd dl Gfd d d jjj} | |j||||S) Ncg|]}|vr|qSr6r6rTrI)rr6r7 z?MsDataset._to_torch_dataset_with_processors..csi|] }|t|qSr6nparrayrTrU)sampler6r7rYrZz?MsDataset._to_torch_dataset_with_processors..cSi|] \}}|t|qSr6rrSr6r6r7rYscSs t|jtjpt|jtjSrE)rZ issubdtypedtypeintegerZfloating)rr6r6r7is_numpy_numberszDMsDataset._to_torch_dataset_with_processors..is_numpy_numberzData of column z is non-numeric, will be removedrcs>eZdZdeffdd ZddZfddZdd ZZS) zAMsDataset._to_torch_dataset_with_processors..MsMapDatasetrcs4tt||_||_||_||_||_||_dSrE) superr9rDrpreprocessor_listrretained_numeric_columnsretained_unumeric_columnsr)rCrrrrrr __class__r6r7rDs  zJMsDataset._to_torch_dataset_with_processors..MsMapDataset.__init__cSrLrE)r2rrMr6r6r7rNrKzIMsDataset._to_torch_dataset_with_processors..MsMapDataset.__len__cs|jr|S|SrE)rZ as_tensor)rCxrr6r7type_converters zPMsDataset._to_torch_dataset_with_processors..MsMapDataset.type_convertercsxj|fddjD}jD]&}|D]\}}jr'|jvr/|||<q|jvr8|||<qq|S)Ncs,i|]}jr |jvr||qSr6)rrrrZ item_dictrCr6r7rYs  zaMsDataset._to_torch_dataset_with_processors..MsMapDataset.__getitem__..)rrrrarrrr)rCindexresrrUrVr6rr7rJs     zMMsDataset._to_torch_dataset_with_processors..MsMapDataset.__getitem__) __name__ __module__ __qualname__rrDrNrrJ __classcell__r6rrr7 MsMapDatasets  r)r0r@r8r=r>rAr^r_rrarrappendrutilsdatar ) rCrrrrrr sample_res processorrrUrr6)rrrr7rsF        &z+MsDataset._to_torch_dataset_with_processorsc sHt|tr|n|gtt|}tt|fddjjDddljj t j t jt jd}|rF|jt jd}dfdd dd jdjgd fd d } dd lm} |j| | d}rfdd} || }n t |dkr|dd}|dkr|j||d}|r|| }|S)Ncrr6r6r)cols_to_retainr6r7rrz.r)r) buffer_sizeFcs`tfddD}D]}|dd|jDq|r(|Stt|S)Ncs"i|] }|tj|qSr6)rrr=r)irCr6r7rYs"zJMsDataset._to_tf_dataset_with_processors..func..cSrr6rrSr6r6r7rYs )intrr=ratupler@r`)rZ return_dictrr)rretained_columnsrCrr7funcs  z6MsDataset._to_tf_dataset_with_processors..funcT)Zinput_signaturecs:j|gfddDdfddtDS)Ncsg|] }j|jqSr6)ZdtypesZas_dtyper)rTval)tfr6r7r-s zTMsDataset._to_tf_dataset_with_processors..fetch_function..)inpZToutcsi|] \}}||qSr6r6)rTrrIoutputr6r7rY2szTMsDataset._to_tf_dataset_with_processors..fetch_function..)Znumpy_functionr` enumerater)rrrrr7fetch_function(s z@MsDataset._to_tf_dataset_with_processors..fetch_function)AUTOTUNE)Znum_parallel_callscsVfdd|D}t|dkrtt|}t|dkr'tt|}||fS)Ncsi|] \}}|vr||qSr6r6)rTrIZtensorrr6r7rY:sz_MsDataset._to_tf_dataset_with_processors..split_features_and_labels..rQ)rar2r^r_r`)Z input_batchlabelsr r6r7split_features_and_labels9s   zKMsDataset._to_tf_dataset_with_processors..split_features_and_labelsrQcSstt|SrE)r^r_r`)rr6r6r7Gsz:MsDataset._to_tf_dataset_with_processors..)r)F)r0r@r8r3r=r>rAZ tensorflowrr Zfrom_tensor_slicesrZaranger2Zint64rfunctionZ TensorSpecZtensorflow.data.experimentalr mapbatchr) rCrrrrrrrZ tf_datasetrr r r6)rrrrrrrCrr7rsJ        z(MsDataset._to_tf_dataset_with_processorsc Ksptstd|s dSd|_|durd|vr|d}|tjkr"dnd}|d|}|durBt|tj r=t |j j dnt dd}| t |d |j}d |vrV|d }t|}d |vrd|d }|durwt|d rw|j} | rwt| |}t|jtr| t |d | |jjt||jd|_dS|dur|dd} |j|| d|_dS|j|jjdddS)aConvert the input datasets to specific custom datasets by given model configuration and preprocessor. Args: custom_cfg (Config): The model configuration for custom datasets. preprocessor (Preprocessor, Optional): Preprocessor for data samples. mode (str, Optional): See modelscope.utils.constant.ModeKeys Returns: `MsDataset` z?The function to_custom_dataset requires pytorch to be installedNTmodetrainrzdataset.)rb)rtaskfieldr)r)cfgrr)rrr)r-rrBrr(ZTRAINZsafe_gethasattrr$modelrrbrdictrpopr)Zfind_field_by_taskrrr0r=rrOrrrr) rCrqrrkwargsZ ds_cfg_keyZdata_cfgr field_nameZpreprocessor_cfgrr6r6r7rPsZ        zMsDataset.to_custom_datasetrE)NNNNT)NNNNNNT)NT)NTNN)NN)>rrr__doc__r=r:r__annotations__r r rrrrr1rDrGrJrNpropertyr;rO classmethodr rrcrrd staticmethodr!r"r'rr&rr rr@r rr boolrrr*rrrrrrrrrrrrrrrrrr6r6r6r7r93s                g  D & &  6   B V Rr9)Frr[typingrrrrrrrr r numpyrrr r r rrZdatasets.packaged_modulesrZmodelscope.hub.repositoryrZ4modelscope.msdatasets.context.dataset_context_configrZ5modelscope.msdatasets.data_loader.data_loader_managerrrrrZ!modelscope.msdatasets.dataset_clsrrZ9modelscope.msdatasets.dataset_cls.custom_datasets.builderrZ(modelscope.msdatasets.utils.delete_utilsrZ,modelscope.msdatasets.utils.hf_datasets_utilrZ(modelscope.msdatasets.utils.upload_utilsrZmodelscope.preprocessorsrZmodelscope.utils.configrrrr rr!r"r#r$r%r&r'r(r)r*Zmodelscope.utils.file_utilsr+Zmodelscope.utils.import_utilsr,r-Zmodelscope.utils.loggerr.rr8r9r6r6r6r7s0,         0