o *j v@sddlZddlZddlZddlZddlmZddlZddlZddl m Z ddl m Z ddl m Z ddlmZddlZddlmZddlmZmZmZmZmZmZmZmZmZmZmZddlm Z m!Z!dd l"m#Z#dd l$m%Z%e e&Z'd Z(e')j*d Z+e#Z,d Z-ddgZ.gdZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:dZ;dZd Z?d!Z@d"ZAd#ZBd$d%ZCeCZDGd&d'd'eEZFGd(d)d)eEZGeGZHd*eIdeeJe fd+dfd,d-ZKd?d/d0ZLd@d1d2ZMd3d4ZNd5d6ZOdd.e,e0fd7d8ZPdAd9d:ZQdBd;d<ZReSd=kr ePd d>ZTeUeTdSdS)CN)datetime)reduce)Path)Union)version) CustomDatasetsHeadsHooks LR_SchedulersMetricsModels Optimizers Pipelines Preprocessors TaskModelsTrainers)FieldsTasks)get_modelscope_cache_dir) default_groupTZregister_module modelscope.)modelsZmetricsZ pipelinesZ preprocessorsZtrainersZ msdatasetsZ exportersZ ast_indexerZ decoratorsexpressZ from_importsZimportsfilepathZmodelscope_pathrmd5index files_mtime requirementsmodule class_nameZ group_key module_nameZ module_cls TEMPLATE_PATHzast_index_file.pycCstd}|tj|S)Nzmodelscope.ast)logging getLoggersetLevelINFO)Z ast_loggerr([/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/utils/ast_utils.pyget_ast_logger;s  r*c @s&eZdZd)ddZdedefddZdejdefdd Z de ejd fdefd d Z d*dejdede fddZ d*dejdedefddZddZde ejdeffddZ d+de ejdefdedede fddZdejde fddZddZd e d!ede fd"d#Zd$edefd%d&Zd'd(ZdS), AstScanningreturnNcC t|_t|_g|_g|_dSN)dict result_importresult_from_importresult_decoratorrselfr(r(r)__init__F zAstScanning.__init__nodecCst|tjo t|tj Sr.) isinstanceastAST expr_contextr4r7r(r(r) _is_sub_nodeLs  zAstScanning._is_sub_nodecCsT|jD]$}t||}||rdSt|ttfr'|D] }||r&dSqqdS)NFT)_fieldsgetattrr=r8listtuple)r4r7fieldattrvalr(r(r)_is_leafPs    zAstScanning._is_leafstrcCs"trt|jdks |dkrdSdS)N FunctionDefTF)SKIP_FUNCTION_SCANNINGtype__name__r<r(r(r)_skip_function\szAstScanning._skip_functionTn show_offsetscCs|r|j|jS|jSr.) _attributesr>)r4rLrMr(r(r)r>bs zAstScanning._fieldscCs^t}t|tjr-t}|j||dD]}|jt|||d}|||<q||t|j<|S|S)NrM) r/r8r9r:r>_leafr?rIrJ)r4r7rMoutputZ local_dictrBZ field_outputr(r(r)rPhs   zAstScanning._leafcCr-r.)r/r0r1r2result_expressr3r(r(r)_refreshur6zAstScanning._refreshcCs||j|ddddS)Nz F)indentrM)Z _setup_global scan_importr<r(r(r)scan_ast{szAstScanning.scan_astparent_node_namecs|dur|S|rj|dS ddttjdtfdtdtffdd }t}t|jdkrat |d }|d krad d g|}t |d d t |d }|durYt |d |nt |d ||j |dD]U} t || } | gkryg|| <n| |rqht| trt| d krt| d tjr| d r|| d } | || <nQt| trt} | D].} || t| j} t| j}|dks|dks|dks|dkr|| vrg| |<| || q| || <nt| tjr|| }||| <n| || <t|jdkst|jdkrt|jdkrX| d krtj|| <| dkrXt|| dtrHg}|| dD]}|dd}||q1|j|d <n|| dd}|gj|d <t|jdkr|| d}t|tr~|D]}|dj|dd<qnn |j|| dd<d| kr| gkr| D] }t |t|jqj| | gkrt| jdkr|dkrj| qhtjtjtjtjiS)NrOrWelrXr,csj||dS)N)rMrX)rU)rYrXr4rMr(r) _scan_imports z-AstScanning.scan_import.._scan_import ImportFromlevelrrrr Importnamesaliasnamedecorator_listCallExpr)rW)rErPrr9r:rFr/rIrJr?joinsetattrr>rKr8r@lenappendr1r0 CLASS_NAMErar2extendrR IMPORT_KEYFROM_IMPORT_KEY DECORATOR_KEY EXPRESS_KEY)r4r7rMrXr[Zoutputsr]Z path_levelr"rBrCZ local_outZel_dictrYrarQZ item_nameitemZ local_nameZ final_dictr(rZr)rUs                         zAstScanning.scan_importcsjdtjdtfdddtdtffdd }dtjdtffdd }|j}||j}||j}|||fS) Nr7r,cSsVd\}}}t|jdkrt|d}t|d}t|d}t|jdkr't|d}||fS)N)NNN AttributevalueidrCName)rIrJr?)r7rqrrrCr(r(r)_get_attribute_items     z9AstScanning._parse_decorator.._get_attribute_itemnodescs`g}|D])}t|jdkr||jdfqt|jdkr&||jdfq||q|S)NStrConstant)rIrJrhsrq)ruresultr7rtr(r)_get_args_namesz4AstScanning._parse_decorator.._get_args_namecsg}|D]C}t|jdkrGt|d}t|jdkr&|t|d|jdfqt|jdkr:|t|d|jdfq|t|df|q|S)Nkeywordrqrvargrw)rIrJr?rhrxrq)ruryr7Zattribute_noderzr(r)_get_keyword_names& z7AstScanning._parse_decorator.._get_keyword_name)r9r:rAr@funcargskeywords)r4r7r{r~ functions args_list keyword_listr(rzr)_parse_decorators     zAstScanning._parse_decoratorcCsH|durdS|dkr tS|d}t|dkr|Stt|d|dS)Nrrrr)rsplitrgr?eval)r4Zkey_itemZ split_listr(r(r)_get_registry_values  zAstScanning._get_registry_value parsed_inputr!c Cs|\}}}t|dkr dS|dg}t|dkr#t|dkr#|tt|dkr4t|dkr4||t|dkrbt|dkrbd}|D]}|\} } } | tkrX|| | f|}qD|durb||t|dkrm|t|D]}|\} } } | tkr}| }qo|| | fqo|D]-}|dur|dq|ddur||dqt|tr||q|d |q|d| |d| |dfS)zformat registry information to a tuple indexer Return: tuple: (MODELS, Tasks.text-classification, Models.structbert) rNrr) REGISTER_MODULErgrhr GROUP_KEYremove MODULE_CLSr8rFrer) r4rr!rrrrQZremove_group_itemrokeyrarCr(r(r)_registry_indexer!sH               zAstScanning._registry_indexerrucCsrg}|D]2}t|jdkrqt|td}t|d}t|ddtkr"q||}|||}d|ur6||q|S)zparse the AST nodes of decorators object to registry indexer Args: nodes (list): list of AST decorator nodes Returns: list: list of registry indexer rcNrrC)rIrJr?rirrrrh)r4ruresultsr7r!rZ parse_outputrr(r(r)parse_decoratorsUs      zAstScanning.parse_decoratorscCs|t|ddd }|}Wdn1swYd|}t|}|j|dd}||t|t<||t |t <|t |t |S)Nrutf8)encodingrWFrO) rSopen readlinesrer9parserUrrmrnrj)r4filecodedatar7rQr(r(r) generate_astns   zAstScanning.generate_astr,N)T)TrW)rJ __module__ __qualname__r5objectboolr=r9r:rErrKrAr>rFrPrSrVrUrrrr@rrr(r(r(r)r+Ds2    j, 4 r+c@seZdZdddZ d dededefddZd d Zd edefd d Z d!ddZ d"ddZ ddZ ddZ ddZddZdeefddZeedfddZdS)#FilesAstScanningr,NcCst|_g|_g|_dSr.)r+ astScaner file_dirsrequirement_dirsr3r(r(r)r5}s zFilesAstScanning.__init__import_package current_pathc Cs|tdrtdd|ddddS|tdrV|d}|d}d}t|D] \}}|dkr<|}nq0|d| }|||ddd|dS|S)z Args: import_package (str): relative import or abs import current_path (str): path/to/current/file r/rrN.pyrW) startswithIGNORED_PACKAGESMODELSCOPE_PATHrer enumeraterj) r4rrZcurrent_path_listZimport_package_listr]rroZ abs_path_listr(r(r)_parse_import_paths&   z#FilesAstScanning._parse_import_pathcCsdSr.r()r4Zimport_abs_pathr(r(r)_traversal_importsz"FilesAstScanning._traversal_import scan_resultcCsg}|t|t|t|tt|D]\}}d|ddkr.d||<q|dd||<qt}|D]}tD] }| |rM| |qAq=tt|t|S)aparse import and from import dicts to a third party package list Args: scan_result (dict): including the import and from import result Returns: list: a list of package ignored 'modelscope' and relative path import rWrr) rjr@rkkeysrlrrsetrradd)r4rrQrroZignoredZignored_packager(r(r) parse_imports    zFilesAstScanning.parse_importFcCsbg|_|dus t|dkr|j||ddS|D]}tj||}tj|r.|j||dqdS)Nr include_init)rrg_traversal_filesospathreisdir)r4rZ check_sub_dirrroZsub_dirr(r(r)traversal_filess z FilesAstScanning.traversal_filescCst|}|D]Q}|jdkr|sq|jdr|jdks(|jds(|jdr)q|r6|j|j|dq|rH|jdrH|j |jq|rXd|jvrX|j |jqdS)Nz __init__.py__z.jsonz.mdrr requirement) rscandirrarendswithis_dirrris_filerrhr)r4rrZdir_listror(r(r)rs&   z!FilesAstScanning._traversal_filescCsz|j|}Wn8ty@}z,t|j}td|d|djd|djd|djdt |j d|d|d d}~ww| |}|t |fS) NzDuring ast indexing the file z', a related error excepted in the file z at line: z: "z" with error msg: "z: z'", please double check the origin file z- to see whether the file is correctly edited.) rr Exception traceback extract_tb __traceback__filenamelinenolinerIrJrrm)r4rrQedetail import_listr(r(r)_get_single_file_scan_results.  z-FilesAstScanning._get_single_file_scan_resultc CsJt}|D]}||tD]}t|t||tt||ti||<q q|Sr.)r/rm FILE_NAME_KEYrk MODULE_KEY)r4 forward_indexinverted_indexrror(r(r)_inverted_indexs   z FilesAstScanning._inverted_indexcCs,t}|D] \}}|t||t<q|Sr.)r/itemsrkr)r4r module_importrZ value_dictr(r(r)_module_importszFilesAstScanning._module_importcCs d|vr|d=d|vr|d=|S)N)Z OPTIMIZERSdefaultra)Z LR_SCHEDULERrrar()r4rr(r(r)_ignore_useless_keyss z%FilesAstScanning._ignore_useless_keyscCst}|dur ||_n|||td|d|t}|jD]'}||dd}|tj ddd}| |\} } t | t | t |i||<q#||} || } ||} t| t| i} tdt| d t|d | S) athe entry method of the ast scan method Args: target_file_list can override the dir and folders combine target_dir (str, optional): the absolute path of the target directory to be scanned. Defaults to None. target_folder (list, optional): the list of sub-folders to be scanned in the target folder. Defaults to SCAN_SUB_FOLDERS. Returns: dict: indexer of registry NzAST-Scanning the path "z!" with the following sub folders rrrrWzScanning done! A number of z. components indexed or updated! Time consumed rx)timerrloggerinfor/rfindreplaceospseprrmrkrrrr INDEX_KEYREQUIREMENT_KEYrg)r4Ztarget_file_list target_dirZtarget_foldersstartryrrr"rbrZinverted_index_with_resultsrrr(r(r)get_files_scan_resultss@      z'FilesAstScanning.get_files_scan_resultsc Csg|_|rt|tr||_n|||g}t}|jD]}tj|}|||||<qt dd|d}t | } | |fS)NcSst|t|Sr.rF)xyr(r(r)?sz2FilesAstScanning.files_mtime_md5..rW)rr8r@rr/rrgetmtimerhrhashlibrencode hexdigest) r4 target_pathZtarget_subfolder file_listrZfiles_mtime_dictromtimeZ result_strrr(r(r)files_mtime_md50s      z FilesAstScanning.files_mtime_md5rr.NFF)rJrrr5rFrrr/r@rrrrrrrrSCAN_SUB_FOLDERSrrr(r(r(r)r{s4      3robjr,cCsdtj|}|rtj|stj|ddt|d}||WddS1s+wYdS)a Write data to a given ``filepath`` with 'wb' mode. Note: ``write`` will create a directory if the directory of ``filepath`` does not exist. Args: obj (bytes): Data to be written. filepath (str or Path): Path to write data. T)exist_okwbN)rrdirnameexistsmakedirsrwrite)rrrfr(r(r) ensure_writeGs   "rFcCsdd|tD|t<ddlm}||t<tj|d\|t<|t<t |t <t |}|r8|t t}t||dd|tD|t<dS)NcSsi|] \}}t||qSr(r.0kvr(r(r) \sz_save_index..r __version__rcSi|] \}}t||qSr(r9 literal_evalrr(r(r)rg )rrmodelscope.versionr VERSION_KEY file_scannerrMD5_KEYFILES_MTIME_KEYras_posixMODELSCOPE_PATH_KEYjsondumpsrr#rr)r file_pathr with_templaterZ json_indexr(r(r) _save_indexZs      rcCsrt|d }|}Wdn1swY|r%|tt}t|}dd|t  D|t <|S)NrbcSrr(rrr(r(r)rurz_load_index..) rreaddecoderr#rr rloadsrr)rrrZ bytes_index wrapped_indexr(r(r) _load_indexms      rc Cs0|t}tt|t|}tt|t|}g}|D]}||vr/||||kr/||q||||t|dkrg}g}|tD]} |t| t|vrd|| ||t| tqH|D]} |t| =qg|D] } | |t vr~|t | =qqt |} |t | t|t  | t dS)Nr) r r@rrhrjrgrrrrr rupdate) rrZorigin_files_mtimeZ new_filesZ removed_filesZ updated_filesrZremove_index_keysZremove_requirement_keysrZ updated_indexr(r(r) _update_index|s<          rcCsFttttjd}d}ttt}|||kr!dSdS)Nz%Y-%m-%d %H:%M:%Si3TF)introundrstrptimerZ__release_datetime__ timestampnow)Zrelease_timestampZSECONDS_PER_YEARZcurrent_timestampr(r(r)__is_develop_models r!c Cs:td|}td|}tj||}d}|r#tdt|}|Strtd|tj |rkt |}tj |d\} } |}ddl m } |t| krT|t| ksZ|t| krjtd t|| t|||ntd |d t|}t|||td |td |tdt|td|St}|S)aget the index from scan results or cache Args: file_list: load indexer only from the file lists if provided, default as None force_rebuild: If set true, rebuild and load index, default as False, indexer_file_dir: The dir where the indexer file saved, default as INDEXER_FILE_DIR indexer_file: The indexer file name, default as INDEXER_FILE Returns: dict: the index information for all registered modules, including key: index, requirements, files last modified time, modelscope home path, version and md5, the detail is shown below example: { 'index': { ('MODELS', 'nlp', 'bert'):{ 'filepath' : 'path/to/the/registered/model', 'imports': ['os', 'torch', 'typing'] 'module': 'modelscope.models.nlp.bert' }, ... }, 'requirements': { 'modelscope.models.nlp.bert': ['os', 'torch', 'typing'], 'modelscope.models.nlp.structbert': ['os', 'torch', 'typing'], ... }, 'files_mtime' : { '/User/Path/To/Your/Modelscope/modelscope/preprocessors/nlp/text_generation_preprocessor.py': 16554565445, ... },'version': '0.2.3', 'md5': '8616924970fe6bc119d1562832625612', 'modelscope_path': '/User/Path/To/Your/Modelscope' } ZMODELSCOPE_CACHEZMODELSCOPE_INDEX_FILENz4Force rebuilding ast index from scanning every file!zLoading ast index from rrrz}Updating the files for the changes of local files, first time updating will take longer time! Please wait till updating done!zNo valid ast index found from z$, generating ast index from scratch!z,Loading done! Current index file version is z , with md5 z and a total number of z components indexed)rgetenvrrerrr rr!rrrrrr r rrrgrload_from_prebuilt) r force_rebuildZindexer_file_dirZ indexer_file cache_dirZ index_filerrrrrrr(r(r) load_indexsV $               r&cCsJ|durtjd}tj|t}tj|r t|dd}|St }|S)NrTr) presolveparentsrrre TEMPLATE_FILErrgenerate_ast_template)r local_pathrr(r(r)r#s  r#cCsTt|d}|durtjd}tj|t}t||ddtj |s(t d|S)Nr$rTr'z;The index file is not create correctly, please double check) r&r(r)r*rrrer+rrr)rr$rr-r(r(r)r,s  r,__main__r.rrr.)NT)Vr9rr$rZos.pathrrrrr functoolsrpathlibrtypingrrrrZmodelscope.metainforrr r r r r rrrrZmodelscope.utils.constantrrZmodelscope.utils.file_utilsrZmodelscope.utils.registryr__file__r(rHr)r*rZINDEXER_FILE_DIRrrrZ INDEXER_FILErmrnrlrkrrr r rr rrrirZ MODULE_NAMErr#r+r*rrr+rr bytesrFrrrrr!r&r#r,rJrprintr(r(r(r)s      4  9J  " Q