o "jF@sddlZddlZddlZddlmZddlZddlmZddl Z ddl m Z ddl m Z ddl mZddlmZddlmZdd lmZgd ZeeZeejejd d ZeZeee ed Z!dZ"dZ#e j$j%j&e j$j%j'gZ(dgZ)d8ddZ*ddZ+ddZ,d9ddZ-ddZ.d:dd Z/d!d"Z0d#d$Z1d%d&Z2d'd(Z3d)d*Z4d+d,Z5d-d.Z6d/d0Z7d1d2Z8 3 d;d4d5Z9d6d7Z:dS)<N) OrderedDict) text_format)base)core)Program) framework_pb2)draw_block_graphviz)io_utils) load_program save_programprogram_type_transcheck_saved_vars_try_dump parse_programcheck_pruned_program_varsgraphvizz)%(asctime)s - %(levelname)s - %(message)s)fmtzvars_persistable.logz vars_all.logzops.logZ lookup_tableFcCs|rt|St|SN)load_program_textload_program_binary)model_filenameis_textrh/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/incubate/distributed/fleet/utils.pyr ;sr cCs<t|d }|}Wdn1swYt|S)z$load program from binary string filerbN)openreadrparse_from_string)rfZprogram_desc_strrrrrAs   rcCsTt|d }|}Wdn1swYt}t||t|S)z*load program from human-readable text filerN) rrrZ ProgramDescrZMergerrZSerializeToString)rrZprogram_desc_textZ prog_descrrrrHs   r __model__cCs|r!t|d}|t|WddS1swYdSt|d}||jWddS1s:wYdS)Nwwb)rwritestrdescZserialize_to_string)programrrrrrrr Rs " "r c Csd}dd|D}t|}t|}td||D]T}||}|jtvr+|Sz ||}Wnt yR}zt d|t |WYd}~qd}~ww|j |j ks_|j |j krqt d ||j |j |j |j d}q|S)NTcSs g|] }t|r|j|fqSr)r is_persistablename.0vrrr ^s z-check_pruned_program_vars..z$persistable vars in pruned program: z>not find variable '%s' in train program. please check pruning.zbvariable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}F) list_varsrlistloggerinfotypefeed_fetch_type_list global_blockvar ValueErrorerrorshapedtypeformat) Z train_progZ pruned_progZis_matchZ pruned_varsZpruned_vars_namevar_namer3Ztrain_prog_varerrrr[sL      rdebugcCsbtj||d}tj||d}t||ddd|d|g}tj|tjtjtjd}|dS)Nz.dotz.pdf)pathdotz-Tpdfz-o)stdinstdoutstderr)osr=joinr subprocessPopenPIPEwait)block output_dirfilenameZdot_pathZpdf_pathcmdprrrrs  rcCsFttj|||}|r|dn|d}t|tj||d||S)Nz.binz.pbtxt)r rBr=rCr )Zprog_dirZprog_fnrprogZ prog_out_fnrrrr sr cCs |jdd|giid|iddS)NsaveX file_pathr0ZinputsZoutputsattrsZ append_oprHr3r=rrrappend_save_ops rVcCs |jdid|gid|iddS)NloadZOutrQrRrTrUrrrappend_load_ops  rXc Cst}t}t|}t|}t|#tjj|||d} t | | ||j ||igdWddS1s.) isinstancer-tuplerstripsplitlenappendnparrayZreshape) batch_sizefndimrar6_tempxlinefieldstmprrrreaders(      r}cCs0g}t|D]\}}|t||||q|Sr) enumeraterrr})rufeeded_vars_dimsfeeded_vars_filelist batch_feedirvrrrfeed_gensrc% st}t|} tj} t| ]|rt|||}tjj j || ||d\} } dd|D} |D]=}t |j }|dusIJd|j t|j}|j | vs]J|j d| |j }||krqtd||j |q4dd| D}| std |std | }| }jdur| jkrtd | jj}}g}t|jD]\}}|jd |jd kr||q|dddD]}||q|j dur"||j kr"td||j fdd|j D}}g}t|jD]\}}|jd |jdkr||q|dddD]}||qt!dd|D}gt"jt"j#krCt"j$ksFJJt%t"jD]C}&j|}t'j#|t(t)fskj#|f}nt)j#|}|j#|<|jdd}||krtdj|||qMj*st+dt%t"jD]`}&j|}|j,dkrЈtjtj--t)|gt(j#|j$|dq|j,dkrtjtj--t)|gt(j#|j$|d}t.|dg|g|qtd| j/fddt|D||d}n7t+dj*dfddt%t"jD} tj0| |d}!t1|j#j*}"|"g}#| j/|!2|#||d}t|D]\}}$t+d |$j t+d!||qP|WdS1suwYdS)"N)rZparams_filenamecSsi|] }|jt|jqSr)r'rnr$r6)r)each_varrrr sz'try_load_model_vars..zcan't not find var: zMUST in var listzShape not matching: the Program requires a parameter with a shape of ({}), while the loaded parameter (namely [ {} ]) has a shape of ({}).cSg|]}|jqSrr'r(rrrr+z'try_load_model_vars..zno feed targets in program.zno fetch targets in program.zQfeed vars in program and config are diff: feed in program: {}. feed in config {}.FrZzTfetch vars in program and config are diff: fetch in program: {}. fetch in config {}.csg|] }|qSr)r2r3r)r)inference_programrrr+6s fetchcss|]}|jdkVqdS)rN) lod_levelr(rrr Esz&try_load_model_vars..rMzSfeed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}zgenerate random feed vars.r)r7zOvars with lod_level >= 2 is not supported now in this infer program check tool.csi|] \}}||qSrr)r)rr') feed_tensorsrrrs)rZr[ return_numpyzload feed vars from files: .cs g|] }j|qSr)r2r3feeded_vars_namesr) feed_configrrrr+s )Z feed_listrezfetch_targets name: %szfetch_targets: )3rr\r]rZScopeZ scope_guardr r_r`ioZload_inference_modelZ global_scopeZfind_varr'rsrtZ get_tensorr6get RuntimeErrorr8r.warningrr2r~opsr$Z set_is_targetr0rrZ _remove_opZfetch_vars_namesallrqrZfeeded_vars_typesranger3rmr-rnrr/rrandomZcreate_lod_tensorrbZ DataFeederrrZ)%dump_dir dump_prog_fnis_text_dump_programrur fetch_config save_filename saved_paramsrerfscopeZfeed_target_namesZ fetch_targetsZorig_para_shaperZvar_tempZ new_shapeZ orig_shapeZfetch_targets_namesr[Zfeed_name_listr2Zneed_to_remove_op_indexropindexrr3Z tensor_shapeZ var_shapetresultsZ feed_varsZfeederrslotsr*r)rrrrtry_load_model_varss\                                 &rcCsNt}|jD]}|jtvr$|j|vr$td|jd||jqdS)Nzfind op type 'z@' in program, please check if your program is pruned correctly !)setr2rr0not_expected_op_typesr.radd)rNZ op_types_setrrrrcheck_not_expected_opss  rrMc Cs^ttj|||}dd|D}tddd|Dt|t||||||||S)NcSg|] }t|r|qSrr r&r(rrrr+  z-check_saved_vars_try_dump..z"persistable vars in dump program: cSrrrr(rrrr+r) r rBr=rCr,r.r/rr) rrrrrrurZ dump_progrrrrr s( r c Csi}dd|D}dd|D|d<ttj|td }|d|dD]}|t||dq(Wdn1sAwYt|}dd|D|d <ttj|t d }|d |d D]}|t||dqiWdn1swY| j }d d|D|d <ttj|t d!}|d |d D]}|t||dqWddS1swYdS)NcSrrrr(rrrr+rz!parse_program..cSs:g|]}t|jt|jt|jt|jt|jdqS))r'r6rr7r0)r#r'r6intrr7r0r(rrrr+spersistable_varsr zpersistable vars:  cSsRg|]%}|jtvrt|jt|jt|jt|jdn t|jt|jdqS))r'r6rr7)r'r0)r0r1r#r'r6rrr7r(rrrr+s all_varsz all vars: cSs&g|]}|jt|jt|jdqS))r0input_arg_namesoutput_arg_names)r0r#rr)r)rrrrr+srzops: ) r,rrBr=rCpersistable_vars_out_fnr"r#r-all_vars_out_fnr2r ops_out_fn) r%rIoutputrrr3rrrrrrrsH              "r)F)rF)r;r<)rMN);loggingrBrD collectionsrnumpyrsZgoogle.protobufrr_rZ paddle.baserZpaddle.base.frameworkrZpaddle.base.protorZ*paddle.distributed.fleet.base.util_factoryrZpaddle.frameworkr __all__ getLogger__name__r.setLevelINFO Formatter formatter StreamHandlerch setFormatter addHandlerrrrZVarDescZVarTypeZFEED_MINIBATCHZ FETCH_LISTr1rr rrr rrr rVrXrhrir}rrrr rrrrrs^               +   N !