o *j/A@s6ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl mZddlmZddlmZddlmZddlmZddlmZmZddlmZmZdd lmZmZm Z dd l!m"Z"e"Z#d gZ$Gd d d e%Z&GdddeZ'ej(e j)ej*dGdd d eZ+   dddZ,dS)N) roi_align)Models) TorchModel)MODELS)FPNTrans)LayoutRobertaModelLayoutRobertaPreTrainedModel)TransformerDecoderTransformerDecoderLayer)ModeKeys ModelFileTasks) get_loggerVLDocForDocVLEmbeddingc@seZdZddZdS)GeoVLDocModelOutputscCs"||_||_||_||_||_dS)N text_featurestext_mm_featuresblock_vis_featuresblock_vis_mm_featuresimage_mm_features)selfrrrrrrj/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/multi_modal/vldoc/model.py__init__"s   zGeoVLDocModelOutputs.__init__N)__name__ __module__ __qualname__rrrrrr s rcsZeZdZd fdd ZdefddZ                  d dd ZZS) GeoVLDocModelFcst|||_||_t|jddr'|jjddkr!t||_n t||_nt||_t|jj dd|_ t ddg|_ t d|jj|_t|jj|jj|jjdd }t|d|_t|jj|jj|jjdd }t|d|_|dS) N architecturesrrF)Zimg_sizeZ inner_vitT)Z self_attn)superrconfighard_negtive_samplinggetattrrr text_encoderrZ image_sizevisual_encodernnZAdaptiveAvgPool2dpoolZLinearZ hidden_size vis_linearr Znum_attention_headsZintermediate_sizer cross_modal_textcross_modal_visualZ init_weights)rr#r$Zcross_modal_text_layerZcross_modal_visual_layer __class__rrr8s<      zGeoVLDocModel.__init__ ckpt_pathcCsFtj|dd}i}|D]\}}|dd}|||<q ||dS)NcpuZ map_locationgeo_vl_doc_model.)torchloaditemsreplaceZload_state_dict)rr/ state_dictZstate_dict_newkvrrrfrom_pretrainedZs   zGeoVLDocModel.from_pretrainedNc+ Ks|j\}}|dur |n|jj}||d<|jjddkr/|j|f|||| | | |||d |}n|j|f|||| | | |||d |}|dd\}}|j\}}tjd||jd|d ||}|||f}|j\}}}| |}tjd||jd|d || d}t ||fd||d j |d jd } |d jtjkrt|d  tj| tjd|d dd d }!|!j |d jd }!nt|d | tjd|d dd d }!|!dd||d}!||!}!|!| d}!||d dd}"||" d}"t |"|!fd}#t|df |j}$t |$|fd}%d|dk}&d|%dk}'|j|dd|#dd|&|'d}(|j|#dd|dd|'|&d})|(dd}(|)dd})|)ddddf}*t||(|!|*|)dS)NZ line_bboxrr) bboxattention_masktoken_type_ids position_ids head_mask inputs_embedsoutput_attentionsoutput_hidden_states return_dict)devicer Zfeat_ms)dtypeg@@)Z spatial_scaler!)ZtgtZmemoryZtgt_key_padding_maskZmemory_key_padding_maskr)shaper#Zuse_return_dictrr&r4ZarangerFZreshapeexpandr'Z unsqueezecattorIZfloat16rZfloat32sizeZsqueezer*r)Zonesr+Z transposer,r)+r input_idsimager<bbox_4p_normalizedr=first_token_idxesfirst_token_idxes_maskr>r?r@rAencoder_hidden_statesencoder_attention_maskpast_key_values use_cacherBrCrDkwargsZ batch_sizeZseq_lenZoutputsZsequence_outputZ pooled_output_Z num_firstZ B_batch_dimZ feature_bboxZ block_numZ visual_outZ batch_idxsZbatch_idx_with_bboxZblk_vis_featuresZfull_img_featuresZvis_inpsZ glb_feat_attnZvis_maskZnew_attention_maskZ new_vis_maskZ text_mm_featZ vis_mm_featrrrrforwardbs                     zGeoVLDocModel.forward)FNNNNNNNNNNNNNNNNNN)rrrrstrr;rZ __classcell__rrr-rr6s," r) module_namecsTeZdZdZdeffdd Z                  dddZZS) rz Generate multi-modal document embeddings in segment-level and token-level. Args: model_dir: the path in model hub, e.g., 'damo/multi-modal_convnext-roberta-base_vldoc-embedding' model_dirc sFtj|d|i|ddlm}tj|d}td |tj |s(J| ||_ t |j |_tj|tj}tj |sDJ|j|td |ddlm}tj|tj}|||_tjrwd ttjd dnd |_tjr|j|jtd ttjd ddS|jtd dS) Nr_r)LayoutRobertaConfigz config.jsonzLoading config file from {}zLoading model from {})VLDocXLMTokenizerzcuda:{}Z LOCAL_RANKr0z%Use GPU {} for finetuning & inferencez"Use CPU for finetuning & inference)r"r;modelscope.models.multi_modal.vldoc.modeling_layout_robertar`ospathjoinloggerinfoformatexistsZfrom_json_filer#r doc_modelr ZTORCH_MODEL_FILEr;Z0modelscope.models.multi_modal.vldoc.tokenizationraZTOKENIZER_FOLDER tokenizerr4cudaZ is_availableintenvirongetrFrMfloat) rr_argsrXr`Zmodel_cfg_pathZ model_pathraZtokenizer_pathr-rrr s6        zVLDocForDocVLEmbedding.__init__NcKs|jdid|d|d|d|d|d|d|d|d | d | d | d | d | d|d|d|d|d||}t|j|jdS)a Args: - input_ids: :math:`(B, T, E)`, the input tokens, where B is the batch size, T is the max token size, E is the embedding dimension. - image: :math:`(B, C, H, W)`, normalized images. - bbox: :math:`(B, T, 4)`, segment boxes denoted by top-left and bottom-right vertexes whose values are normalized to [0, 1000). - bbox_4p_normalized: :math:`(B, T, 8)`, word boxes denoted by 4 vertexes, whose values are normalized to [0, 1). - attention_mask: :math:`(B, T)`, mask for input tokens, where 0 means masked. - first_token_idxes: :math:`(B, S)`, indexes of the corresponding first tokens of all segments, where S is the max segment size. - first_token_idxes_mask: :math:`(B, S)`, mask for segments, where 0 means masked. Optional: - line_rank_id: :math:`(B, T)`, orders of segments. - line_rank_inner_id: :math:`(B, T)`, BIE-like tags. To be more specific, please refer to the class `TextLayoutSerializer` in `modelscope/models/multi_modal/vldoc/processing.py`. rOrPr<rQr=rRrSr>r?r@rArTrUrVrWrBrCrD)Z img_embeddingZtext_embeddingNr)rjdictrr)rrOrPr<rQr=rRrSr>r?r@rArTrUrVrWrBrCrDrXZ vldoc_outputsrrrrZ*sT )     zVLDocForDocVLEmbedding.forwardr[)rrr__doc__r\rrZr]rrr-rrs,"robertac sdur tj|ddg}g}t}|dkrFtt|D]&}||} d} | dr7| dd} t | } | rE| ||| | qt ||D] \} }  | | <qKgggt dddurn_dfdd d} t|d std d Drd } || dtdkrtd|jjtdkrtd|jjtdkrtd|jjd|S)Nr0r1rtzroberta.zgeo_vl_doc_model.text_encoder. _metadatar3c shdurin |ddi}|||d|jD]\}}|dur1|||dq dS)NrGT.)roZ_load_from_state_dictZ_modulesr6)moduleprefixZlocal_metadatanamechild error_msgsr5metadataZ missing_keysr8Zunexpected_keysrrr5s  z$init_pretrained_weight..loadZgeo_vl_doc_modelcss|]}|dVqdS)r2N) startswith).0srrr s  z)init_pretrained_weight..r2)rxrz7Weights of {} not initialized from pretrained model: {}z0Weights from pretrained model not used in {}: {}z*Error(s) in loading state_dict for {}: {}z )r3)r4r5listkeysrangelenr~r7copydeepcopyappendzippopr%ruhasattranyrfrgrhr.r RuntimeErrorre) modelZpretrained_model_pathr8 cache_dirZ init_backboneZold_keysZnew_keysZstate_dict_keysikeyZnew_keyold_keyZ start_prefixrr{rinit_pretrained_weightnsf          r)NNrt)-rloggingmathrcresysjsonr4Ztorch.distributed distributeddistZtorch.nnr(Ztorchvision.opsrZmodelscope.metainforZmodelscope.modelsrZmodelscope.models.builderrZ2modelscope.models.multi_modal.vldoc.conv_fpn_transrrbrrZ5modelscope.models.multi_modal.vldoc.transformer_localr r Zmodelscope.utils.constantr r r Zmodelscope.utils.loggerrrf__all__objectrrZregister_moduleZdocument_vl_embeddingZvldocrrrrrrs<        Jq