o *j@sDdZddlZddlZddlmZddlZddlmmZ ddl Zddl Z ddlm Z m Z mZddlmZddl mZmZddlmZddlmZmZmZmZdd lmZmZmZdd lmZdd lm Z dd l!m"Z"m#Z#dd l$m%Z%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.e j /e 0Z1dZ2dZ3dZ4ddZ5ddZ6Gdddej7Z8Gdddej7Z9Gdddej7Z:Gdddej7Z;Gd d!d!ej7ZGd&d'd'ej7Z?Gd(d)d)ej7Z@Gd*d+d+ej7ZAGd,d-d-ej7ZBGd.d/d/ej7ZCGd0d1d1ej7ZDGd2d3d3ej7ZEGd4d5d5ej7ZFGd6d7d7ej7ZGGd8d9d9eZHd:ZId;ZJedd>eHZKGd?d@d@eHZLedAeIGdBdCdCeHZMGdDdEdEeHZNGdFdGdGeZOGdHdIdIeOZPGdJdKdKeOZQGdLdMdMeOZRGdNdOdOeZSGdPdQdQeSZTGdRdSdSeSZUdS)TzPyTorch MPLUG model. N)Tuple)Tensordevicenn)CrossEntropyLoss) BertConfig BertTokenizer)ACT2FN)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardreplace_return_docstrings))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentions)PreTrainedModel)logging) HiTeAConfig MPlugConfig)MViTv2MViTv2_Base_config) TextGenerator) ModelFile)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layerz config.yamlrrc Cszddl}ddl}ddl}Wn tytdwtj|}t d ||j |}g}g} |D]\} } t d | | |j || } || | | q6t|| D]\} } | d} tdd| Drxt d d| qZ|} | D]|}|d |r|d |}n|g}|dd ks|dd krt| d} nH|ddks|ddkrt| d} n6|ddkrt| d} n*|ddkrt| d} nz t| |d} Wntyt d d| Yq|wt|dkrt|d}| |} q||dddkrt| d} n |d kr|| } z| j| jks'Jd| jd| jdWntyA}z |j| j| jf7_d}~wwt d | t| | _qZ|S)z'Load tf checkpoints in a pytorch model.rNzLoading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z(Converting TensorFlow checkpoint from {}z"Loading TF weight {} with shape {}/css|]}|dvVqdS))Zadam_vZadam_mZAdamWeightDecayOptimizerZAdamWeightDecayOptimizer_1Z global_stepN).0nrrs/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/multi_modal/mplug/modeling_mplug.py Xsz*load_tf_weights_in_bert..z Skipping {}z [A-Za-z]+_\d+z_(\d+)ZkernelgammaweightZ output_biasbetabiasZoutput_weightsZsquadZ classifieriZ _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight {})renumpyZ tensorflow ImportErrorloggererrorospathabspathinfoformattrainZlist_variablesZ load_variableappendzipsplitanyjoin fullmatchgetattrAttributeErrorlenint transposeshapeAssertionErrorargstorchZ from_numpydata)modelconfigZtf_checkpoint_pathr(nptfZtf_pathZ init_varsnamesZarraysnamer>arraypointerZm_nameZ scope_namesnumerrr load_tf_weights_in_bert;s                     rMcCsB|jtjkrt|rt|jjd}tj|| |d}|S)Ni)minmax)dtyperAZfloat16isinfr6ZfinforOclamp)tensorZ clamp_valuerrr clamp_infsrTcs4eZdZdZfddZ     dddZZS) BertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|_|dt|jdt|dd|_||_dS)N)Z padding_idxeps position_ids)r'position_embedding_typeabsolute)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsZtype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrAarangeexpandr9rZrDselfrD __class__rr r]s2  zBertEmbeddings.__init__Nrc Cs|dur |}n|dd}|d}|dur&|jdd|||f}|dur5tj|tj|jjd}|dur>||}||}||} |jdkrU| |} | | 7} | | } | | } | S)NrYr'rPrr[) sizerXrAzeroslongrrbrerZrdrfrj) ro input_idstoken_type_idsrX inputs_embedspast_key_values_length input_shape seq_lengthre embeddingsrdrrr forwards0         zBertEmbeddings.forward)NNNNr)__name__ __module__ __qualname____doc__r]r} __classcell__rrrpr rUs rUcsZeZdZfddZddZddZddZd d Zd d Z dddZ Z S)BertSelfAttentioncs"t||_|j|jdkrt|dstd|j|jf|j|_t|j|j|_|j|j|_ t |j|j |_ |rQt |j |j |_t |j |j |_nt |j|j |_t |j|j |_t |j|_t|dd|_|jdks{|jdkr|j|_t d|jd |j|_d |_dS) NrZembedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)rZr[ relative_keyrelative_key_queryr&r'F)r\r]rDr`num_attention_headshasattr ValueErrorr<attention_head_size all_head_sizerLinearqueryZ encoder_widthkeyvaluerhZattention_probs_dropout_probrjr9rZrcr^distance_embeddingsave_attentionrorDis_cross_attentionrprr r]sB    zBertSelfAttention.__init__cC ||_dSNattn_gradients)rorrrr save_attn_gradients z%BertSelfAttention.save_attn_gradientscC|jSrrrorrr get_attn_gradientsz$BertSelfAttention.get_attn_gradientscCrr attention_map)rorrrr save_attention_maprz$BertSelfAttention.save_attention_mapcCrrrrrrr get_attention_maprz#BertSelfAttention.get_attention_mapcCs6|dd|j|jf}|j|}|ddddS)NrYrr&r')rsrrviewpermute)roxZ new_x_shaperrr transpose_for_scoress  z&BertSelfAttention.transpose_for_scoresNFcCs||}|du} | r|||} |||} |}n;|durI|||} |||} tj|d| gdd} tj|d| gdd} n|||} |||} ||} | | f}t| | dd} t| } |j dksz|j dkr| d}tj |tj |j d dd}tj |tj |j d dd}||}|||jd}|j| jd }|j dkrtd | |}| |} n|j dkrtd | |}td | |}| ||} | t|j} |dur| |} tjdd| }| r|jr||||j||}|dur||}t|| }|dddd }| dd|j f}|j|}|r;||fn|f}||f}|S)Nrr&dimr'rYrrrrrPzbhld,lrd->bhlrzbhrd,lrd->bhlrr)!rrrrrAcatmatmulr=rTrZrsrlrurrrrctorPZeinsummathsqrtrrZSoftmaxrr register_hookrrjr contiguousr)ro hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsZmixed_query_layerrZ key_layerZ value_layerZ query_layerZattention_scoresr{Zposition_ids_lZposition_ids_rZdistancepositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZattention_probs_droppedZ context_layerZnew_context_layer_shapeoutputsrrr r}s                zBertSelfAttention.forwardNNNNNF) r~rrr]rrrrrr}rrrrpr rs " rc$eZdZfddZddZZS)BertSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNrV) r\r]rrr`denserfrgrhrirjrnrprr r]j zBertSelfOutput.__init__cCs&||}||}|||}|Sr)rrjrfrorZ input_tensorrrr r}qs  zBertSelfOutput.forwardr~rrr]r}rrrrpr rh rcs<eZdZd fdd ZddZ      d ddZZS) BertAttentionFcs,tt|||_t||_t|_dSr)r\r]rroroutputset pruned_headsrrprr r]zs    zBertAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrr'r)r;rrorrrrrrrrrrunion)roheadsindexrrr prune_headss   zBertAttention.prune_headsNc Cs<||||||||}||d|} | f|dd} | S)Nrr')ror) rorrrrrrrZ self_outputsattention_outputrrrr r}s   zBertAttention.forward)Fr)r~rrr]rr}rrrrpr rxsrcr)BertIntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr) r\r]rrr`intermediate_sizer isinstance hidden_actstrr intermediate_act_fnrnrprr r]s   zBertIntermediate.__init__cC||}||}|Sr)rrrorrrr r}  zBertIntermediate.forwardrrrrpr rs rcr) BertOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) r\r]rrrr`rrfrgrhrirjrnrprr r]rzBertOutput.__init__cCs6||}t|}||}t|}|||}|Sr)rrTrjrfrrrr r}s  zBertOutput.forwardrrrrpr rrrcs<eZdZfddZ       d ddZddZZS) FusionLayercs^t||_t|jdd|_|j|_d|_t||_t|dd|_ t ||_ t ||_ dS)N stride_layerdr'Tr)r\r]rDr9rchunk_size_feed_forward seq_len_dimr attentioncrossattentionr intermediaterrrorD layer_numrprr r]s   zFusionLayer.__init__NFc Cs2|dur |ddnd} |dks||jdkrO|j||||| d} | d} | dd} | d} |dus7Jd|j| |||||d}|d} | |dd} n/|dkr~||jdkr~|jt||gdt||gd||| d} | d} | dd} | d} t|j|j|j| }|f| } | | d| df} | S) Nr&rrrr'rY>encoder_hidden_states must be given for cross-attention layersrr) rrrrArrfeed_forward_chunkrr)rorrrrrZ layer_numsrrself_attn_past_key_valueself_attention_outputsrrpresent_key_valuecross_attention_outputs layer_outputrrr r}sb    zFusionLayer.forwardcC||}|||}|SrrrrorZintermediate_outputrrrr r  zFusionLayer.feed_forward_chunk)NNNNNNFr~rrr]r}rrrrrpr rs  =rcs:eZdZfddZ      d ddZddZZS) BertLayercsdt||_|j|_d|_t||_t|jdd|_|jr&t|dd|_ t ||_ t ||_ dS)Nr'add_cross_attentionFTr)r\r]rDrrrrr9has_cross_attentionrrrrrrrprr r]!s   zBertLayer.__init__NFc Cs*|dur |ddnd}|j|||||d} | d} | dd} | d} |jrz|dus/Jdt|tkrc|j| ||||j|jjt|||j|jjt||d} | d} | | dd} n|j| |||||d} | d} | | dd} t |j |j |j | }|f| } | | d| df} | S)Nr&rrr'rYrr) rrtypelistrrrDZ fusion_layerr;rrrr)rorrrrrrrrrrrrrrrrr r}0sv     zBertLayer.forwardcCrrrrrrr rsrzBertLayer.feed_forward_chunkrrrrrpr rs  Crc8eZdZfddZ         dddZZS) FusionEncodercsHt|_tfddtjD|_tdjj |_ dS)Ncg|]}t|qSr)rrirDrr z*FusionEncoder.__init__..r) r\r]rDr ModuleListrangenum_hidden_layerslayerrOZ fusion_layers start_layerrnrprr r]{s   zFusionEncoder.__init__NFTc  s| rdnd} r dnd} |rdnd} t|jdd|_|jd}|jd}t|jt|jD]}|j|}| r;| |f} |durC||nd}|durM||ndt|jddr||jr||rbt dd}fdd }t j j |||||||||j}n||||||||j}|d }|r| |d f7} r| |df} |jd||krt |||fd\}}||7}q-| r| |f} ||gS) Nrrrr'gradient_checkpointingFh`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting `use_cache=False`...cfdd}|S)Nctg|RSrtupleZinputsmodulerrrr custom_forwardzLFusionEncoder.forward..create_custom_forward..custom_forwardrr r rr r create_custom_forwardz4FusionEncoder.forward..create_custom_forwardrrY)r9rDrr>rrr;rtrainingr+warningrAutils checkpointr5)rorrrrrpast_key_values use_cacheroutput_hidden_states return_dictall_hidden_statesall_self_attentionsnext_decoder_cacheZ image_length text_lengthr layer_modulelayer_head_maskr layer_outputsZencoder_hidden_states_newrrr r}s~          zFusionEncoder.forward NNNNNNFFTrrrrpr rys  rcr) BertEncodercs4t|_tfddtjD|_dS)Ncrr)rrrrr rrz(BertEncoder.__init__..)r\r]rDrrrrrrnrprr r]s  zBertEncoder.__init__NFTc  s^| rdnd} r dnd} r|jjrdnd} |rdnd}tt|jD]k}|j|}| r1| |f} |dur9||nd}|durC||ndt|jddrn|jrn|rXtdd}fdd}t j j |||||||}n ||||||}|d}|r||df7}r| |d f} q#| r| |f} | st d d ||| | | fDSt ||| | | d S) NrrFrcr)Ncrrrrrrr r r zJBertEncoder.forward..create_custom_forward..custom_forwardrr rr r rrz2BertEncoder.forward..create_custom_forwardrrYr'css|] }|dur|VqdSrr)rvrrr r!!s z&BertEncoder.forward..)last_hidden_staterr attentionscross_attentions)rDrrr;rr9rr+rrArrrr)rorrrrrrrrrrrrZall_cross_attentionsrrrrrrrrr r}s       zBertEncoder.forwardrrrrrpr r s r cr) BertPoolercs*tt|j|j|_t|_dSr)r\r]rrr`rZTanh activationrnrprr r]3s zBertPooler.__init__cCs(|dddf}||}||}|S)Nr)rr&)rorZfirst_token_tensor pooled_outputrrr r}8s  zBertPooler.forwardrrrrpr r%1 r%cr)BertPredictionHeadTransformcsVtt|j|j|_t|jtrt |j|_ n|j|_ tj |j|j d|_ dSr) r\r]rrr`rrrrr transform_act_fnrfrgrnrprr r]Cs   z$BertPredictionHeadTransform.__init__cCs"||}||}||}|Sr)rr*rfrrrr r}Ms   z#BertPredictionHeadTransform.forwardrrrrpr r)As  r)cr)BertLMPredictionHeadcsLtt||_tj|j|jdd|_t t |j|_ |j |j_ dS)NF)r%) r\r]r) transformrrr`r_decoder ParameterrArtr%rnrprr r]Vs   zBertLMPredictionHead.__init__cCrr)r,r-rrrr r}drzBertLMPredictionHead.forwardrrrrpr r+Ts r+cr)BertOnlyMLMHeadcstt||_dSr)r\r]r+ predictionsrnrprr r]ls zBertOnlyMLMHead.__init__cC||}|Sr)r0)rosequence_outputprediction_scoresrrr r}p zBertOnlyMLMHead.forwardrrrrpr r/j r/cr)BertOnlyNSPHeadcstt|jd|_dSNr&)r\r]rrr`seq_relationshiprnrprr r]ws zBertOnlyNSPHead.__init__cCr1r)r8)ror'seq_relationship_scorerrr r}{r4zBertOnlyNSPHead.forwardrrrrpr r6ur5r6cr)BertPreTrainingHeadscs(tt||_t|jd|_dSr7)r\r]r+r0rrr`r8rnrprr r]s  zBertPreTrainingHeads.__init__cCs||}||}||fSr)r0r8)ror2r'r3r9rrr r}s  zBertPreTrainingHeads.forwardrrrrpr r:r(r:c@s*eZdZdZeZeZdZdgZ ddZ dS)BertPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. bertrXcCs~t|tjtjfr|jjjd|jjdnt|tj r(|j j |jj dt|tjr;|j dur=|j j dSdSdS)z Initialize the weights g)meanZstd?N) rrrr^r#rBZnormal_rDZinitializer_rangerfr%Zzero_Zfill_)ror rrr _init_weightss  z!BertPreTrainedModel._init_weightsN) r~rrrr config_classrMZload_tf_weightsZbase_model_prefix_keys_to_ignore_on_load_missingr?rrrr r;s r;a This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch `torch.nn.Module `__ subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. a Args: input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using :class:`~transformers.BertTokenizer`. See :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for details. `What are input IDs? <../glossary.html#input-ids>`__ attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. `What are attention masks? <../glossary.html#attention-mask>`__ token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: - 0 corresponds to a `sentence A` token, - 1 corresponds to a `sentence B` token. `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. `What are position IDs? <../glossary.html#position-ids>`_ head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert :obj:`input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (:obj:`bool`, `optional`): Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned tensors for more detail. output_hidden_states (:obj:`bool`, `optional`): Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for more detail. return_dict (:obj:`bool`, `optional`): Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. z^The bare Bert Model transformer outputting raw hidden-states without any specific head on top.c eZdZdZdfdd ZddZddZd d Zee d e e d e ed dedeedededef ddZ               dddZZS) BertModel= The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in `Attention is all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an input to the forward pass. TcsDt|||_t||_t||_|rt|nd|_| dSr) r\r]rDrUr|r encoderr%pooler init_weightsrorDadd_pooling_layerrprr r]s    zBertModel.__init__cC|jjSrr|rbrrrr get_input_embeddingszBertModel.get_input_embeddingscC ||j_dSrrKrorrrr set_input_embeddings zBertModel.set_input_embeddingscC*|D]\}}|jj|j|qdSz Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel NitemsrErrrroZheads_to_prunerrrrr _prune_headszBertModel._prune_headsbatch_size, sequence_lengthbert-base-uncasedZprocessor_classr output_typer@rrzr is_decoderreturnc CX|dkr|dddddddf}n|dkr|r|\}}tj||d}|ddddf||d|ddddfk} | |j} | jd|jdkrl|jd| jd} tjtj||| f|| jd| gdd} | dddddddf|ddddddf}n|ddddddf}n t d ||j|j|jd }d |d }|S aW Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: attention_mask (:obj:`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. input_shape (:obj:`Tuple[int]`): The shape of the input to the model. device: (:obj:`torch.device`): The device of the input to the model. Returns: :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`. rNr&rr')rrPrY)ZaxiszAWrong shape for input_ids (shape {}) or attention_mask (shape {})rr>g rrArlrepeatrrPr>ronesrr1 rorrzrr]extended_attention_mask batch_sizer{Zseq_idsZ causal_maskZprefix_seq_lenrrr get_extended_attention_masks\       z%BertModel.get_extended_attention_maskNFc s| dur| njj} | dur| njj} |dur|njj}|r+| dur&| njj} nd} |dur9|dur9td|durI|}|\}}|j}n,|dur]|dd}|\}}|j}n|durq|dd}|\}}|j}ntd| dur| ddjdnd}|durt j |||f|d}|durt j |t j |d } ||||}|durt|tkr|d\}}}n|\}}}||f}t| tkrׇfd d | D}n| durt j ||d} | }n| }nd}|jj}|durj|||||d }n|}j|||||| | | | |d }|d}jdur(|nd}|s7||f|ddSt|||j|j|j|jdS)~ encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. use_cache (:obj:`bool`, `optional`): If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up decoding (see :obj:`past_key_values`). NFDYou cannot specify both input_ids and inputs_embeds at the same timerYGYou have to specify either input_ids or inputs_embeds or encoder_embedsrr&rarrcg|]}|qSrinvert_attention_maskrmaskrrr rz%BertModel.forward..rvrXrwrxry rrrrrrrrrr'r"Z pooler_outputrrr#r$rDrruse_return_dictrrrsrr>rArdrtrurhrrrnZ get_head_maskrr|rErFrrrr#r$rorvrrwrXrrxencoder_embedsrrrrrrrr]rzrgr{rryrfZencoder_batch_sizeZencoder_sequence_length_Zencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputsr2r'rrr r}Ts(         zBertModel.forwardTNNNNNNNNNNNNNNFr~rrrr]rLrPrWr BERT_INPUTS_DOCSTRINGr1r _TOKENIZER_FOR_DOCr_CONFIG_FOR_DOCrrr<rboolrhr}rrrrpr rCsP  HrCc rB) FusionModelrDTcs:t|||_t||_|rt|nd|_|dSr)r\r]rDrrEr%rFrGrHrprr r]s   zFusionModel.__init__cCrJrrKrrrr rL rMz FusionModel.get_input_embeddingscCrNrrKrOrrr rP rQz FusionModel.set_input_embeddingscCrRrSrTrVrrr rWrXzFusionModel._prune_headsrYrZr[rrzrr]r^c Cr_r`rbrerrr rhs\       z'FusionModel.get_extended_attention_maskNFc s~| dur| njj} | dur| njj} |dur|njj}|r+| dur&| njj} nd} |dur9|dur9td|durI|}|\}}|j}n,|dur]|dd}|\}}|j}n|durq|dd}|\}}|j}ntd| dur| ddjdnd}|durt j |||f|d}|durt j |t j |d } ||||}|durt|tkr|d\}}}n|\}}}||f}t| tkrׇfd d | D}n| durt j ||d} | }n| }nd}|jj}|durj|||||d }n|}j|||||| | | | |d }|\}}jdur(|nd}|s1||gSt|||j|j|j|jdS)riNFrjrYrkrr&rarrcrlrrmrorrr rrqz'FusionModel.forward..rrrsrtrurwrrr r}gs&         zFusionModel.forwardrzr{r|rrrpr rsP GrzGBert Model with a `language modeling` head on top for CLM fine-tuning. cseZdZdgZddgZfddZddZdd Zee d e e e d    dddZ dddZddZZS)BertLMHeadModelrFrXpredictions.decoder.biasc0t|t|dd|_t||_|dSNFrIr\r]rCr<r/clsrGrnrprr r]   zBertLMHeadModel.__init__cC |jjjSrrr0r-rrrr get_output_embeddingsrz%BertLMHeadModel.get_output_embeddingscC||jj_dSrrroZnew_embeddingsrrr set_output_embeddingsz%BertLMHeadModel.set_output_embeddingsrY)r\r@NTr=rFcCs|dur|n|jj}| durd} |j||||||||| | | | ||d}|d}||}|r=|ddddddfSd}| dur}|ddddddf}| ddddf} t|d}||d|jj| d}||dd d}|durt j t j |dd|dd }|| d k d}d||||}|s|f|d d}|dur|f|S|St |||j|j|j|jd S) av encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]`` past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. use_cache (:obj:`bool`, `optional`): If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up decoding (see :obj:`past_key_values`). Returns: Example: >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig >>> import torch >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased') >>> config = BertConfig.from_pretrained("bert-base-cased") >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits NF rrwrXrrxrrrrrrrr]rrYr') reductionrr&lossZlogitsrrr#r$)rDrvr<rrrrr_rssumrAF log_softmaxrrrr#r$rorvrrwrXrrxrrlabelsrrrrrr]r soft_labelsalpha return_logitsrr2r3Zlm_lossZshifted_prediction_scoresZloss_fctZ loss_distillrrrr r}"sr@   zBertLMHeadModel.forwardcKsV|j}|dur ||}|dur|ddddf}||||dd|ddddS)NrYrrT)rvrrrrr])r>Znew_onesget)rorvpastrZ model_kwargsrzrrr prepare_inputs_for_generations   z-BertLMHeadModel.prepare_inputs_for_generationcs.d}|D]}|tfdd|Df7}q|S)Nrc3s|] }|dVqdS)rN) index_select)rZ past_statebeam_idxrr r!s  z1BertLMHeadModel._reorder_cache..r)rorrZreordered_pastZ layer_pastrrr _reorder_caches zBertLMHeadModel._reorder_cacheNNNNNNNNNNNNNNTr=NrF)NN)r~rr"_keys_to_ignore_on_load_unexpectedrAr]rrr r}r1r rrr}rrrrrrpr r sL x rcseZdZdgZddgZfddZddZdd Zee d e e d e ed     dddZZS)BertPrefixModelrFrXrcrrrrnrprr r]rzBertPrefixModel.__init__cCrrrrrrr rrz%BertPrefixModel.get_output_embeddingscCrrrrrrr rrz%BertPrefixModel.set_output_embeddingsrYrZr[NTr=rFcCsp|dur|n|jj}| durd} |j||||||||| | | | ||d}|d}||}|r=|ddddddfSd}| duro|ddddddf}| ddddf} t}||d|jj| d}|durtj t j |dd|dd }|| dk }d||||}|s|f|dd}|dur|f|S|St |||j|j|j|jd S) NFrrrYr'rrr&r)rDrvr<rrrrr_rArrrr=rrrr#r$rrrr r}sp  zBertPrefixModel.forwardr)r~rrrrAr]rrr r}r1r r~rrr}rrrrpr rsH rcseZdZeZfddZedddZeddd Z d d Z d d Z ddZ e ddZe ddZdddZeddZZS)MPlugcsft|||_ttj|jt j |_ | || ||_t|jdd|_t|jdd|_dSr)r\r]rDrfrom_pretrainedr-r.r7 model_dirr VOCAB_FILE tokenizermodule_setting_initialize_clipvisual_encoderrCconfig_encoder text_encoderr config_fusionfusion_encoderrnrprr r]3s    zMPlug.__init__NTc Csddlm}|jt|jt|jti}|j t j |t }||_|dur&|j}|||}|rgt j |tj}tj|dd} d| vrE| d} d| vrM| d} dd| D} |j| d d } td |t| |S) NrTaskscpuZ map_locationrCr cSi|] \}}|dd|qSzmodel.replacerkr!rrr V z)MPlug.from_pretrained..Fstrictzload checkpoint from %s)modelscope.utils.constantrZvisual_question_answeringMPlugForVisualQuestionAnsweringZimage_captioningMPlugForImageCaptionZimage_text_retrievalMPlugForImageTextRetrievalr@from_yaml_filer-r.r7 CONFIG_NAMErtaskrTORCH_MODEL_BIN_FILErAloadrUload_state_dictprint) rrrload_checkpointr task_mappingrDrCcheckpoint_pathrmsgrrr r?s8    zMPlug.from_pretrainedcCsdd}ddlm}||}d|jvr,t|j|jd}tt|dd }nt|j|jd}tt|dd  }||j j d | d |_ ||j _ |S) NcSs|jd}|ddddf|dddf}}|d8}ttt|}tt|}|d||ddddd}|j}tj | ||fdd}| |}|ddddd||d}t j ||gdd}|S) Nr'rrYrr&Zbilinear)rsmoder)r>r<rrr;ZreshaperrPrZ interpolatefloatrrAr)ZposembZ posemb_newZntok_newZ posemb_tokZ posemb_gridZgs_oldZgs_neworigrrr resize_pos_embedcs( &    z0MPlug._initialize_clip..resize_pos_embedr')clipzViT-B-16iir)rZload_from_configZ clip_namer< image_resrr.rArtrvisualrZ unsqueezer#)rDZ num_patchesrrZ clip_modelZ pos_embedrrr r`s     zMPlug._initialize_clipcCs|j|_|jrq|||_t|jdd|_t|jdd|_t |j |_ |j |jg|j |jg|j|j gg|_|jj|jkrht|j|jj|_tj|jjdd|_t|jj|_|j|j|jg|j|jgg|d|_dSdSNFr-q=rVףp= ?)distillrvisual_encoder_mrCrtext_encoder_mrrfusion_encoder_mrconfig_decodertext_decoder_mrr text_decoder model_pairsr` vision_widthrr visn_fc_mrfvisn_layer_norm_mrhri dropout_mextendvisn_fcvisn_layer_norm copy_paramsmomentumrnrrr init_distills@        zMPlug.init_distillcOtrNotImplementedErrorror@kwargsrrr r}z MPlug.forwardcCstj|j|j}t||_|jj|j_ t||_ t||_ d|j _ |j j |j _ d|_|jj|jkrVt|j|jj|_tj|jjdd|_t|jj|_d|_dSdS)NTFrrV)r-r.r7r bert_configrfrom_json_filertext_encoder_layersrrrrtext_decode_layerslarger`rrrrrfrrhrirjrorDZbert_config_pathrrr rs$      zMPlug.module_settingcCH|jD]}t|d|dD]\}}|j|jd|_qqdSNrr'Frr4 parametersrBZcopy_Z requires_gradroZ model_pairparamZparam_mrrr r    zMPlug.copy_paramscCR|jD]#}t|d|dD]\}}|j|j|jd|j|_qqdSNrr'r>rr4rrBrrrrr _momentum_update     zMPlug._momentum_updater'cC$||g}|jj||d\}}||fSNout_sizebeam_generatorZtranslate_batchroquestion_states question_attsr Zencoder_inputstopk_idsZ topk_scoresrrr generation  zMPlug.generationcnddl||dg|}||<|j|}tfddtD}t||| |j S)Nrr'cg|] }|qSrrlrZinit_dimn_tilerErr rzMPlug._tile.. r)rsrrcrAZ LongTensorZ concatenaterrrrrrrZ repeat_idxZ order_indexrrr _tile  z MPlug._tileNT)rr')r~rrrr@r] classmethodr staticmethodrrr}rrAno_gradrrrrrrrrpr r0s"   '   rc0eZdZfddZ     dddZZS) rc4t|t|j|_t||j|_||dSrr\r]rrrrrrrnrprr r]  z(MPlugForVisualQuestionAnswering.__init__NrTc! Cs.|jt|jd}|jj|dd}|jr!||| |}t j | ddt j d|j} |r^ |j|j|jjkd} |j|j|jdd} | j} |j| |j|| dd} | \}}t ||gd }t | |jgd }|dur|d g|jd }g}g}t|D]\}}|||g|7}|||g|7}qt |d }t |d }|jr9t l||jj|dd}|jr|| |!|}|j"|j|jdd}|j}|j#||j|| dd}|\}}t ||gd }g}t|D] \}}|||g|7}qt |d }|j$|j|j||dd }Wdn 1swY|j%|j|j||| dt&j'|dd d d}n|j%|j|j||| dd d}|durNd }||j(}|)| d }|S|j|j|jdd} | j} |j| |j|| dd} | \}}t ||gd }t | |jgd }|*||\}} || fS)NrTZskip_last_layerrYrrrFrxrrrrr'rrrrrrnonerrrrrrrrrrrrr)+rnextrrPrrrrjrrrArdrsrurrv masked_fillrrarrr"rrr> enumeratestackrr#rrrrrrrrrrsoftmaxrrr)!roimagequestionanswerrrweightsr2 image_embeds image_attsanswer_targets text_output text_embeds fusion_output image_outputquestion_outputmerge_text_attentionrrbrimage_embeds_m text_output_m text_embeds_mfusion_output_mimage_output_mquestion_output_mquestion_states_mlogits_m answer_outputrr topk_probsrrr r}s     %     z'MPlugForVisualQuestionAnswering.forwardNrNNTrrrrpr r  rc>eZdZfddZ   d ddZ    d d d ZZS) rc*t|t|j|_t||j|_dSrr\r]rrrrrrnrprr r]m  zMPlugForImageCaption.__init__NTcCs|jj|dd}|jr||||}tj|ddtj d |j }|j |j |jdd}|j} |j| |j||dd} | \} } t| | gd} t||jgd} |j| | |d \}}||fS) NTr(rYrr)Fr*r'r )rrrrjrrrArdrsrurrrrvrr"rrr)ror4r5r6r2r r8r9r;r<r=r>r?r@rrKrrr beam_searchrs@  z MPlugForImageCaption.beam_searchFc Cs|r |j|||d|dS|jt|jd}|jj|dd}|jr-|| | |}t j | ddt jd|j}|r`|j|j|jjkd} |j|j|j||| ddd} | j} | S|||\} } | | fS) NTr2r rr(rYrr,r.)rSrr/rrPrrrrjrrrArdrsrurrvr0rrarrrr)ror4r5r6r2r scstr8r9r:rJrrrKrrr r}s@ zMPlugForImageCaption.forwardNTrRNTrRFr~rrr]rSr}rrrrpr rks  "rcs>eZdZfddZddZeddZd d d ZZ S) rcs&t||j|_ttg|j|_|j|_|j |_ |j |_ |j|_|j j |_ |j|_t|j |j|_t|j |j|_t|j d|_|dt|j|j|dt|j|j|dtd|jfd|dtjdtjdtj|jd d |_tj|jd d |_||dS) Nr& image_queue text_queue idx_queuer'r queue_ptrrrr)r\r] embed_dimrr.rArdtemp queue_sizerrrr` text_widthr vision_proj text_projitm_headrkZrandnfullrtrur normalizerYrZrrnrprr r]s2  z#MPlugForImageTextRetrieval.__init__cCs|j|_|jr|||_t|jdd|_t|jdd|_t |j |j |_ t |j |j |_|j|jg|j|jg|j|jg|j|j gg|_|jj|jkryt |j|jj|_t j|jjdd|_t |jj|_|j|j|jg|j|jgg|d|_ dSdSr)!rrrrCrrrrrrrr`r] vision_proj_m text_proj_mrrrbrarr`rrrfrrhrirrrrrrrnrrr rsD        z'MPlugForImageTextRetrieval.init_distillc Csdd}||}||}||}|jd}t|j} |j|jdd| | |f<|j|jdd| | |f<|j|jdd| | |f<| ||j} | |jd<dS)NcsNtjsSfddttjD}tjj|ddtj|dd}|S)z Performs all_gather operation on the provided tensors. *** Warning ***: torch.distributed.all_gather has no gradient. csg|]}tqSr)rAZ ones_like)rryrSrr rrqz^MPlugForImageTextRetrieval._dequeue_and_enqueue..concat_all_gather..F)Zasync_oprr)rA distributedZis_initializedrZget_world_sizeZ all_gatherr)rSZtensors_gatherrrrhr concat_all_gathers   zJMPlugForImageTextRetrieval._dequeue_and_enqueue..concat_all_gatherr)r>r<r\TrYrZr[r_) ro image_feat text_featidxrjZ image_featsZ text_featsZidxsrgptrrrr _dequeue_and_enqueues  z/MPlugForImageTextRetrieval._dequeue_and_enqueueNTc6 Cs(|r|jj|dd}|jr||||}tj|ddtj d |j }t j ||dddddfdd}|j|j|jdd}|j} t j || dddddfdd} |dd}tj||jgdd} t|| } | | jddd } t||jj|dd}|jr| |!|"|}t j |#|dddddfdd}tj||j$gdd}|j%|j|jdd}t j |&|jdddddfdd}tj||j'gdd}|j(r)|||j)}|||j)}|j*t j+|ddd|j*| }|j*t j+|ddd|j*| }Wdn 1s4wY|||j)}| ||j)}|j(rntjt j,|dd|dd- }tjt j,|dd|dd- }n"tjt j,|dd| dd- }tjt j,|dd| dd- }||d }|.||||j/| |j||d d \}}t>|d}t j+|ddd|fdd} t j+|ddd|fdd}!t||j0}"| 1|"d|!1|"dWdn 1swYg}#t2|D]}$t3|!|$d4}%|#5||%qtj6|#dd}#g}&g}'t2|D]}$t3| |$d4}%|&5| |%|'5|j|%qtj6|&dd}&tj6|'dd}'tj| |&gdd}(tj|j|'gdd})tj|#|gdd}*tj||gdd}+|j/|(|)|*|+d d \}},tj|dddddf|,dddddfgdd}-|7|-}.tj|tj d}/tj8d |tj d}0tj|/|0gdd |j }1t 9|.|1}2||2S|j|j|jd }|j} |jj|dd}|||}tj|ddtj |j d}3|j/| |j||3d d \}}4|7|4dddddf}5t j+|5dd}5|5S)NTr(rYrrrr)r')Zkeepdimr&Fr*)rrr):rrrrjrrrArdrsrurrrrerarrvrr"rbrrtr[clonedetacheqrrr#rrrrrrfrYrrgrZrr^rr3rr=rprrkZ masked_fill_rZ multinomialitemr3r2rcrtZ cross_entropy)6ror4textrnr2r8r9rlr;r<rmZidx_allZpos_idxZ sim_targetsrBZ image_feat_mZimage_feat_allrCZ text_feat_mZ text_feat_allZ sim_i2t_mZ sim_t2i_mZsim_i2t_targetsZsim_t2i_targetsZsim_i2tZsim_t2iZloss_i2tZloss_t2iZloss_itaryZ output_posbsZ weights_i2tZ weights_t2irpZimage_embeds_negrAZneg_idxZtext_embeds_negZ text_atts_negZtext_embeds_allZ text_atts_allZimage_embeds_allZimage_atts_allZ output_negZ vl_embeddingsZ vl_outputZones_tmpZ zeros_tmpZ itm_labelsZloss_itmZ image_attrZscoresrrr r} sr      !         ,    z"MPlugForImageTextRetrieval.forwardr) r~rrr]rrAr#rpr}rrrrpr rs   %rcs|eZdZeZfddZedddZddZdd Z d d Z e d d Z e ddZdddZeddZZS)HiTeAcsnt|||_ttj|jt j |_ | |t |jt|jd|_t|jdd|_t|jdd|_dS)NZimg_sizerD num_framesFr)r\r]rDrrr-r.r7rrrrrrrrrzrrCrrrrrrnrprr r] s"   zHiTeA.__init__Tc Csddlm}|jt|jti}|jtj |t }||_ ||j |}|rTtj |tj}tj|dd}d|vr<|d}d|vrD|d}dd|D}|j|d d |S) NrrrrrCr cSrrrrrrr r rz)HiTeA.from_pretrained..Fr)rrZvideo_question_answeringHiTeAForVideoQuestionAnsweringZvideo_captioningHiTeAForVideoCaptionr@rr-r.r7rrrrrrArrUr) rrrrrrDrCrrrrr r s.  zHiTeA.from_pretrainedcCs|j|_|jrBt|jt|jd|_t|jdd|_t |j dd|_ t |j |_|j|jg|j|jg|j|jgg|_|d|_dSdS)NryFrr)rrrrrzrrCrrrrrrrrrrrrrrrnrrr r s*     zHiTeA.init_distillcOrrrrrrr r} rz HiTeA.forwardcCsZtj|j|j}t||_|jj|j_ t||_ t||_ d|j _ |j j |j _ dSr)r-r.r7rrrrrrrrrrrrrrr r s    zHiTeA.module_settingcCrrrrrrr r rzHiTeA.copy_paramscCrrrrrrr r rzHiTeA._momentum_updater'cCr r r rrrr r rzHiTeA.generationcr)Nrr'crrrrrrr r( rzHiTeA._tile..rrrrr r rz HiTeA._tilerzr )r~rrrr@r]r!rrr}rrAr#rrrr"rrrrrpr rx s     rxcr$) r{cr%rr&rnrprr r]. r'z'HiTeAForVideoQuestionAnswering.__init__NrTc! Cs|jt|jd}||}tj|ddtjd|j } |r; |j |j |j j kd} |j|j |jdd} | j} |j| |j|| dd} | \}}t||gd}t| |jgd}|durkdg|jd }g}g}t|D]\}}|||g|7}|||g|7}qst|d }t|d }|jrt[|||}|j|j |jdd}|j}|j||j|| dd}|\}}t||gd}g}t|D] \}}|||g|7}qt|d }|j|j |j||dd }Wdn1swY|j|j |j||| dtj|dd d d }n|j|j |j||| dd d}|dur+d}||j }|!|d }|S|j|j |jdd} | j} |j| |j|| dd} | \}}t||gd}t| |jgd}|"||\}} || fS)NrrYrTr)Fr*r'rr+rr,r-r.)#rr/rrPrrArdrsrurrvr0rrarrr"rrr>r1r2rr#rrrrrrrr3rrr)!rovideor5r6rrr7r2 video_embeds video_attsr:r;r<r= video_outputr?r@rrrArZvideo_embeds_mrCrDrErFrGrHrIrJrrrKrrr r}4 s             z&HiTeAForVideoQuestionAnswering.forwardrLrrrrpr r{, rMr{crN) r|crOrrPrnrprr r] rQzHiTeAForVideoCaption.__init__NTrRcCs||}tj|ddtjd|j}|j|j|j dd}|j } |j | |j ||dd} | \} } t | | gd} t ||j gd} |j | | |d\}}||fS) NrYrTr)Fr*r'r )rrArdrsrurrrrvrr"rrr)ror}r5r6r2r r~rr;r<r=rr?r@rrKrrr rS s8   z HiTeAForVideoCaption.beam_searchFc Cs|r |j|||d|dS|jt|jd}||}tj|ddtj d|j }|rO|j |j |j jkd} |j|j |j||| ddd} | j} | S|||\} } | | fS)NTrTrrYrr,r.)rSrr/rrPrrArdrsrurrvr0rrarrrr)ror}r5r6r2r rUr~rr:rJrrrKrrr r} s8  zHiTeAForVideoCaption.forwardrVrWrXrrrpr r| s  r|)Vrrr-typingrrAZtorch.nn.functionalrZ functionalrZtorch.utils.checkpointZ transformersrrZtorch.nnrrrZtransformers.activationsr Ztransformers.file_utilsr r r r Ztransformers.modeling_outputsrrrZtransformers.modeling_utilsrZtransformers.utilsrZ7modelscope.models.multi_modal.mplug.configuration_mplugrrZ(modelscope.models.multi_modal.mplug.mvitrrZ-modelscope.models.multi_modal.mplug.predictorrrrZmodelscope.utils.torch_utilsrrrZset_verbosity_errorZ get_loggerr+rrr~rMrTModulerUrrrrrrrrr r%r)r+r/r6r:r;ZBERT_START_DOCSTRINGr}rCrrrrrrrrxr{r|rrrr s        JB4QZ\\   *7m+Kq