o *j@sdZddlZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z m Z mZddlZddlmmZddlZddlmZddlmZmZddlmZddlmZddlmZmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*dd l+m,Z,dd l-m.Z.m/Z/m0Z0ddl1m2Z2ddl3m4Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ejdej=?dej=@dej=Ade5BZ4dZCdZDdgZEGdddeZFddZGGdddejjHZIejJjKd d!ZLd"d#ZMGd$d%d%ejjHZNd&d'ZOejJjKd(d)ZP   d>d*d+ZQGd,d-d-ejjHZRGd.d/d/ejjHZSGd0d1d1ejjHZTGd2d3d3ejjHZUGd4d5d5e0e&ZVd6ZWd7ZXe)d8eWGd9d:d:eVZYe.jZe7j[e,j\d;Gd_sz0load_tf_weights_in_chatglm_6b..z Skipping z [A-Za-z]+_\d+z_(\d+)ZkernelgammaweightZ output_biasbetabiasZoutput_weightsZsquadZ classifierr iZ _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )renumpyZ tensorflow ImportErrorrerrorospathabspathinfotrainZlist_variablesZ load_variableappendzipsplitr+join fullmatchgetattrAttributeErrorlenint transposeshapeAssertionErrorargsr)Z from_numpydata)modelconfigZtf_checkpoint_pathr?nptfZtf_pathZ init_varsnamesZarraysnamerRarraypointerZm_nameZ scope_namesnumer.r.r/load_tf_weights_in_chatglm_6bBs                     r`cs0eZdZdZfddZdejfddZZS) PrefixEncoderz The torch.nn model to encode the prefix Input shape: (batch-size, prefix-length) Output shape: (batch-size, prefix-length, 2*layers*hidden) c st|j|_|jr7tj|j|j|_tj tj |j|jtj tj |j|j |jd|_ dStj|j|j |jd|_dS)Nr>)super__init__prefix_projectionr)r Embedding pre_seq_len hidden_size embeddingZ SequentialLinearZTanh num_layerstransr-rW __class__r.r/rcs     zPrefixEncoder.__init__prefixcCs,|jr||}||}|S||}|SN)rdrhrk)r-ro prefix_tokenspast_key_valuesr.r.r/forwards    zPrefixEncoder.forward) r1r2r3__doc__rcr)Tensorrs __classcell__r.r.rmr/ras racCs*d|dtd|dd||S)zOpenAI's gelu implementation.??gQ63E?gHm?)r)tanhxr.r.r/ gelu_impls  r|cCst|Srp)r|rzr.r.r/gelusr}csDeZdZdejdffdd ZddZd d d Zfd d ZZ S)RotaryEmbedding'Fcsxtd|td|d|}|}||_|r(tj||_ d|_ n| d|d|_ d|_ d|_ ||_dS)Nrxrr>inv_freq)rbrcr)arangefloathalf learnabler Parameterrmax_seq_len_cachedZregister_buffer cos_cached sin_cached precision)r-dimbaserrrrmr.r/rcs   zRotaryEmbedding.__init__cCsdSrpr.)r-Z state_dictroZlocal_metadatastrictZ missing_keysZunexpected_keys error_msgsr.r.r/_load_from_state_dictz%RotaryEmbedding._load_from_state_dictr Nc Cs|dur |j|}|jdus||jkr{|jrdn||_tj||j|jjd}td||j}tj ||fdd |j}|j tj krE| }|dddddf}|dddddf}|j tj krm| }| }|jrt||fS|||_|_|jd|df|jd|dffS)N)devicedtypezi,j->ijr.)rRrrr)rrrrZeinsumcattorZbfloat16rcossinrr) r-r{Zseq_dimseq_lentZfreqsZembrrr.r.r/rss*      $zRotaryEmbedding.forwardcs8|jdur ||j|_|jdur||j|_t|Srp)rrrb_apply)r-fnrmr.r/rs     zRotaryEmbedding._apply)r N) r1r2r3r)rrcrrsrrvr.r.rmr/r~s  r~cCsP|dd|jddf|d|jdddf}}tj| |f|jddS)N.rr>r r)rRr)rndim)r{x1x2r.r.r/ rotate_halfs 6rcCsdt||ddt||dd}}||t||||t||}}||fS)Nr r>)Frhsqueeze unsqueezer)qkrrZ position_idr.r.r/apply_rotary_pos_emb_indexs rc CsV|dur|d|d} } tj| |fdd}tj| |fdd}|j\} } }}| r-||f}nd}t|d}|r@|t||}|d|d|d|df}||d|d|dd}||d|d|dd}tjddd|j |j d}tj || dd| dd dddd d }|j|}|j r||j _| ||}n#|dks||d |j }|}||}tj|dd}||}|d|d|d|df}||d|d|dd}||d|d|dd}t|| dd}|j|}|dddd}|dd |f}|j|}|||f}|S) Nrr rr>rrrgrx)r<alpha)r)rrRrmathsqrtsizeviewzerosrrZbaddbmmrQscale_mask_softmaxscale contiguousall masked_fill_rsoftmaxtypeZbmmpermute)r- query_layer key_layer value_layerattention_maskhidden_size_per_partitionlayer_id layer_pastZscaling_attention_score use_cacheZpast_keyZ past_valuerbZnhrgpresentZquery_key_layer_scaling_coeffZ output_sizeZ matmul_resultattention_scoresattention_probsr context_layerZnew_context_layer_shapeoutputsr.r.r/ attention_fns              rc szeZdZddejdffdd ZeddZ ddd Z   dd ej d ej d e e ej ej fd e de f ddZ ZS) SelfAttentionNTcstt|||_||_||_||_||_||_t |r$|j|jdn|j|jdt j dd|_ d|_ |dur>|||_n||_||j|_tt jj|d|j||d|_tt jj|j|||d|_dS)Nr>rF)rrrrr=r)rbrrcrrgrnum_attention_heads!num_attention_heads_per_partitionposition_encoding_2dr~r)r rotary_embrhidden_size_per_attention_headinner_hidden_sizer r riquery_key_valuedense)r-rgrrrr= params_dtyperrmr.r/rcwsH     zSelfAttention.__init__cCs||d|S)Nr)r)rrr.r.r/attention_mask_funcs z!SelfAttention.attention_mask_funcFcCsF|d}|||}tj|||d}|r!tdd|DS|S)a#Split a tensor along its last dimension. Arguments: tensor: input tensor. num_partitions: number of partitions to split the tensor contiguous_split_chunks: If True, make each chunk contiguous in memory. r rcss|]}|VqdSrp)r)r7chunkr.r.r/r9z.)rrr)rJtuple)r-tensorZnum_partitionsZcontiguous_split_chunksZlast_dimZ last_dim_sizeZ tensor_listr.r.r/split_tensor_along_last_dims z)SelfAttention.split_tensor_along_last_dim hidden_statesrrroutput_attentionsc Cs||}|dd|jd|jf} |j| }||d\} } } |jr| jd| jdd\} }| jd| jdd\}}|j | | dd\}}|dddddf dd |dddddf dd }}t | ||||\} }t |||||\}}tj| |g| jdd} tj||g|jdd} n| dd}|j | | dd\}}t | | |||\} } t|| | | ||j|||d \}}}||}||f}|r||f7}|S) q hidden_states: [seq_len, batch, hidden_size] attention_mask: [(1, 1), seq_len, seq_len] Nrrr>r r)rr) r-rrrrrrrr)rrrrrrrrrrmaxrQrrr)concatrrr)r-r position_idsrrrrrZmixed_raw_layerZnew_tensor_shaperrrq1q2Zk1Zk2rrblock_position_idsrrroutputrr.r.r/rssZ            zSelfAttention.forwardFNFF)r1r2r3r)rrc staticmethodrrrurrboolrsrvr.r.rmr/rus03  rcs$eZdZfddZddZZS)GEGLUcsttj|_dSrp)rbrcrr} activation_fnr-rmr.r/rcs  zGEGLU.__init__cCs&|jd|jdd\}}|||S)Nr>r r)rrr)r-r{rrr.r.r/rssz GEGLU.forward)r1r2r3rcrsrvr.r.rmr/r s rcs2eZdZdddeejffdd ZddZZS)GLUNTcsrtt|||_||_||_|durd|}||_ttj j |j|j||d|_ ttj j |j|j||d|_ dS)Nr) rbrrcractivation_funcrgrr r)r ri dense_h_to_4h dense_4h_to_h)r-rgrrr=rrrmr.r/rcs* z GLU.__init__cCs"||}||}||}|S)z> hidden_states: [seq_len, batch, hidden_size] )rrr)r-rZintermediate_parallelrr.r.r/rs:s   z GLU.forward) r1r2r3r}r)rrcrsrvr.r.rmr/rs rc sheZdZddedejddffdd Z   ddejdejd ee ejejfd e d e f d d Z Z S)GLMBlockNTc sntt|||_|||d|_| |_t|||||| |jd|_|||d|_| |_ t ||||| d|_ dS)Neps)rr=rr)rr=rr) rbrrcrinput_layernormrr attentionpost_attention_layernormrjrmlp) r-rgrlayernorm_epsilonrrr layernormuse_biasrrjrrmr.r/rcKs0   zGLMBlock.__init__Frrrrrc Cs||}|j|||||||d} | d} | dd} d|jd} || | }||} || }| | |}|rA|f| } | S|f| dd} | S)r)rrrrrrr Nr>rw)rrrjrr)r-rrrrrrrZattention_inputZattention_outputsZattention_outputrrZ mlp_inputZ mlp_outputrr.r.r/rszs,       zGLMBlock.forwardr) r1r2r3r r)rrcrurrrrsrvr.r.rmr/rIs.5rcsreZdZdZdZdZeZdZdgZ fddZ de j fd d Z d d Zdd dZdddZefddZZS)ChatGLMPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. FT transformerrc s*tj|jfi|tt||dSrp)rbrc name_or_pathr)r-rWkwargsrmr.r/rcszChatGLMPreTrainedModel.__init__modulecCsdS)zInitialize the weights.Nr.)r-rr.r.r/ _init_weightsrz$ChatGLMPreTrainedModel._init_weightsc sz|j\}}fdd|D}tj|||f|d}|t|D]\}}d||ddd|f<q |d|dk}|S)Ncg|] }|jjqSr.tolistindexrW bos_token_idr7seqrr.r/ z4ChatGLMPreTrainedModel.get_masks..rr rw)rRr)onesZtril_ enumerateZ unsqueeze_r) r-r%r batch_size seq_lengthcontext_lengthsricontext_lengthr.rr/ get_maskss     z ChatGLMPreTrainedModel.get_masksc s|j\}fdd|D}jrPtjtjdd|d}t|D]\}} ||||| df<q%fdd|D} tj| dd} tj|| fdd}|Stjtjdd|d}|sst|D] \}} |||| d<qf|S)Ncrr.rrrr.r/rrz;ChatGLMPreTrainedModel.get_position_ids..rrr c s>g|]}ttj|tjdtj|tjddfqS)rr )r)rrlongr)r7r)rr r.r/rs" r) rRrr)rrrrepeatr stack) r-r%mask_positionsrgmaskr r rrrrr.)rr-r r/get_position_idss>      z'ChatGLMPreTrainedModel.get_position_idscCst|tr ||_dSdSrp) isinstance ChatGLMModelgradient_checkpointing)r-rvaluer.r.r/_set_gradient_checkpointings  z2ChatGLMPreTrainedModel._set_gradient_checkpointingc s<|dd}|ddtt|jdd|i|}||_|S)a'Instantiate the model. Args: kwargs: Input args. model_dir: The model dir used to load the checkpoint and the label information. Returns: The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained model_dirNcfgZpretrained_model_name_or_pathr.)poprbrfrom_pretrainedr)clsrrrVrmr.r/ _instantiates z#ChatGLMPreTrainedModel._instantiater)r1r2r3rtZis_parallelizableZsupports_gradient_checkpointingr! config_classZbase_model_prefixZ_no_split_modulesrcr Modulerrrr classmethodr!rvr.r.rmr/rs   $raM This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`~ChatGLM6BConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a: Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`ChatGLM6BTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zdThe bare ChatGLM-6B Model transformer outputting raw hidden-states without any specific head on top.cseZdZdZdeffdd ZddZdejfdd Z ej fd d Z e e d eeeed          ddeejdeejdeejdeeeejejfdfdeejdeedeedeedeedeeejdfeffddZZS)ra The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. rWcs$t||j_|j_tj_|j_|j_|j _ |j _ |j _ jj_ |j _ |j_|j_ttjjjjjd_d_fddtjfddtj D_tjj d_jdurD]}d|_qstj_t|_ tj!d_"dSdS) N)Znum_embeddingsZ embedding_dimrFc s*tjjj|jjtdjjd S)NT)rrrrrr) rrgrrrrr rr)rrr.r/ get_layerwsz(ChatGLMModel.__init__..get_layercsg|]}|qSr.r.)r7r)r%r.r/rsz)ChatGLMModel.__init__..rg?)#rbrcmax_sequence_lengthrgr)rrr vocab_sizerjrrrrrfrdr r reword_embeddingsrZ ModuleListrangelayersr final_layernorm parametersZ requires_gradrrrqraprefix_encoderZDropoutdropout)r-rWparamrm)r%r-r/rc_sD     zChatGLMModel.__init__cC|jSrpr(rr.r.r/get_input_embeddingsz!ChatGLMModel.get_input_embeddingsnew_embeddingscC ||_dSrpr1r-r4r.r.r/set_input_embeddings z!ChatGLMModel.set_input_embeddingscCsp|jd|d|}|||}|||j|jd|j |j |j }| |}| gd d}|S)Nrrr>)r>r rrr)rqrexpandrr-rrrfrjrrgr.rrJ)r-r rrrqrrr.r.r/ get_prompts  zChatGLMModel.get_promptzbatch_size, sequence_length) checkpoint output_typer"Nr%rrrr. inputs_embedsrroutput_hidden_states return_dictr'c  s<|dur|n|jj}|dur|n|jj}|dur|n|jj}| dur$| n|jj} |jr2|jr2|r2d}|dur>|dur>td|durL|jdd\} } n|dur[|jdd\} } } ntd|durh| |}|dur|j dur|j |jd|j |j d}n tdgt|j}|dur|j||j d}|dur|jj|jj} }||vr|n| ||vrdnd}fd d |D}|j|||j |d }|j dur|durt| d |d |j |j }|dk}tj||fdd}|dd }|rdnd}|rdnd}|rdnd}|durtjd d |j d}n||j }t|jD]V\}}|r.||f}||}|jrL|jrLtjj||||t ||||}n||||t ||||d}|d}|rh||d f}|rw|||rrdnd f}q"|!|}|r||f}| stdd||||fDSt"||||dS)NFzDYou cannot specify both input_ids and inputs_embeds at the same timer>z5You have to specify either input_ids or inputs_embedsr)r rrrTcsg|] }|qSr.)rrr mask_tokenr.r/rz(ChatGLMModel.forward..)rrrr rrwrrr.)rrrrrrcss|] }|dur|VqdSrpr.)r7vr.r.r/r92s z'ChatGLMModel.forward..)Zlast_hidden_staterrr attentions)#rWrr>ruse_return_dictrZtraining ValueErrorrRr(rfr:rrrrOr*r mask_token_idgmask_token_idrr)r rrrrrQrr utilsr;rr+r)r-r%rrrrr=rrr>r?r r _MASKgMASK use_gmaskrZprefix_attention_maskrZpresentsZall_self_attentionsZall_hidden_statesrlayerrZ layer_retr.r@r/rss                  zChatGLMModel.forward) NNNNNNNNN)r1r2r3rtr!rcr2r)rur7rr:rCHATGLM_6B_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOCrr4rrrrrsrvr.r.rmr/rKs\9    r) module_namecsZeZdZdeffdd ZddZddZ  d>d ed ee e fd e d e d ee e ff ddZ    d?de jdee jdee jdee jdee jd ef ddZ          d@dee jdee jdee jdeee jdee jdee jdee dee dee dee fddZedeee je jfd fd!e jd eee je jfd ffd"d#Zd$d%Ze   & ' ( ) * dAd+e d,eee e fd-efd.d/Ze   & ( ) * dBd+e d,eee e fd-efd0d1Ze     d?d2eed3eed4ee d5ee!ee jgeeffd6d7Z"dCd8efd9d:Z#d;ed efdr.)Z_extract_past_from_model_outputrr)rrZnew_onesrRclone) r-rr^r_r`rZnew_attention_maskrZnew_position_idr.r.r/#_update_model_kwargs_for_generationds2   zCChatGLMForConditionalGeneration._update_model_kwargs_for_generationNr%pastrrrrc s|j\}jjjj}} | |vr| n|| |vrdnd} |} fdd| D} |dus3|dur|dddfd} |durV|jtjkrV|ddddddf}nd}|dure|dddf}n6fdd| D}j rtj fddt | |Dtj |j d d}ntj d d| Dtj |j d d}|dur|}| |||d S|dur|jtjkrd}|durj||j d }|durΈj||j | | d }||||d S)NTFcsg|]}|qSr.)rrr@r.r/rszQChatGLMForConditionalGeneration.prepare_inputs_for_generation..r.csg|] }|jjqSr.)rrWrrrr.r/rrBcsg|] \}}||gqSr.r.)r7 mask_positionr)r r.r/rsrcSsg|]}|qSr.r.)r7rdr.r.r/rs)r%rrrrr)rrr)rRrWrGrHrrrr)rrrrIrrrr)r-r%rcrrrrrr rKrLrMZseqsrZ last_tokenr r.)rAr-r r/prepare_inputs_for_generationsx    z=ChatGLMForConditionalGeneration.prepare_inputs_for_generationr=labelsrrr>r?c  Cs:|dur|n|jj}| dur| n|jj} |j|||||||| | d } | d} || ddd} d}|dur{| tj } | dddddf}|dddf}t dd}||j }|| d| d| d}| | j} || j}| s| f| dd}|dur|f|S|St|| | j| j| jd S) N) r%rrrrr=rrr>r?rr r>.ri)Z ignore_index)losslogitsrrrrD)rWrrErrVrrrr)Zfloat32r rrrrrrrrrD)r-r%rrrrr=rfrrr>r?Ztransformer_outputsrZ lm_logitsrgZ shift_logitsZ shift_labelsZloss_fctrr.r.r/rssL       z'ChatGLMForConditionalGeneration.forward.beam_idxcstfdd|DS)aL This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct beam_idx at every generation step. Output shares the same memory storage as `past`. c 3sH|]}|dd|dj|dd|djfVqdS)rr N)Z index_selectrr)r7rrir.r/r9s  zAChatGLMForConditionalGeneration._reorder_cache..)r)rcrir.rjr/_reorder_cache s z.ChatGLMForConditionalGeneration._reorder_cachecCs|}|dd}ddgddgddgd d gd d gg}|D] }td |dd|d|}td|dd|d|}q|S)Nu[[训练时间]]u2023年,u，!u！:u：;u；z\?u？z([\u4e00-\u9fff])%srz\1%sr z%s([\u4e00-\u9fff])z%s\1)stripreplacer?sub)r-responseZpunktsitemr.r.r/process_responses    z0ChatGLMForConditionalGeneration.process_responser Tffffff?ffffff?queryhistory max_lengthc Ks|durg}| dur t} | t|||||| d| } |s#|} n d} t|D]\} \}}| d| ||7} q)| dt||7} || gdd}||j}|jd i|| }| dt|ddd}| |}| |}|||fg}||fS) N)r{ num_beams do_sampletop_p temperaturelogits_processor[Round {}] 问：{} 答：{} [Round {}] 问：{} 答：ptZreturn_tensorsrr%r.) rrHr$r rPrOrrgeneraterdecoderu)r-rZryrzr{r|r}r~rrr gen_kwargspromptr old_queryrsinputsrr.r.r/_chat,s<       z%ChatGLMForConditionalGeneration._chatc ks |durg}|durt}|t|||||d| } |s#|} n d} t|D]\} \} }| d| | |7} q)| dt||7} || gdd}||j}|jd i|| D](}| dt|ddd}| |}| |}|||fg}||fVqZdS) N)r{r}r~rrrrrrrrr%r.) rrHr$r rPrOrrstream_generaterrru)r-rZryrzr{r}r~rrrrrrrrsrrZ new_historyr.r.r/ stream_chatWs>       z+ChatGLMForConditionalGeneration.stream_chatgeneration_configrstopping_criteriaprefix_allowed_tokens_fnc +s|jd|jd}}|dur|j}t|}|jdi|} |j|j}} t| tr/| g} | dduo:|j du} | rN|j durNt d|j dtn|j durj|j ||_ | sjt d|j d|j dt||j kr|jjrud nd } td | d |d |j d|dur|nt}|dur|nt}|j|||||d}|j||d}||} ||jdd}d} |j|fi| }|di|dddd}|jdddddf}|||}| ||}tjj|dd}|jrt j!|dd"dnt j#|ddt j$|dddfgdd}|j%|| |jjd} |&t'fdd| D(}|)dks<|||r>dS|Vq)Nrrr{zUsing `max_length`'s default (z) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.zBoth `max_new_tokens` (=z) and `max_length`(=z) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)Zdecoder_input_idsr%zInput length of z is z, but `max_length` is set to zX. This can lead to unexpected behavior. You should consider increasing `max_new_tokens`.)rinput_ids_seq_lengthZencoder_input_idsrr)rrr TF)r?rr>r)Z num_samples)r_c3s|]}|kVqdSrpr.)r7rZ next_tokensr.r/r9rzBChatGLMForConditionalGeneration.stream_generate..r.)*rRrcopydeepcopyupdater eos_token_idrrPgetr{Zmax_new_tokenswarningswarn UserWarningrrWr_warningrrZ_get_logits_processorZ_get_stopping_criteriaZ_get_logits_warpernewZfill_rerhr functionalrr}r)Z multinomialrZargmaxrrbmulsumrr)r-r%rrrrrrJrr^rZhas_default_max_lengthZinput_ids_stringZ logits_warperZunfinished_sequencesr&Z model_inputsrZnext_token_logitsZnext_token_scoresZprobsr.rr/rs            z/ChatGLMForConditionalGeneration.stream_generatebitscKsX|dkrdSddlm}|jrtd|Sd|_||j_||j|fd|i||_|S)Nrr )rYzAlready quantized.TrU)Z quantizationrYrWrrFrWrXr)r-rrUrrYr.r.r/rYs   z(ChatGLMForConditionalGeneration.quantizeinputc Cs|d}|d}d|vr|d}nd}d|vr|d}nd}d|vr'|d}nd}d |vr2|d }nd }t|tjkr?|}|j|j||||||d \}}td tj |tj |iS) Ntextrzr{rvrrxr|r r}T)r{rr|r}zGeneration finished.) rr)rurrrZrrFrZRESPONSEZHISTORY) r-rrrzr{rr|r}rsr.r.r/chats6     z$ChatGLMForConditionalGeneration.chat)FF)NNNN) NNNNNNNNNN)Nrvr TrwrxN)NrvTrwrxNr)&r1r2r3r!rcr\r]rrstrrrrbr)r4rrudictrerr5rsrrkruZno_gradrrPrrrrrrrrYrrvr.r.rmr/rT>s   & H    =  * (  nrT)NTF)^rtrrrCr?sysrtypingrrrrrrrr)Ztorch.nn.functionalr rrZtorch.utils.checkpointZtorch.nnr r Ztorch.nn.utilsr Z&transformers.generation.logits_processr Ztransformers.generation.utilsrrrrZtransformers.modeling_outputsrrrZtransformers.modeling_utilsrZtransformers.utilsrrrZmodelscope.metainforZmodelscope.modelsrrrZmodelscope.outputsrZmodelscope.utilsrloggingZmodelscope.utils.constantr configurationr!Z tokenizationr"platformZ_CZ_jit_set_profiling_modeZ_jit_set_profiling_executorZ_jit_override_can_fuse_on_cpuZ_jit_override_can_fuse_on_gpuZ get_loggerrQrRZ(CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LISTr$r`r#raZjitscriptr|r}r~rrrrrrrrZCHATGLM_6B_START_DOCSTRINGrOrZregister_modulerZ chatglm_6brTr.r.r.r/s$                J 6  v 1f_ 2p