o *jD@sdZddlZddlmZmZmZmZddlZddl Z ddl m Z ddl mZmZddlmZddlmZeZdd iZGd d d ZGd d d ZGddde ZdS)z!Tokenization classes for ChatGLM.N)DictListOptionalUnion)PreTrainedTokenizer) BatchEncoding EncodedInput)PaddingStrategy)loggerzTHUDM/chatglm-6bic@sVeZdZddZddZdeefddZdd Zd d Z d d Z ddZ ddZ dS) TextTokenizercCs&t|_|j||j|_dSN)spmZSentencePieceProcessorspLoad vocab_size num_tokens)selfZ model_pathrk/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/nlp/chatglm/tokenization.py__init__s  zTextTokenizer.__init__cC |j|Sr )rZ EncodeAsIdsrtextrrrencode zTextTokenizer.encodeidscCrr )rZ DecodeIds)rrrrrdecoderzTextTokenizer.decodecCrr )rZEncodeAsPiecesrrrrtokenize!rzTextTokenizer.tokenizecsfdd|DS)Ncsg|]}j|qSrrZ PieceToId).0tokenrrr %z7TextTokenizer.convert_tokens_to_ids..r)rtokensrr!rconvert_tokens_to_ids$sz#TextTokenizer.convert_tokens_to_idscCrr rrr rrrconvert_token_to_id'rz!TextTokenizer.convert_token_to_idcCrr )rZ IdToPiece)ridxrrrconvert_id_to_token*rz!TextTokenizer.convert_id_to_tokencC|jSr )rr!rrr__len__-zTextTokenizer.__len__N) __name__ __module__ __qualname__rrrintrrr%r'r)r+rrrrr s r c@seZdZ   d#ddZddZedefd d Zed d Ze d dZ e ddZ ed$de defddZ d%de fddZ   d&de deefddZdeede fddZ   d&de dee fddZdeee ffd d!Zd"S)' SPTokenizer NPTcCs<|dusJ||_||_gd|_||_||_t||_dS)N)[MASK][gMASK]z[sMASK]z zz) vocab_filenum_image_tokensZspecial_tokensmax_blank_length byte_fallbackr text_tokenizer)rr8r9r:r;rrrr3s  zSPTokenizer.__init__cCr*r )r<r!rrr_get_text_tokenizerEr,zSPTokenizer._get_text_tokenizerlengthcCs|dksJd|dS)Nz<|blank_z|>r)r>rrrget_blank_tokenHs  zSPTokenizer.get_blank_tokencCsdS)Nz<|tab|>rrrrr get_tab_tokenMszSPTokenizer.get_tab_tokencC|jjSr )r<rr!rrrnum_text_tokensQszSPTokenizer.num_text_tokenscCs |j|jSr )r9rCr!rrrrUs zSPTokenizer.num_tokensrmax_lencCs<|dt}t|ddD] }|d|t|}q|S)N  )replacer1rAranger@)rrDirrr_encode_whitespacesYszSPTokenizer._encode_whitespacescCs(|r|dd}|r|j||jd}|S)N )rD)rIrLr:)rr linebreak whitespacesrrr _preprocess`s zSPTokenizer._preprocessreturncsN|||}|s d|}|}fdd|D}|r!|S|ddS) @param text: Text to encode. @param linebreak: Whether to encode newline ( ) in text. @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. @param add_dummy_prefix: Whether to add dummy blank space in the beginning. rNcsg|]}|jqSrr9)rxr!rrr"xsz&SPTokenizer.encode..r?N)rQr=r)rrrOrPadd_dummy_prefixtmpr$rr!rrhs  zSPTokenizer.encodetext_idscszfdd|D}dd|D}|}|dd}|td}tdjdD] }||d |}q-|S) Ncsg|] }t|jqSr)r0r9rZ_idr!rrr"|sz&SPTokenizer.decode..cSsg|]}|dkr|qS)rrrYrrrr"}r#rNrMrEr?rFrH)r=rrIr1rArJr:r@)rrXrrrKrr!rr{s zSPTokenizer.decodecCs<||||}|s d|}||}|r|S|ddS)rSrNr?N)rQr=r)rrrOrPrVr$rrrrs  zSPTokenizer.tokenizerUcCst|tr||jkrd|S|j||jSt|tr@|dr7|dr7|dd r7t|ddS|j ||jSt d)Nz zrGzThe key should be str or int.) isinstancer0r9formatr<r)str startswithendswithisdigitr' ValueError)rrUrrr __getitem__s&    zSPTokenizer.__getitem__N)r2r3T)r3)TT)TTT)r-r.r/rr= staticmethodr0r@rApropertyrCrr^rLrQrrrrrrcrrrrr11sF        r1c sVeZdZdZddiZeZgdZ      d4 d5fdd Ze d e e fddZ e d e e fddZ e ddZddZddZddZ  d6dee ee fded ed efd!d"Zd#d$Zd%d&Zd7d'd(Z d7d)ee d*e ee d ee fd+d,Zdejddfd-eeeefefd.e e d/ed0e e d1e ed e f d2d3Z!Z"S)8ChatGLMTokenizera Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding. Args: vocab_file: Path to the vocabulary file. do_lower_case: Use lower case letters. remove_space: Remove spaces. bos_token: The bos token eos_token: The Eos Token end_token: The end token mask_token: The mask token gmask_token: The gmask token padding_side: The padding side num_image_tokens: The `num_image_tokens` in `SPTokenizer` r8zice_text.model)Z input_idsattention_mask position_idsFr6r7r4r5leftr2rRNc sht|| d|_tjd||| |||||| d | ||_||_||_||_||_||_ ||_ ||_ dS)NrT) do_lower_case remove_space padding_side bos_token eos_token end_token mask_token gmask_tokenr9r) r1 sp_tokenizersuperrrkrlr8rnrorprqrr) rr8rkrlrnrorprqrrrmr9kwargs __class__rrrs2   zChatGLMTokenizer.__init__cCs|jdurdS||jSr )rrr%r!rrrgmask_token_ids  zChatGLMTokenizer.gmask_token_idcCs|jdurdS||jS)z `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been set. N)rpr%r!rrr end_token_ids  zChatGLMTokenizer.end_token_idcCrB)z Returns vocab size )rsrr!rrrrszChatGLMTokenizer.vocab_sizecs(fddtjD}|j|S)z Returns vocab as a dict csi|]}||qSr)_convert_id_to_token)rrKr!rr s z.ChatGLMTokenizer.get_vocab..)rJrupdateZadded_tokens_encoder)rZvocabrr!r get_vocabs  zChatGLMTokenizer.get_vocabcCs0|jr d|}n|}|jr|}|S)NrH)rljoinstripsplitrklower)rZinputsZoutputsrrrpreprocess_texts z ChatGLMTokenizer.preprocess_textcKs||}|j|}|S)z Returns a tokenized string. )rrsr)rrruseqrrr _tokenizes  zChatGLMTokenizer._tokenizeT token_idsskip_special_tokensclean_up_tokenization_spacescKsHt|tr|g}t|dkrdS|j|vrtt|jj|}|j|S)Nr) r\r0len pad_token_idlistfilter__ne__rsr)rrrrrurrr_decodes    zChatGLMTokenizer._decodecC |j|S)z2 Converts a token (str) in an id using the vocab. rsr&rrr_convert_token_to_id# z%ChatGLMTokenizer._convert_token_to_idcCr)z=Converts an index (integer) in a token (str) using the vocab.r)rindexrrrrz'rz%ChatGLMTokenizer._convert_id_to_tokencCstj|rtj||jd}n|}t|jd }|}Wdn1s(wYt|d}||Wd|fS1sDwY|fS)a Save the vocabulary and special tokens file to a directory. Args: save_directory (`str`): The directory in which to save the vocabulary. filename_prefix (`str`, *optional*): An optional prefix to add to the named of the saved files. Returns: `Tuple(str)`: Paths to the files saved. r8rbNwb) ospathisdirr~vocab_files_namesopenr8readwrite)rZsave_directoryZfilename_prefixr8ZfinZ proto_strwriterrrrsave_vocabulary+s     z ChatGLMTokenizer.save_vocabulary token_ids_0 token_ids_1cCs|j|j}|j|j}|j|j}||vr||vr||g7}|d|kr4|d|kr4||j|jg7}||j|jg7}|durR|rI|d|krN||g7}||7}|S)a Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERT sequence has the following format: - single sequence: `[CLS] X [SEP]` - pair of sequences: `[CLS] A [SEP] B [SEP]` Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. rGN)rsrqrrrorprn)rrrZmask_idsZ gmask_idsZeos_idrrr build_inputs_with_special_tokensFs     z1ChatGLMTokenizer.build_inputs_with_special_tokensencoded_inputs max_lengthpadding_strategypad_to_multiple_ofreturn_attention_maskcCsb|j|j}|j|j}|j|j}|jdksJ||jd} t| } |tjkr-t| }|durC|durC||dkrC||d|}|tj koMt| |k} |durd|vr|| vr`| |} n| } t d| | f} t | } d| ddddd| f<t | dk} | |d<d|vrt j| t jd}|| vr|n|}|| vr| |}||| d<t t j| t jdt jd| | dt jdg}t j||gdd |d<| r/|t| }d|vrt j|dd |df|dfgd d d |d<d|vr|jg||d|d<d|vrdg||d|d<d|vr"t j|dd |dfgd|d<|jg|| ||jd<|S)a? Pad encoded inputs (on left/right and up to predefined length or max length in the batch) Args: encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). max_length: maximum length of the returned list and optionally padding length (see below). Will truncate by taking into account the special tokens. padding_strategy: PaddingStrategy to use for padding. - PaddingStrategy.LONGEST Pad to the longest sequence in the batch - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) - PaddingStrategy.DO_NOT_PAD: Do not pad The tokenizer padding sides are defined in self.padding_side: - 'left': pads on the left of the sequences - 'right': pads on the right of the sequences pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta). return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics) rjrNrFrgg?rh)Zdtype)Zaxis)rrZconstantT) pad_widthmodeZconstant_valuesZtoken_type_idsZspecial_tokens_mask)r)rsrnrqrrrmmodel_input_namesrr ZLONGEST DO_NOT_PADrnpZonesZtrilZbool_ZarangeZint64Z concatenateZzerosstackpadZpad_token_type_idr)rrrrrrZ bos_token_idZ mask_token_idrxZrequired_inputZ seq_lengthZneeds_to_be_paddedZcontext_lengthrgrhrqZ mask_positionZblock_position_ids differencerrr_padls                 zChatGLMTokenizer._pad) FFr6r7rir4r5rjr2)rRN)FTr )#r-r.r/__doc__r&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESZmax_model_input_sizesrrrerr0rxryrr}rrrrboolr^rrrzrrr rrrrdictr __classcell__rrrvrrfs '         )rf)rrtypingrrrrnumpyrZ sentencepiecer Ztransformers.tokenization_utilsrZ$transformers.tokenization_utils_baserrZtransformers.utilsr Zmodelscope.utilsr loggingZ get_loggerrr r1rfrrrrs   x