o 0j'@sddlZddlZddlZddlZddlmZmZmZm Z m Z m Z ddl m Z ddlmZdgZddiZd Zd Zd Zd Zed dedDZeeefeZdZddZdedeeeffddZGddde Z dS)N) CollectionDictListSetTupleUnion)PretrainedTokenizer) AddedToken QWenTokenizer vocab_file qwen.tiktokenzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+z <|endoftext|>z <|im_start|>z <|im_end|>ccs|] }d|dVqdS)z<|extra_z|>N.0irry/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddlex/inference/models/common/tokenizer/qwen_tokenizer.py $rcCstjdduS)Ntiktoken) importlibutil find_specrrrris_tiktoken_available.srtiktoken_bpe_filereturncCsNt|d }|}Wdn1swYdddd|DDS)NrbcSs i|] \}}t|t|qSr)base64 b64decodeint)rtokenrankrrr 5sz&_load_tiktoken_bpe..css|] }|r|VqdSN)split)rlinerrrr7rz%_load_tiktoken_bpe..)openread splitlines)rfcontentsrrr_load_tiktoken_bpe2s  r,c seZdZdZgdZeZ   d2fdd Zdefd d Z de e effd d Z d e e eee e effdeefddZd3d eededefddZ d3de eeeefdedefddZdedeefddZ  d4dede eefde eefdee e effdd Zd ee e efdefd!d"Zed#d$Zd%ede e effd&d'Zd(e e efdefd)d*Zdefd+d,Z  d5d-e eeefd.ed/edefd0d1Z Z!S)6r zQWen tokenizer.)Z input_idsZattention_maskZ position_idsreplaceleftNc  sFtjdi|tstdddl}|}||_t||_ddtt t |jdD|_ |r5|j ||j dt|j|j d}t |jt |j |jks`Jt |jt |j d|jd d d|jD|_|j d d|j D||_|jj|_|j t|_|j t|_d |vr|d |_d |vr|d |_dSdS)NzFtiktoken is not installed, please install it use: pip install tiktokenrcSi|]\}}||qSrr)rindexr!rrrr#Vsz*QWenTokenizer.__init__..)startQwenpat_strmergeable_ranksspecial_tokensz != z in encodingcSr/rrrkvrrrr#is cSr/rrr7rrrr#ls pad_token_id eos_token_idr)super__init__r ValueErrorrerrorsr,r5 enumerateSPECIAL_TOKENSlenr6updateEncodingPAT_STRn_vocabitemsdecoder tokenizerZ eot_tokeneod_idIMSTARTZ im_start_idIMENDZ im_end_idr:r;) selfr r?Z padding_sideZextra_special_tokenskwargstkrenc __class__rrr=AsL        zQWenTokenizer.__init__rcC|jjSr$rIrFrMrrr__len__yszQWenTokenizer.__len__cCs|jSr$)r5rUrrr get_vocab|szQWenTokenizer.get_vocabtokenscCsng}t|ttfr||jvr|j|S|j|S|D]}||jvr+||j|q||j|q|Sr$) isinstancestrbytesr6r5getappend)rMrXZidsr!rrrconvert_tokens_to_idss    z#QWenTokenizer.convert_tokens_to_idsFr6cCs|r)rMrXr6Z added_tokensr!Ztoken_idrrrr_update_tiktokens$    zQWenTokenizer._update_tiktoken new_tokenscCsH|s|rtdg}|D]}t|tr|jn|}||q |||S)Nr_)r>rYr contentr]r`)rMrar6Znew_tokens_strr!Z surface_formrrr _add_tokenss  zQWenTokenizer._add_tokenssave_directorycKstj|d}t|ddd)}|jD]\}}t|ddt |d}| |qWd|fS1s:wY|fS)z Save only the vocabulary of the tokenizer (vocabulary). Returns: `Tuple(str)`: Paths to the files saved. r wutf8)encoding  N) ospathjoinr'r5rGr b64encodedecoderZwrite)rMrdrN file_pathrer8r9r&rrrsave_vocabularys   zQWenTokenizer.save_vocabularyallrtextallowed_specialdisallowed_specialcKs<g}td|}|jj|||dD] }||j|q|S)a Converts a string in a sequence of tokens. Args: text (`str`): The sequence to be encoded. allowed_special (`Literal["all"]` or `set`): The surface forms of the tokens to be encoded as special tokens in regular texts. Default to "all". disallowed_special (`Literal["all"]` or `Collection`): The surface forms of the tokens that should not be in regular texts and trigger errors. Default to an empty tuple. kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific encode method. Returns: `List[bytes|str]`: The list of tokens. NFC)rtru) unicodedata normalizerIencoder]rH)rMrsrtrurNrXtrrrtokenizes  zQWenTokenizer.tokenizecCsxd}d}|D]'}t|tr |r||jd|jd7}d}||7}qt|tr*||7}qtd|r:||jd|jd7}|S)zC Converts a sequence of tokens in a single string. zutf-8r?z)token should only be of type types or str)rYrZrnr?r[ TypeError)rMrXrstemprzrrrconvert_tokens_to_strings    z&QWenTokenizer.convert_tokens_to_stringcCrSr$rTrUrrr vocab_sizeszQWenTokenizer.vocab_sizer0cCs||jvr |j|Std)z2Converts an id to a token, special tokens includedz unknown ids)rHr>)rMr0rrr_convert_id_to_tokens  z"QWenTokenizer._convert_id_to_tokenr!cCs0||jvr |j|S||jvr|j|Std)zBConverts a token to an id using the vocab, special tokens includedz unknown token)r6r5r>)rMr!rrr_convert_token_to_ids    z"QWenTokenizer._convert_token_to_idcKst)a Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Do NOT take care of added tokens. )NotImplementedError)rMrsrNrrr _tokenizeszQWenTokenizer._tokenize token_idsskip_special_tokensr?c s<t|tr|g}|rfdd|D}jj||pjdS)Ncsg|] }|jkr|qSr)rJrrUrr "sz)QWenTokenizer._decode..r~)rYr rIrnr?)rMrrr?rNrrUr_decodes zQWenTokenizer._decode)r-r.N)F)rrr)FN)"__name__ __module__ __qualname____doc__Zmodel_input_namesVOCAB_FILES_NAMESZresource_files_namesr=r rVrr[rWrrZrr^boolr`r rcrrqrrr{rpropertyrrrrr __classcell__rrrQrr ;sj8     $  )!rimportlib.utilrrjrwtypingrrrrrrZtokenizer_utilsr Ztokenizer_utils_baser __all__rrEZ ENDOFTEXTrKrLtuplerangeZEXTRASrArrrZr[r r,r rrrrs0