o *╬jу@sNddlZddlZddlZddlZd ddДZd ddДZddДZGdd Дd ГZdS)щNcCsPi}tаб|d<tjаб|d<tаб|d<|dur&|jdkr&tjа|б|d<|SйNZrng_state_torchZrng_state_npZ rng_state_rnd┌cudaZrng_state_torch_cuda)┌torchZ get_rng_state┌np┌randomZ get_state┌getstate┌typer)┌device┌ random_statesйr·e/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/nlp/peer/sas_utils.py┌get_random_statess r cCsZtа|dбtjа|dбtа|dб|dur)|jdkr+tjа|dбdSdSdSr)rZ set_rng_staterrZ set_state┌setstaterr)r r rrr┌set_random_statess rcCs|durdSddg}tа|бr&tа|бабrd|d<tа|бабr%d|d<nTt|Гturztt|ГГD]+}tа||бr]tа||бабrL|dd7<tа||бабr]|dd7<q2|ddkrl|dd7<|ddkrz|dd7<t |ГdkrВ|SdS)Nrщщ ) rZ is_tensor┌isnan┌any┌isinfr┌tuple┌range┌len┌sum)┌data┌result┌irrr┌ check_nan_inf%s, ААrc@s&eZdZdddДZddДZddДZdS) ┌SequenceSideInfoNcsФ|dur|И_nddlm}|аdбИ_tjjабИ_ЗfddДt Иjj ГDГЙtЗfddДt tИГГDГГИ_ ddДИDГ}tj|tjdНИ_dS) Nr)┌ElectraTokenizerzgoogle/electra-small-generatorcsg|] }Иjа|gбСqSr)┌ tokenizer┌decodeй┌.0r)┌selfrr┌ Ls z-SequenceSideInfo.__init__..cs$g|]}И|ddЕdkr|СqS)rщ·##rr!)┌tokensrrr$Qs$cSs\g|]*}|ddkr|ddkrdn|ddЕdkr%dtdt|ГdГntdt|ГГСqS)r·[щ ·]r%r&rщ)┌minr)r"┌trrrr$Rs¤ ¤йZdtype)rZtransformersrZfrom_pretrained┌nltk┌tokenizeZpunktZPunktSentenceTokenizer┌ sen_tokenizerrZ vocab_size┌setr┌ ind_subtokensr┌tensor┌int8┌ len_tokens)r#rr┌tmpr)r#r'r┌__init__Bs" ■ №zSequenceSideInfo.__init__csТ|jа|б}tаddД|jа|бdDГбЙ|ИddЕабИd<tаЗfddДtt ИГГDГб}tаЗfddДtt ИГГDГб}tа||fбS)NcSsg|]}t|ГdСqS)r%)r)r"┌xrrrr$]s z1SequenceSideInfo.getSenTokIdx..Z input_idsrrcs$g|]}|tjИ|tjdНСqSйr.)rZonesr5r!йZsen_lengthsrrr$ds cs g|]}tjИ|tjdНСqSr:)rZaranger5r!r;rrr$hs ) r1r0r┌arrayrZbatch_encode_plusrZconcatenaterr)r#┌sentence_position_embedding┌ inputs_strZ seq_len_totalZ sentencesZidx_senZidx_tokrr;r┌getSenTokIdxZs ■ ■ ■zSequenceSideInfo.getSenTokIdxcsd}tИdttjfГrd}tаИбЙtИjdГr4ИjаИб}tjtа ЗЗЗfddД|DГбИj dН}ntjtа ЗЗЗfddДИабDГбИj dН}tГ}Иj d }|ddЕd|Еf|d <|ddЕd |d|Еf|d<ИdkrюtjИdd Н\}} Иjаt|Гб} t| ГdkrЬtаЗfddД| DГбjddНаб}n tjИj tjdН}d|ddЕdf<|ddЕd dЕf}td dГD]"} tа|| k|ddЕddЕf| kб}t|Гdkr╪n| d ||<q╝||d<ИjИаб}||d<|r |абD] }||аб||<qЇ|S)NFrT┌batch_decodecs g|]}ИаИ|ИjdбСqSйr)r?┌shape)r"Z input_strй┌ inputs_idr#r=rrr$xs■ z;SequenceSideInfo.generate_seq_side_info..)r cs(g|]}ИаИИjа|бИjdбСqSrA)r?rr rB)r"Z input_orirCrrr$Аs¤ ■ rZ ss_sentence_position_in_sequencer%Zss_token_position_in_sentence)Zreturn_inversecsg|]}И|kСqSrr)r"┌st)rDrrr$Цs )Zaxisr.щr)Zss_token_position_in_whole_wordZss_token_string_length)┌ isinstance┌listrZndarrayrr4┌hasattrrr@r<r ┌numpy┌dictrB┌uniquer3┌intersectionr2r┌stackr┌charZzerosr5r┌logical_andr6┌long┌keys)r#r=rDZis_np_arrayr>Zsen_tok_idxZside_info_dictZ seq_lengthrL┌_r3Z idx_tok_wwZidx_tok_ww_1r┌posZinputs_str_len┌keyrrCr┌generate_seq_side_infoost ¤·№∙ ■■ z'SequenceSideInfo.generate_seq_side_infoйN)┌__name__┌ __module__┌__qualname__r8r?rVrrrrr@s rrW) rr/rJrrr rrrrrrr┌s