o *jn@sjddlZddlmZeZdZddZddZdd Zd d Zd d Z ddZ ddZ ddZ ddZ dS)N) get_loggeruT[’!"#$%&'()*+,-./:;<>=?@，。?★、…【】《》？“”‘’！[\]^_`{|}~]+cCsrg}|}t|dkr7td|}|dur|d}n|dd}||||ddd}t|dks |S)Nrz[A-Za-z!?,<>()\']+ )lowerlenrematchgroupappendreplacestrip) input_strtokenssr wordro/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/audio/kws_utils/file_utils.pysplit_mixed_labels      rcCs$t|}ddd|D}|S)Nrcss|]}|dVqdS)rNr).0subrrr (sz$space_mixed_label..)rjoinr )rZsplitsZ space_strrrrspace_mixed_label&srcCs\g}t|ddd}|D]}|dkr||q Wd|S1s'wY|S)Nrutf8encodingr)openr r )Z list_filelistsfinlinerrr read_lists,s  r"c Csi}|D]+}|dd}t|dkr!td|q||dd||d<qg}|D]6}|dd}t|dkr`|d|vr`|t|d||d|ddd q4td |dq4|S) N rzinvalid line in trans file: {}rrri>)keytxtZwavZ sample_ratez*can't find corresponding trans for key: {}) r r splitrloggerdebugformatr dict)Z wav_listsZ trans_listsZ trans_tabler!arrrrrr make_pair5s0  r-cCs|i}t|ddd'}|D]}|}t|dksJt|dd||d<q Wdn1s3wY||S)Nrrrr$rr)rr r'rintclose)Z token_fileZ tokens_tabler r!r,rrr read_tokenQs r0cCsi}t|ddd)}|D]}|dd}t|dks J|dd||d<q Wdn1s5wY||S) Nrrrr#rr$rr)rr r r'rr/)Z lexicon_file lexicon_tabler r!r,rrr read_lexicon\sr2cCst}t}t|}|D]a}|dks|dks|dkr|d}q |dks'|dkr,|d}q |dks<|d ks<|d ks<|d krA|d }q ||vrK||f}q ||vr\||D]}||f}qSq ttd |}|D]}||f}qeq |D]c}||vr~|||f}qp|dkrd|vr||df}qp||df}qp|dkrd|vr||df}qp||df}qpd|vr||df}td|dqp||df}td|dqp||fS)N!sil(sil))r3)r6(noise)noise)(noise)rsilr<')' is not in token set, replace with )' is not in token set, replace with )tuplerrr symbol_strr(infor& symbol_tabler1Z tokens_strZ tokens_idxpartspartchrrrquery_token_setgsR          rIcCsg}g}t|}|D]d}|dks|dks|dkr|dq |dks&|dkr,|dq |dks<|dks<|dks<|d krB|d q ||vrL||q ||vr]||D]}||qTq ttd |}|D]}||qfq |D]c}||vr|||qq|dkrd |vr||d qq||dqq|d krd |vr||d qq||dqqd |vr||d td |dqq||dtd |dqq||fS)Nr3r4r5r6r7r8r9r:r;r<rr=r>r?r@)rr rrrBr(rCrDrrrquery_token_listsR          rJcCsF|D]}d|vs J|d}t|||\}}||d<||d<q|S)Nr&r)r rJ)Z data_listrEr1sampler&strsZindexsrrrtokenizes   rM)rZmodelscope.utils.loggerrr(rBrrr"r-r0r2rIrJrMrrrrs    1 1