o *Îj1Vã@s€ddlZddlZddlZddlZddlmZmZmZddlZddl m Z ddl Z ddl mZddlmZddlmZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZm Z dd l!m"Z"ddl#m$Z$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-m.Z.m/Z/ddl0m1Z1ddl2m3Z3m4Z4m5Z5ddl6m7Z7m8Z8m9Z9e)ƒZ:ej;ej¡|d%|d0d1}t}tBjB C¡}tD|j*|ƒD]ž}|j E|¡||d <|jFdd%}t  :d? "||¡¡tG|j||j|j.||ƒtH|j|j|j.|ƒ\}}t  :d@ "|||¡¡|j dkrtj |jdA "|¡¡}tA|j|dddd dƒt$ %dd|¡} tI|||dB}t&| d+ƒ}t' 7|¡}| 9|¡Wdƒn 1s wY| JdC||¡| JdD||¡|}| K|¡q‡|dur5|j dkr5| L¡tBjB C¡|}t  :dE "| M¡dF¡¡dS)Gau Args: kwargs: train_data (int): wave list with kaldi style for training cv_data (int): wave list with kaldi style for cross validation trans_data (str): transcription list with kaldi style, merge train and cv checkpoint (str): basemodel checkpoint, if None, default to use base.pt in model path tensorboard_dir (str): path to save tensorboard results, create 'tensorboard_dir' in work_dir by default need_dump (bool): wether to dump data with mapping tokens or not Ú checkpointNz train/base.ptÚtensorboard_dirZ tensorboardÚ train_dataz'please config train data in dict kwargsÚcv_dataz$please config cv data in dict kwargsÚ trans_dataz/please config transcription data in dict kwargsÚ need_dumpFrTÚ preprocessorÚ speed_perturbÚspec_augÚshufflezdump_train.txtz dump_cv.txtÚ pin_memoryÚprefetché)Ú batch_sizer[Ú num_workersÚprefetch_factorzthe number of model params: {}z.pt$z.yamlÚr)ÚLoaderz'Training with random initialized paramsÚepochéÿÿÿÿrr8Ú start_epochÚlrÚ optimizerÚdeviceÚgpuúcuda:r+r,zStart training...)Úexist_okÚ log_intervalé Ú weight_decay)rfrnÚmingà?égíµ ÷Æ°>g{®Gáz„?)ÚmodeÚfactorZpatienceZmin_lrÚ thresholdzinit.ptZ grad_clipZ grad_accumÚ world_sizer;Ú local_rankZ max_epochszEpoch {} TRAIN info lr {}z0Epoch {} EVAL info cv_loss {:.6f}, cv_acc {:.2f}z{}.pt)rcrfÚcv_lossz epoch/cv_losszepoch/lrúTotal time spent: {:.2f} hoursç ¬@)Nr9r0r1r<rQr2r/rRrSrTrUr;rVrBÚcopyÚdeepcopyr%r r@rAZ train_datasetZ cv_datasetrr8Ú dataloaderÚworkers_per_gpuZtrain_dataloaderÚ evaluationZ cv_dataloaderÚ build_modelr$r!r>ÚwarningÚformatrÚreÚsubrCrDÚloadZ FullLoaderrertrurZ train_deviceÚtorchÚcudaÚ is_availableÚtorZparallelZDistributedDataParallelr=rErFrGr?rrZAdamÚ parametersÚ lr_schedulerZReduceLROnPlateaurÚdatetimeÚnowÚrangeZ set_epochZ param_groupsrrÚdictZ add_scalarÚstepÚcloseÚ total_seconds) rHÚargsrIZtrain_checkpointZ train_confZcv_confZdump_train_fileZ dump_cv_fileZ num_paramsZ info_pathÚinfosZfinZ lr_last_epochÚ device_namerJrKrLÚwriterrlZ optim_confrgr‰Z final_epochZsave_model_pathZtraining_configZ max_epochÚ totaltimercrfrvZcv_accZ info_dictrOrOrPr8ksX ÿÿ   ÿÿ   þ ü  û   ÿû   ÿ€ ÿ         þ  ÿýú ÿ        ÿ ÿÿÿ  ÿÿý   þ  ÿzKWSNearfieldTrainer.trainÚcheckpoint_pathÚreturnc sŠd|_|durtj |¡rt d|›¡|}nU|jdura| dd¡}tj |jd|›d¡|_t d|j›¡t |j|jd|d }t d>i|¤Ž|_|  |j ¡}t ||j|jƒ}t d |›¡|j}t d |j›¡| d d¡dur‹| d d¡dur‹t d¡|d } |d } n t d¡|j} |j} t d| ›¡t d| ›¡t |j d¡} d| dd<d| dd<d| d<d| d<d| d<| dd¡durÖ|d| dd<t| | | |j|jdddƒ} t| d| dd¡d|j jjj|j jj d d!¡d"} | d#d¡dus Jd$ƒ‚|d#}| ¡ d%d¡ d&¡}i}dh‰d'h‰d'di}|D]O}t||j|jƒ\}}i||<|||d(<d d)d*„|Dƒ¡||d+<‡fd,d-„|Dƒ‡fd.d-„|Dƒt||ƒD]\}}| |d¡durr|||<qaq&d}| ¡D]\}}||›d/|›d07}q|t d1|›¡| d2d3¡dko¡tj  !¡}| d4d5¡}|j"d6kr·|r·d7|j#›}t$|ƒ|_%|  |j ¡|_&t'||j&ƒ|j& (|j%¡|_&i}| d8d¡durã|d8|d8<ntj )|¡}tj |jd9|¡|d8<|d8|_*tj |j*¡st +|j*¡t ,d:¡t-j- .¡}t/|j&| |j%|ˆ|ƒ}t-j- .¡|}t ,d; 0| 1¡d<¡¡t || | |d=}t2d>i|¤Ž}t3|ƒdS)?ab Args: checkpoint_path (str): evaluating with ckpt or default average ckpt kwargs: test_dir (str): local path for saving test results test_data (str): wave list with kaldi style trans_data (str): transcription list with kaldi style average_num (int): the NO. to do model averaging(checkpoint_path==None) batch_size (int): batch size during evaluating keywords (str): keyword string, split with ',' gpu (int): evaluating with cpu/gpu: -1 for cpu; >=0 for gpu Nz evaluating with specific model: Z average_numrmZavg_z.ptz!default average model not exist: T)Z dst_modelZsrc_pathZval_bestÚavg_numz(average model convert to kaldi network: zevaluating with average model: Ú test_datarUz/evaluating with specific data and transcriptionz5evaluating with cross validation data during trainingz test data: z trans data: rWiZ filter_confÚ max_lengthrZ min_lengthFrXrYrZr^Z batch_confÚr[r\r])r^r[Zpersistent_workersr_r`Úkeywordszat least one keyword is neededú ú,zZtoken_idcss|] }dt|ƒVqdS)z%s N)r.©Ú.0ÚirOrOrPÚ ‘s€ÿz/KWSNearfieldTrainer.evaluate..Z token_strcóg|]}ˆ |¡‘qSrO©ÚaddrŸ)Úkeywords_strsetrOrPÚ “óz0KWSNearfieldTrainer.evaluate..cr£rOr¤rŸ)Úkeywords_idxsetrOrPr§”r¨ú(z) zToken set is: rirdrhÚcpurrjÚtest_dirZtest_zStart evaluating...rwrx)rœr™rUÚ score_filerO)4Zavg_checkpointr0r1r<r>rr9r2r%rrr~rBr rTrUryrzr r@rArr}r{r|ÚstripÚreplaceÚsplitrÚzipÚitemsr„r…r†rtrurZ test_deviceZ test_modelrr‡Úbasenamer¬r=r?rŠr‹rr€rrÚprint) rHr–r‘rIZeval_checkpointr˜Z avg_kwargsZ model_cvtZ kaldi_cvtr™rUZ test_confZ test_datasetZtest_dataloaderZ keywords_strZ keywords_listZkeywords_tokenZkeywords_tokenmapÚkeywordÚstrsZindexesÚtxtÚidxZ token_printZuse_cudar“Ztesting_configÚ base_namer•r­Z det_kwargsZ det_resultsrO)r©r¦rPÚevaluate-sÿ    ÿ ÿü ýÿ ÿ  ÿ  ý   ÿú ÿÿÿÿ  ÿ€þ      ÿ    þ ÿü zKWSNearfieldTrainer.evaluatecCs@tj|j|dd}t|tƒrt|dƒr|jSt|tjƒr|SdS)z¸ Instantiate a pytorch model and return. By default, we will create a model using config from configuration file. You can override this method in a subclass. T)Zcfg_dictZtrainingr$N) r Zfrom_pretrainedr/r-r Úhasattrr$rÚModule)rHrBr$rOrOrPr~Êsÿ ÿzKWSNearfieldTrainer.build_modelrc CsFt dd¡dur dtjd<t dd¡durdtjd<t dd¡dur'dtjd<t dd¡dur4dtjd<t dd¡durAd tjd<ttjdƒ|_ttjdƒ|_ttjdƒ|_tjd|_tjd|_|d kr€|jd krt   d   |j¡¡t j |jd d n|d krt j |jd d t  ¡t   d  |j|j|j|j|j¡¡dS)NZRANKÚ0Z LOCAL_RANKZ WORLD_SIZEÚ1Z MASTER_ADDRÚ localhostZ MASTER_PORTZ29500rz'init dist on multiple gpus, this gpu {}zenv://)ÚbackendZ init_methodz-RANK {}/{}/{}, Master addr:{}, Master port:{})r0ÚgetenvÚenvironÚintr;rurtZ master_addrZ master_portr>r?r€ÚdistZinit_process_groupr)Úbarrier©rHZ train_nodesrOrOrPr:ØsB        ÿÿ€ÿ þzKWSNearfieldTrainer.init_distcCsT|dkr|jdkrt d |j¡¡t ¡dSdS|dkr(t ¡t ¡dSdS)Nrz*destory dist on multiple gpus, this gpu {})rtr>r?r€rurÄZdestroy_process_grouprÅrÆrOrOrPÚ uninit_distùs ÿÿ ü þzKWSNearfieldTrainer.uninit_dist)r)Ú__name__Ú __module__Ú __qualname__rr.rrr5r8rÚfloatrºrr¼r~r:rÇÚ __classcell__rOrOrMrPr#%s0ûÿþýüûBC  ÿ !r#)>ryrŠr0rÚtypingrrrr„Ztorch.distributedÚ distributedrÄrDZ tensorboardXrrrZtorch.utils.datarZmodelscope.metainfor Zmodelscope.modelsr r ZMmodelscope.msdatasets.dataset_cls.custom_datasets.audio.kws_nearfield_datasetr Zmodelscope.trainers.baser Zmodelscope.trainers.builderrZmodelscope.utils.checkpointrrZmodelscope.utils.configrZmodelscope.utils.constantrrZmodelscope.utils.devicerZmodelscope.utils.loggerrZmodelscope.utils.torch_utilsrZkws_utils.batch_utilsrrrZkws_utils.det_utilsrZkws_utils.file_utilsrrrZkws_utils.model_utilsrr r!r>Zregister_moduleZ"speech_kws_fsmn_char_ctc_nearfieldr#rOrOrOrPÚs@              ÿ