o
    *Îj62  ã                   @   sT  d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZ d dl	Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z) e$ƒ Z*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5ej6ej7dG dd„ deƒƒZ8dS )é    N)ÚCallableÚDictÚOptional)Únn)Úoptim)ÚTrainers)ÚModelÚ
TorchModel)ÚKWSDataLoaderÚ
KWSDataset)ÚBaseTrainer)ÚTRAINERS)Úupdate_conf)ÚDEFAULT_MODEL_REVISIONÚ	ModelFile)Ú	to_device)Úcreate_device)Ú
get_logger)Úget_dist_infoÚget_local_rankÚ	init_distÚ	is_masterZbasetrain_easyZbasetrain_normalZbasetrain_hardZfinetune_easyZfinetune_normalZfinetune_hardÚ
checkpointgš™™™™™¹?g333333ã?g333333Ó?g      à?)Úmodule_namec                       s¶   e Zd ZdZeeeeee	fZ
ddedfdededee dee dee dee f‡ fd	d
„Zdejfdd„Zdd„ Zdd„ Zdd„ Zdd„ Zdedeeef fdd„Zdd„ Z‡  ZS )ÚKWSFarfieldTrainerz
./work_dirNÚmodelÚwork_dirÚcfg_fileÚarg_parse_fnÚmodel_revisionÚcustom_confc                    s„  t |tƒr|  ||¡| _|d u rtj | jtj¡}n|d us"J dƒ‚tj 	|¡| _t
ƒ  ||¡ | dd ¡}|r=|| jj_| jjj| _|  ¡ | _|| _| dd ¡d urYt|d ƒ tƒ \}	}
|
dk| _| dd¡}| jrttƒ }d|› }t|ƒ| _| jjdkr†| j | j¡ d	|vrœt| jjd	ƒs•J d
ƒ‚| jjj| _n|d	 | _| dd ¡| _| dd ¡| _ | jd u rº| jjj!| _| j d u rÅ| jj"j#| _ | jjj$}| dd ¡| _%| j%d u rÚ|j&| _%t'| _(d|v ræ|d | _(|j)| _*| dd¡}|d | _+d|v rtj | j|d ¡}t,j-|dd| _nK| j+dkrXtj | jt.› d| j+d›d¡}t/ /|¡}t0|ƒdkrBt1 2d|d ¡ t,j-|d dd| _nt0|ƒdkrQt3d|› dƒ‚t4d|› ƒ‚| jjj5j6}t7 8| j 9¡ |¡| _5t: ;¡ | _<d | _=tj | jd >| j?¡¡| _@g | _A| jBD ]$}tj | j|¡}tj | j|› d¡}tC|||| ƒ | jA D|¡ qƒtE F| jtG ¡tE F| jtH ¡tE F| jtI ¡f| _Jd S )Nz?Config file should not be None if model is not from pretrained!Únum_synÚlauncheré   ÚdeviceZgpuzcuda:ÚcudaÚ
max_epochsz1max_epochs is missing from the configuration fileÚtrain_iters_per_epochÚval_iters_per_epochÚworkersZsingle_rateÚ
next_epochZ	model_binT)Zweights_onlyr   Ú_Z04dz*.pthz!Loading model from checkpoint: %sz$Failed to load checkpoint file like z. File not found!z7Expecting one but multiple checkpoint files are found: z{}.log.jsonz.conf)KÚ
isinstanceÚstrZget_or_download_model_dirÚ	model_dirÚosÚpathÚjoinr   ZCONFIGURATIONÚdirnameÚsuperÚ__init__ÚgetÚcfgr   r!   Ú_num_classesÚbuild_modelr   r   r   Ú_distr   r   r$   ÚtypeÚtoÚhasattrÚtrainr&   Ú_max_epochsÚ_train_itersÚ
_val_itersr'   Z
evaluationr(   Ú
dataloaderÚ_threadsZworkers_per_gpuÚBASETRAIN_RATIOÚ_single_rateZbatch_size_per_gpuÚ_batch_sizeÚ_current_epochÚtorchÚloadÚCKPT_PREFIXÚglobÚlenÚloggerÚinfoÚFileNotFoundErrorÚAssertionErrorÚ	optimizerÚlrr   ZAdamÚ
parametersr   ZCrossEntropyLossÚloss_fnÚdata_valÚformatÚ	timestampÚjson_log_pathÚ
conf_filesÚ	conf_keysr   ÚappendÚmathÚfloorÚ
EASY_RATIOÚNORMAL_RATIOÚ
HARD_RATIOÚstages)Úselfr   r   r   r   r   r    Úkwargsr!   r+   Z
world_sizeZdevice_nameZ
local_rankZdataloader_configr*   Zmodel_bin_fileZckpt_file_patternZ
ckpt_filesrQ   Zconf_keyZtemplate_fileZ	conf_file©Ú	__class__© úo/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/audio/kws_farfield_trainer.pyr4   2   s´   
	ÿ
ÿ€





ÿþ







ÿ
ÿÿÿ


ÿ

þzKWSFarfieldTrainer.__init__Úreturnc                 C   sB   t j| j| jdd}t|tƒrt|dƒr|jS t|tj	ƒr|S dS )z¸ Instantiate a pytorch model and return.

        By default, we will create a model using config from configuration file. You can
        override this method in a subclass.

        T)Zcfg_dictZtrainingr   N)
r   Zfrom_pretrainedr.   r6   r,   r	   r<   r   r   ÚModule)ra   r   re   re   rf   r8   ›   s   
ÿÿzKWSFarfieldTrainer.build_modelc                 O   s‚   | j s|  ¡  t d¡ tj ¡ }d}t| jƒD ]\}}||7 }|| j }|  	||¡ qtj ¡ | }t d 
| ¡ d ¡¡ d S )NzStart training...r   zTotal time spent: {:.2f} hours
ç      ¬@)rT   Úgen_valrL   rM   ÚdatetimeÚnowÚ	enumerater`   rF   Ú	run_stagerU   Útotal_seconds)ra   Úargsrb   Z	totaltimeZnext_stage_head_epochÚstageZ	num_epochÚepochs_to_runre   re   rf   r=   ©   s   




ÿzKWSFarfieldTrainer.trainc              
   C   s8  |dkrt  d|› d¡ dS t  d|› d¡ |  | j|d  | j|d d  ¡\}}t|ƒ}t|ƒD ]Ô}|  jd7  _tj 	¡ }t  d	| j¡ d
}d}	t| j
ƒD ]e}
t|ƒ\}}t |d¡}t|| jƒ}t|| jƒ}| j ¡  |  |¡}|  t |d| jf¡|¡}t | ¡ ¡s| ¡  | j ¡  || ¡ 7 }|	d7 }	d | j| j|
d | j
| ¡ ¡}t  |¡ |  |¡ qR||	 }|  d¡}d | j||¡}t  |¡ |  |¡ d t| j||¡}tj  !| j"|¡}t  d|› ¡ t #| j|¡ tj 	¡ | }t  d | j| $¡ d ¡¡ q4| %¡  | &¡  t  d|› d¡ dS )z¬
        Run training stages with correspond data

        Args:
            stage: id of stage
            epochs_to_run: the number of epoch to run in this stage
        r   zInvalid epoch number, stage z exit!NzStarting stage z...é   r#   zStart epoch %d...ç        ©éÿÿÿÿrv   z8Epoch: {:04d}/{:04d}, batch: {:04d}/{:04d}, loss: {:.4f}Ú z<Evaluate epoch: {:04d}, loss_train: {:.4f}, loss_val: {:.4f}z/{}_{:04d}_loss_train_{:.4f}_loss_val_{:.4f}.pthzSave model to z%Epoch {:04d} time spent: {:.2f} hoursri   zStage z is finished.)'rL   ÚwarningrM   Úcreate_dataloaderrX   ÚiterÚrangerF   rk   rl   r?   ÚnextrG   Úreshaper   r$   rP   Z	zero_gradr   rS   r7   ÚnpÚisnanÚitemZbackwardÚsteprU   r>   Ú	_dump_logÚevaluaterI   r/   r0   r1   r   Úsavero   ÚstopÚrelease)ra   rq   rr   ÚdatasetrA   Úitr+   Z	epochtimeZloss_train_epochZvalidbatchsÚbiÚfeatÚlabelÚpredictÚlossZtrain_resultÚloss_val_epochZ
val_resultZ	ckpt_nameZ	save_pathre   re   rf   rn   º   sr   ÿ


ÿ

þ

ÿ

þ
þzKWSFarfieldTrainer.run_stagec           	      C   s6  t j | jd¡}| jdkr4t d¡ t|dƒ}t 	|¡| _
W d  ƒ n1 s(w   Y  t d¡ dS t d¡ |  | jd | jd	 ¡\}}t|ƒ}g | _
t| jƒD ]}t d
|¡ t|ƒ\}}t |d¡}| j
 ||g¡ qS| ¡  | ¡  t|dƒ}t | j
|¡ W d  ƒ n1 sw   Y  t d¡ dS )z)
        generate validation set
        zval_dataset.binr   zStart loading validation set...ÚrbNzFinish loading validation set!z"Start generating validation set...rs   é   zIterating validation data %dru   Úwbz!Finish generating validation set!)r/   r0   r1   r   rF   rL   rM   ÚopenÚpicklerH   rT   ry   rX   rz   r{   r@   r|   rG   r}   rZ   r…   r†   Údump)	ra   Zval_dump_fileÚfr‡   rA   rˆ   r‰   rŠ   r‹   re   re   rf   rj   ý   s2   

ÿ

ÿÿzKWSFarfieldTrainer.gen_valc                 C   s8   t ||| j| j| jƒ}t|| j| jd}| ¡  ||fS )N)Z	batchsizeZ
numworkers)r   rB   rD   r7   r
   rE   Ústart)ra   Ú	base_pathZfinetune_pathr‡   rA   re   re   rf   ry     s   
ÿ
ÿz$KWSFarfieldTrainer.create_dataloaderÚcheckpoint_pathc           	   	   O   s¢   t  d¡ d}t ¡ 4 | jD ](\}}t|| jƒ}t|| jƒ}|  |¡}|  t 	|d| j
f¡|¡}|| ¡ 7 }qW d   ƒ n1 sBw   Y  t  d¡ || j S )NzStart validation...rt   rv   zFinish validation.)rL   rM   rG   Zno_gradrT   r   r$   r   rS   r}   r7   r€   r@   )	ra   r˜   rp   rb   rŽ   rŠ   r‹   rŒ   r   re   re   rf   rƒ   $  s   


ÿøÿ


zKWSFarfieldTrainer.evaluatec                 C   sP   t ƒ r&t| jdƒ}| |¡ | d¡ W d   ƒ d S 1 sw   Y  d S d S )Nza+Ú
)r   r’   rW   Úwrite)ra   Úmsgr•   re   re   rf   r‚   6  s   
"þÿzKWSFarfieldTrainer._dump_log)Ú__name__Ú
__module__Ú__qualname__ZDEFAULT_WORK_DIRÚBASETRAIN_CONF_EASYÚFINETUNE_CONF_EASYÚBASETRAIN_CONF_NORMALÚFINETUNE_CONF_NORMALÚBASETRAIN_CONF_HARDÚFINETUNE_CONF_HARDrY   r   r-   r   r   Údictr4   r   rh   r8   r=   rn   rj   ry   r   Úfloatrƒ   r‚   Ú__classcell__re   re   rc   rf   r   +   s@    þúÿþýüûúiC

ÿr   )9rk   rJ   r[   r/   r“   Útypingr   r   r   Únumpyr~   rG   r   r   Zmodelscope.metainfor   Zmodelscope.modelsr   r	   Z7modelscope.msdatasets.dataset_cls.custom_datasets.audior
   r   Zmodelscope.trainers.baser   Zmodelscope.trainers.builderr   Z"modelscope.utils.audio.audio_utilsr   Zmodelscope.utils.constantr   r   Zmodelscope.utils.data_utilsr   Zmodelscope.utils.devicer   Zmodelscope.utils.loggerr   Zmodelscope.utils.torch_utilsr   r   r   r   rL   rŸ   r¡   r£   r    r¢   r¤   rI   r]   r^   r_   rC   Zregister_moduleZspeech_dfsmn_kws_char_farfieldr   re   re   re   rf   Ú<module>   sF    