o
    *js                     @   s   d dl Z d dlZd dlmZ d dlZd dlZd dlZd dlm	Z	 d dl
m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ G d	d
 d
eZG dd deZdS )    N)OrderedDict)AdamW)tqdm)get_linear_schedule_with_warmup)MetricsTracker)	ModelFile)
get_loggerc                   @   sd   e Zd Z				dddZdd Z			dddZdd	 ZdddZdddZdddZ	dd Z
dS )TrainerNc                 C   sL  || _ || _|j| _|j| _|jjd dk| _|jjdd  | _|jj| _|jj| _|jj	| _	|jj
| _
|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|| _ | jdkrm| j jn| j | _|| _|j| _|| _|| _|pt | _t | _t | _ t | _!t | _"t#| jrdnd| _$d| _%d| _&d S )Nr   -   infz-inf)'model	to_tensordo_traindo_inferr	   valid_metric_nameis_decreased_valid_metric
num_epochssave_dir	log_stepsvalid_stepssave_checkpointZsave_summaryZDatasetlearning_methodModelweight_decaywarmup_stepsbatch_size_labelbatch_size_nolabelgpulrmodule
func_modelreader	tokenizerlr_scheduler	optimizerr   loggerr   batch_metrics_tracker_labeltoken_metrics_tracker_labelbatch_metrics_tracker_nolabeltoken_metrics_tracker_nolabelfloatbest_valid_metricepoch	batch_num)selfr   r   configr"   r&   r$   r%    r1   u/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/nlp/space/trainer/intent_trainer.py__init__   sN   













zTrainer.__init__c                    s  ddg  fdd| j  D | jd fdd| j  D ddg}t|| jd}|| j }| jd	kr7| jnt|d
 }t|||d}|| _	|| _
| jd| j d | jd| j | jd| | jd| j | jd| j | jd| | jd| | jd dS )z
        Setup the optimizer and the learning rate scheduler.

        from transformers.Trainer

        parameters from cfg: lr (1e-3); warmup_steps
        Zbiasznorm.weightc                    s*   g | ]\ }t  fd dD s|qS )c                 3       | ]}| v V  qd S Nr1   .0ndnr1   r2   	<genexpr>V       4Trainer.set_optimizers.<locals>.<listcomp>.<genexpr>anyr7   pZno_decayr9   r2   
<listcomp>T       z*Trainer.set_optimizers.<locals>.<listcomp>)paramsr   c                    s*   g | ]\ }t  fd dD r|qS )c                 3   r4   r5   r1   r6   r9   r1   r2   r;   ^   r<   r=   r>   r@   rB   r9   r2   rC   \   rD           )r   r   g?)num_warmup_stepsnum_training_stepsz***** Running training: z *****z  Num Epochs = %dzC  Num Training steps(one turn in a batch of dialogs) per epoch = %dz"  Batch size for labeled data = %dz$  Batch size for unlabeled data = %dz  Total optimization steps = %dz  Total warmup steps = %dz$************************************N)r   named_parametersr   r   r   r   r   intr   r%   r$   r&   infor   r   r   )r/   Znum_training_steps_per_epochZoptimizer_grouped_parametersr%   rH   rG   r$   r1   rB   r2   set_optimizersH   sR   	

	
zTrainer.set_optimizersc                 C   s0   | j | j }t|D ]}| j||||d q
d S )N)train_label_itertrain_nolabel_itervalid_label_itervalid_nolabel_iter)r   r-   rangetrain_epoch)r/   rM   rN   rO   rP   r   r-   r1   r1   r2   train   s   zTrainer.trainc                 C      t )!
        Train an epoch.
        NotImplementedError)r/   rM   rN   rO   rP   r1   r1   r2   rR      s   zTrainer.train_epochTc                 C   rT   r5   rV   )r/   data_label_iterdata_nolabel_iterZ	need_saver1   r1   r2   evaluate      zTrainer.evaluatec                 C   rT   r5   rV   )r/   	data_iternum_batchesr1   r1   r2   infer   r[   zTrainer.inferFc              	   C   s.  | j | j| j| j d}| jdur| j |d< | jrYtj	| j
d| j  d}t| j | | jd| d tj	| j
d| j  d}t|| | jd	| d |rtj	| j
tj}t| j | tj	| j
d
tj}t|| | jd| d| j  d| jd dS dS )z save )r-   r.   r,   r%   Nr$   Zstate_epoch_z.modelzSaved model state to ''.trainzSaved train state to 'z{}.trainzSaved best model state to 'z' with new best valid metric =.3f)r-   r.   r,   r%   Z
state_dictr$   r   ospathjoinr   torchsaver   r&   rK   r   ZTORCH_MODEL_BIN_FILEformatr   upper)r/   is_bestZtrain_stateZ
model_file
train_fileZbest_model_fileZbest_train_filer1   r1   r2   rg      sH   




zTrainer.savec                    s^    fdd} fdd} j jdu r jd dS  jr"|  dS  jr-|  |  dS dS )z load c                     sd  t j jj dd d} dt|  d v r6t }|  D ]\}}|d d dks+J |||dd  < q|} t }dd  j D }|  D ]\}}||v r|j	|| j	krt
|d	s^J | }tjj jj|| j	d
d}|dkr|||j	d  d < td| d n-|j	d |j	d k r|d |j	d  }td| d n||d |j	d < td| d |j|j}	}
t j||	|
d}|||< qG|||< qGtd| d qG|} |D ]A}|| vr|| jrtd| d tjj jj|| j	d
d}|| j|| j}	}
t j||	|
d| |< q|| | |< q܈ j|   jd jj d d S )Nc                 S      | S r5   r1   Zstoragelocr1   r1   r2   <lambda>       z9Trainer.load.<locals>._load_model_state.<locals>.<lambda>Zmap_locationzmodule.r      c                 S   s   i | ]\}}||qS r1   r1   )r7   nameparamr1   r1   r2   
<dictcomp>   s    z;Trainer.load.<locals>._load_model_state.<locals>.<dictcomp>numpy)scalesizeZfloat32zembedder.token_embedding.weightzpart of parameter(z) random normalize initializez) are dropped)dtypedevicez
parameter(zLoaded model state from 'z.model')rf   loadr!   init_checkpointlistkeysr   itemsrI   shapehasattrrv   nprandomnormalZinitializer_rangeZastypeprintry   rz   ZtensorZrequires_gradload_state_dictr&   rK   )Zmodel_state_dictZnew_model_state_dictkv
parametersrs   rt   Zarrzry   rz   r/   r1   r2   _load_model_state   s   





z'Trainer.load.<locals>._load_model_statec               	      s    j j d} tj| rWtj| dd d}|d  _|d  _ j	d ur1d|v r1 j	
|d   jd urBd|v rB j
|d   jd	|  d
 j d jdd d S  jd d S )Nr`   c                 S   rl   r5   r1   rm   r1   r1   r2   ro     rp   z9Trainer.load.<locals>._load_train_state.<locals>.<lambda>rq   r-   r,   r%   r$   zLoaded train state from 'z' with (epoch-z best_valid_metric=rb   )zLoaded no train state)r!   r|   rc   rd   existsrf   r{   r-   r,   r%   r   r$   r&   rK   )rk   Ztrain_state_dictr   r1   r2   _load_train_state  s*   

z'Trainer.load.<locals>._load_train_stateNzLoaded no model !!!)r!   r|   r&   rK   r   r   )r/   r   r   r1   r   r2   r{      s   C
zTrainer.load)NNNN)NNN)Tr5   )F)__name__
__module____qualname__r3   rL   rS   rR   rZ   r^   rg   r{   r1   r1   r1   r2   r	      s     
2;



%r	   c                       s\   e Zd Zd fdd	Zdd Zdd Zdd	 Zdd
dZdd Z	dddZ	dd Z
  ZS )IntentTrainerNc                    s.   t t| |||| |jj| _|jj| _d S r5   )superr   r3   r   exampler	   can_norm)r/   r   r   r0   r"   	__class__r1   r2   r3   )  s   
zIntentTrainer.__init__c                 C   sD  t |d|kg}d| }d}t j|ddd d | d f }||jddd }|t | d t | }d}	|||	k  }
|||	k }|||	k  }|||	k }t|ra|
d|k nd}t|rp|d|k nd}|d	t||f 7 }|d
t||f 7 }t | jj	}|D ]\}\}}|d D ]
}||  d7  < qq||  }d\}}}t
|D ]B\}}t j|
|d  gdd}t|D ]}|| }||jddd }||d  9 }||jddd }q|d }| || kr|d7 }q|t|
 | t| }t|r|d|t|  7 }n|d7 }|d| 7 }|||fS )Nr   zoriginal acc: %s   axisT)r   Zkeepdimsgffffff?rF   z   (%s) confident acc: %sz   (%s) unconfident acc: %sintent_labelg      ?)r   r   r   r   z   new unconfident acc: %sz   no unconfident predictionsz   final acc: %s)r   meanargmaxsortsumloglenzerosr!   
num_intent	enumerateconcatenaterQ   )r/   y_predy_trueex_data_iteracc_originalmessager   Zy_pred_topkZy_pred_uncertainty	thresholdZy_pred_confidentZy_pred_unconfidentZy_true_confidentZy_true_unconfidentZacc_confidentZacc_unconfidentZprior_batch
batch_sizer   rightalphaZitersiyYj	acc_finalr1   r1   r2   can_normalization.  sn    





zIntentTrainer.can_normalizationc           &   
      s  g }  j d7  _  j   j   j   j  t|}|dur)t|nd}t||}t|}	|dur<t|nd}
|durDdnd}t	d|d D ]a}t

 }g g g g g f\}}}}}g }z
t|	\}\}}W n ty   t|}	t|	\}\}}Y nw || || |d || |durz
t|
\}\}}W n ty   t|}
t|
\}\}}Y nw || || |d || t||||D ]r\}}}}t|t fdd| } jr!|r!|j| } jj||d |d	 d
d}t|t fdd| }| D ]	\} }!|!|| < q j |d<  j|d<  j|d||d}" j|"|d\}#}"||# ||" qt|dkrS|d n|d |d  }# jj|# j jd t

 | }$||$   jd7  _t|||D ]\}}"} j|"|||||d q~ jdkr|dur|dur| j dkr j||d qM j ||d}% j!||% d dS )rU   r   Nr   TFc                       | d   | d fS Nr   r   r   kvr   r1   r2   ro         z+IntentTrainer.train_epoch.<locals>.<lambda>r   ZidsZintent)ZdatasetlabelsZindstaskc                    r   r   r   r   r   r1   r2   ro     r   r-   Z	num_steps)Zis_training
with_label	data_file)metricsr   )r%   r$   )r   batch_idr   r]   timesr   )rX   rY   )r\   r   )cur_valid_metric)"r-   r'   clearr(   r)   r*   r   maxiterrQ   timenextStopIterationappendziptypemapr   r   Zdata_file_to_datasetr"   Zretrieve_examplesr.   r   balance_metricsr!   	_optimizer%   r$   track_and_log_messager   rZ   r^   save_and_log_message)&r/   rM   rN   rO   rP   r   Znum_label_batchesZnum_nolabel_batchesr]   Ztrain_label_iter_loopZtrain_nolabel_iter_loopreport_for_unlabeled_datar   
start_timeZ
batch_listZbatch_size_listZwith_label_listZ	loss_listZmetrics_listZdata_file_listZdata_file_labelZbatch_labelr   Zdata_file_nolabelZbatch_nolabelr   r   r   r   r   Zcurrent_datasetZexample_batchr   r   r   losselapsedaccuracyr1   r   r2   rR   m  s  




















 	
zIntentTrainer.train_epochc                    s   g }t  H t|tfdd| }jj|d  fdd D   d }jr3||g7 }n|tj	|dd
 7 }W d    |S W d    |S 1 sQw   Y  |S )	Nc                    r   r   r   r   r   r1   r2   ro     r   z'IntentTrainer.forward.<locals>.<lambda>Zinputsc                    "   i | ]}| |     qS r1   cpudetachrv   r7   rs   resultr1   r2   ru         z)IntentTrainer.forward.<locals>.<dictcomp>intent_probsr   r   )rf   no_gradr   r   r   r   r^   r   r   r   tolist)r/   r   predr   r1   r   r/   r2   forward  s(   



zIntentTrainer.forwardc              
      s  j d tjjdj d}d}g g }}g g }}	t }
t	 , j
rt|ddD ]7\}\}}t|tfdd| }jj|d	  fd
d D  |t d  |	|d  7 }	q2tj|dd}jjry| n|}t|	d}	jjr|	 n|	}	j d|   t||dD ]\}\}}t|tfdd| }jj|d	  fdd D  j
rt d }jjr| n|}tj|| dd}t|djj}jjr| n|}| d|	!|dd|}|" # $ }n d }j%r*||g7 }||d " #  7 }n|t&j'|dd 7 }||d " #  7 }|d7 }||krL nqW d   n	1 sXw   Y  j%rt&(|}t&j)|dd}j*|||d\}}}t+||}|| |d}d| d| }nt,dd t-||D t.| }||d}d| }j d|  t/|d }t0j1||d!d" W d   n	1 sw   Y  d#j d$}d%t |
 d&}d|||g}j | |S )'z&
        Inference interface.
        zGeneration starts ...Zinfer_z.result.jsonr   zBuilding train memory.)descc                    r   r   r   r   r   r1   r2   ro     r   z%IntentTrainer.infer.<locals>.<lambda>r   c                    r   r1   r   r   r   r1   r2   ru     r   z'IntentTrainer.infer.<locals>.<dictcomp>featuresr   )dimzMemory size: )totalc                    r   r   r   r   r   r1   r2   ro   &  r   c                    r   r1   r   r   r   r1   r2   ru   )  r   r   r   r   r   N)r   r   r   )r   pred_labelsr   z
Accuracy:    c                 s   s    | ]	\}}||kV  qd S r5   r1   )r7   rA   tr1   r1   r2   r;   U  s    z&IntentTrainer.infer.<locals>.<genexpr>)r   r   zSaved inference results to w   )indentz[Infer][]zTIME-rb   )2r&   rK   rc   rd   re   r   r-   r   rf   r   r   r   r   r   r   r   r^   r   Z
from_numpyr   catr!   Zuse_gpucudaZ
LongTensorZ	unsqueezerx   Zsoftmaxmmr   r   r   Zscatter_addrepeatr   r   rv   r   r   r   arrayr   r   r   r   r   r   openjsondump)r/   r\   r]   r   Zinfer_save_fileZ	batch_cntr   trueZoutputsr   Z
begin_timer   r   r   Zmemr   Zprobsr   r   r   r   r   Zinfer_resultsZmetrics_messagefpmessage_prefixZ	time_costr1   r   r2   r^     s   








9

 

zIntentTrainer.inferc              	   C   s  |r| j n| j}|r| jn| j}dd | D }|dd}	dd | D }
dd | D }||
| |||	 | jdkr|| j dkr| }| }|rUdnd}d	| j	 d
| d| d
| d	}dt
|| j d  | j d}d||||g}| j| d S d S d S )Nc                 S   s2   i | ]\}}|t |tjr|   n|qS r1   )
isinstancerf   ZTensorr   r   rv   r7   r   r   r1   r1   r2   ru   h  s     z7IntentTrainer.track_and_log_message.<locals>.<dictcomp>mlm_numr   c                 S   s   i | ]\}}d |vr||qS tokenr1   r   r1   r1   r2   ru   n      c                 S   s   i | ]\}}d |v r||qS r  r1   r   r1   r1   r2   ru   o  r  ZLabeledZ	Unlabeledz[Train][z][/r   z	AVG_Time-rb   r   )r'   r)   r(   r*   r   popupdater   valuer-   r   re   r&   rK   )r/   r   r   r   r]   r   r   Zbatch_metrics_trackerZtoken_metrics_trackerr  Zbatch_metricsZtoken_metricsbatch_metrics_messagetoken_metrics_messageZlabel_prefixr   Zavg_timer   r1   r1   r2   r   b  s,    "z#IntentTrainer.track_and_log_messagec                 C   s   | j  }| j }d| j d}d|||g}| j| |r@| j }| j }d| j d}d|||g}| j| |d usFJ | j	rO|| j
k }n|| j
k}|rY|| _
| | d S )Nz[Valid][z
][Labeled]r   z][Unlabeled])r'   summaryr(   r-   re   r&   rK   r)   r*   r   r,   rg   )r/   r   r   r	  r
  r   r   rj   r1   r1   r2   r     s,   




z"IntentTrainer.save_and_log_messagec                 C   s,  | j dkr|D ]}|d urt|| j ksJ q|\}}}}}}	i }t|}||d< |}
|d ur`t|}t||| j   | }t|}||d< ||d< ||d< |
| jjrY|n|| jj  }
|d urut|}||d< |
|| jj  }
|	d urt|	}	|	|d< |
|	 }
|
|d< d|v sJ |d |fS )	Nr   intent_lossr  	token_mlmmlmklconr   )	r   r   rf   r   r   r!   Z
token_lossZ	mlm_ratioZkl_ratio)r/   r   r   Zmetricr  r  r  r  r  r  r   r1   r1   r2   r     s>   





zIntentTrainer.balance_metricsr5   )NN)r   r   r   r3   r   rR   r   r^   r   r   r   __classcell__r1   r1   r   r2   r   '  s    ? 
a 
r   )rc   r   collectionsr   r   rv   r   rf   Ztorch.optimr   r   Ztransformers.optimizationr   Z5modelscope.trainers.nlp.space.metrics.metrics_trackerr   Zmodelscope.utils.constantr   Zmodelscope.utils.loggerr   objectr	   r   r1   r1   r1   r2   <module>   s      