o
    *j;                     @   s:  d Z ddlmZ ddlZddlmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZ ddlZddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZm Z m!Z! ddl"m#Z# ddl$m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z= ddl>m?Z? ddl@mAZA ddlBmCZCmDZDmEZEmFZFmGZG ddlHmIZI ddlJmKZK eK ZLG dd deZMdeeeNe
f  d e9d!eeejO  fd"d#ZPdeeeNe
f  d$eQd e9d!eeeNe
f  fd%d&ZRe;jSe)jTd'G d(d) d)e?ZUdS )*z PyTorch trainer for UniTE model.    N)ceil)mkdir)AnyCallableDictListOptionalTupleUnion)	DataFrame)pad)clip_grad_norm_)AdamW	Optimizer)BatchSampler
DataLoaderDatasetSamplerSequentialSamplerSubsetRandomSampler)SummaryWriter)tqdm)AutoTokenizer)MetricsTrainers)Metric)
MetricKeysbuild_metric)
TorchModel)InputFormat)UniTEForTranslationEvaluationcombine_input_sentences)	MsDataset)Preprocessor)TRAINERS)Hook)EpochBasedTrainer)
ConfigDict)
ConfigKeysFieldsModeKeys	ModelFileTrainerStages)create_device)
get_loggerc                   @   s4   e Zd ZdedefddZdd Zdefdd	Zd
S )$TranslationEvaluationTrainingSamplernum_of_samples batch_size_for_each_input_formatc           	      C   s
  || _ || _| j d | _| jd }td| j | j| j | f  || _ t| j   }t	 | _
t	 | _t	 | _ttjtjtjfD ]1\}}|| j }|| j }||| | j|< tt| j| | jdd| j|< t| j| | j
|< qCd| _|  dkrtdddS )	a  Build a sampler for model training with translation evaluation trainer.
        The trainer should derive samples for each subset of the entire dataset.

        Args:
            num_of_samples: The number of samples in total.
            batch_size_for_each_input_format: During training, the batch size for each input format

        Returns:
            A data sampler for translation evaluation model training.

           zn%d samples are given for training. Using %d samples for each input format. Leaving the last %d samples unused.T
batch_sizeZ	drop_lastr   zCThe dataset doesn't contain enough examples to form a single batch.z?Please reduce the batch_size or use more examples for training.N)r0   r1   $num_of_samples_for_each_input_formatloggerinfotorchZrandpermcputolistdictsubset_iteratorssubset_samplersZindices_for_each_input_format	enumerater   SRC_REFSRCREFr   r   iternum_of_sampled_batches__len__
ValueError)	selfr0   r1   Znum_of_samples_to_useZrandom_permutationsinput_format_indexinput_format	start_idxend_idx rK   w/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/nlp/translation_evaluation_trainer.py__init__,   sV   




z-TranslationEvaluationTrainingSampler.__init__c              	   c   s    	 z[| j |  kr7tjtjtjfD ]!}	 z	t| j|  W n ty1   t	| j
| | j|< Y nw qqd| _ t }ttjtjtjfD ]\}}|t| j| 7 }qD|  j d7  _ |V  W n
 tyg   Y d S w q)NTr      )rC   rD   r   r?   r@   rA   nextr<   StopIterationrB   r=   listr>   )rF   rH   outputZinput_format_idxrK   rK   rL   __iter__e   s8   

z-TranslationEvaluationTrainingSampler.__iter__returnc                 C   s   | j | j S N)r5   r1   rF   rK   rK   rL   rD      s   z,TranslationEvaluationTrainingSampler.__len__N)__name__
__module____qualname__intrM   rS   rD   rK   rK   rK   rL   r/   *   s    
9r/   batchpreprocessorrT   c                    s>   t  }| d  D ] t fdd| D | < q	||}|S )Nr   c                 3   s    | ]}|  V  qd S rU   rK   .0xkeyrK   rL   	<genexpr>   s    z,convert_csv_dict_to_input.<locals>.<genexpr>)r;   keysrQ   )r[   r\   Z
input_dictrK   r`   rL   convert_csv_dict_to_input   s
   rd   r4   c                    s  t  }t |d< jtjkr\ttjtjtj	fD ]B\}}|| }|| }| || }|d  |g| 7  < 
| t|}| D ]\}	}
|	| vrRt ||	< ||	 |
 qCqn<jtjkr|d  jgt|  7  < t| } |  D ]\}	}
|	| vrt ||	< ||	 |
 qyntdj tdd |d D  t fdd|d D |d< tj|d dd|d< t|d	 d
|d	< jtjkrt|d t |d< t|d t |d< t|d t |d< |S )NrH   z9During training, %s mode is not allowed for preprocessor.c                 s   s    | ]}| d V  qdS )N)sizer]   rK   rK   rL   rb      s    z"data_collate_fn.<locals>.<genexpr>Z	input_idsc                 3   s.    | ]}t |d  |d fjdV  qdS )r   re   )r   valueN)r   rf   Zpad_token_idr]   Zinput_max_lengthsr\   rK   rL   rb      s    
r   )dimZscorere   ZlpZ	raw_scoreZ
segment_id)r;   rQ   moder*   ZTRAINr>   r   r?   r@   rA   change_input_formatrd   itemsrc   appendZEVALrH   lenrE   maxr8   catTensorviewsum)r[   r4   r\   Zoutput_dictrG   rH   rI   rJ   Zbatch_to_processra   rg   rK   rh   rL   data_collate_fn   sV   





rt   )module_namec                       s   e Zd Z			ddeeeejje	f  dee	 de	f fddZ
ded	ef fd
dZd	efddZd	efddZdd Z  ZS )TranslationEvaluationTrainerNgpumodelcfg_filedevicec           	         sX    fdd} fdd}t j|t j|i}t j|g|R ||d| d _d _dS )a  Build a translation evaluation trainer with a model dir or a model id in the model hub.

        Args:
            model: A Model instance.
            cfg_file: The path for the configuration file (configuration.json).
            device: Used device for this trainer.

        c                       t |  jjj jdS N)r4   r\   )rt   cfgtrainr4   Ztrain_preprocessorr_   rV   rK   rL   data_collator_for_train   
   zFTranslationEvaluationTrainer.__init__.<locals>.data_collator_for_trainc                    r{   r|   )rt   r}   
evaluationr4   eval_preprocessorr   rV   rK   rL   data_collator_for_eval   r   zETranslationEvaluationTrainer.__init__.<locals>.data_collator_for_eval)ry   data_collatorN)r(   r~   valsuperrM   train_dataloadereval_dataloader)	rF   rx   ry   rz   argskwargsr   r   r   	__class__rV   rL   rM      s$   z%TranslationEvaluationTrainer.__init__r}   rT   c                    s    j jjjdkrt |S  jjj	 D ]}d|_
qtd t fddtd j jjD }| jjj	  j jjjd | jj	  j jjjd | jj	  j jjjd t| j jjj j jjj j jjj j jjjd}|S )	z/Sets the optimizers to be used during training.r   FzBuilding AdamW optimizer ...c                 3   sB    | ]} j jjj|   jjjj jjjj|  d V  qdS )paramslrN)	rx   encoderlayer
parametersr}   r~   	optimizerplm_lrZplm_lr_layerwise_decay)r^   irV   rK   rL   rb      s    
z?TranslationEvaluationTrainer.build_optimizer.<locals>.<genexpr>r   r   )r   betasepsweight_decay)r}   r~   r   typer   build_optimizerrx   r   Z
embeddingsr   Zrequires_gradr6   r7   rQ   rangeZnum_hidden_layersrm   r   Z	estimatorZmlp_lrZlayerwise_attentionr   r   r   r   )rF   r}   paramZlearning_rates_and_parametersr   r   rV   rL   r      s8   









z,TranslationEvaluationTrainer.build_optimizerc                 C   s   t d | jd u r*t d| jjjj  tjt	
| j| jjjj| jjjjd| _t| jtt| j| jjjdd| jd d}t dt| j  |S )Nz$Building dataloader for training ...z"Reading train csv file from %s ...split)r1      Zbatch_samplerZnum_workersZ
collate_fn	generatorReading done, %d items in total)r6   r7   Ztrain_datasetr}   datasetr~   namer"   loadospjoin	model_dirr   r   r/   rn   r4   Ztrain_data_collator)rF   r   rK   rK   rL   get_train_dataloader&  s.   



	z1TranslationEvaluationTrainer.get_train_dataloaderc                 C   s   t d | jd u r*t d| jjjj  tjt	
| j| jjjj| jjjjd| _t| jtttdt| j| jjjddd| jd d}t d	t| j  |S )
Nz&Building dataloader for evaluating ...z!Reading eval csv file from %s ...r   r   Fr3   r   r   r   )r6   r7   eval_datasetr}   r   validr   r"   r   r   r   r   r   r   r   r   r   rn   r   r4   Zeval_data_collator)rF   r   rK   rK   rL   get_eval_data_loader>  s,   




z1TranslationEvaluationTrainer.get_eval_data_loaderc           
      C   s  d}t | jjdr| jjj}t| jfd| ji|}| tj t	 }t
jt
jt
jfD ]W}| j| | jrZddlm} ||| || j||| jjdd| jjdd| jd	 ndd
lm} ||| || j||| jd |D ]}	t |	drt|	jr|	  qqq+| tj |S )a   Evaluation loop used by `TranslationEvaluationTrainer.evaluate()`.

        The evaluation process of UniTE model should be arranged with three loops,
        corresponding to the input formats of `InputFormat.SRC_REF`, `InputFormat.REF`,
        and `InputFormat.SRC`.

        Here we directly copy the codes of `EpochBasedTrainer.evaluation_loop`, and change
        the input format during each evaluation subloop.
        Nvisualizationr   r   )multi_gpu_test	cache_dirgpu_collectF)rz   metric_classesvis_closureZtmpdirr   Zdata_loader_iters_per_gpu)single_gpu_test)rz   r   r   Zdata_loader_itersclear)hasattrr}   r   r   partialr   Zinvoke_hookr,   Z
before_valr;   r   r?   r@   rA   r   rk   _distZ#modelscope.trainers.utils.inferencer   updaterz   getZ_eval_iters_per_epochr   callabler   Z	after_val)
rF   Zdata_loaderr   r   Zvis_cfgZmetric_valuesrH   r   r   mrK   rK   rL   evaluation_loopW  sd   

	z,TranslationEvaluationTrainer.evaluation_loop)NNrw   )rW   rX   rY   r   r
   r   r8   nnModulestrrM   r'   r   r   r   r   r   r   __classcell__rK   rK   r   rL   rv      s"    -1rv   )V__doc__Zos.pathpathr   randommathr   osr   typingr   r   r   r   r   r	   r
   r8   Zpandasr   Ztorch.nn.functionalr   Ztorch.nn.utilsr   Ztorch.optimr   r   Ztorch.utils.datar   r   r   r   r   r   Ztorch.utils.tensorboardr   r   Ztransformersr   Zmodelscope.metainfor   r   Zmodelscope.metricsr   Zmodelscope.metrics.builderr   r   Zmodelscope.models.baser   Z)modelscope.models.nlp.unite.configurationr   Z2modelscope.models.nlp.unite.translation_evaluationr    r!   Zmodelscope.msdatasetsr"   Zmodelscope.preprocessorsr#   Zmodelscope.trainers.builderr$   Zmodelscope.trainers.hooksr%   Zmodelscope.trainers.trainerr&   Zmodelscope.utils.configr'   Zmodelscope.utils.constantr(   r)   r*   r+   r,   Zmodelscope.utils.devicer-   Zmodelscope.utils.loggerr.   r6   r/   r   rq   rd   rZ   rt   Zregister_moduleZtranslation_evaluation_trainerrv   rK   rK   rK   rL   <module>   s\   $ Z

3