o
    *j7                     @   s   d dl mZ d dlZd dlmZmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ ejd	krGejjZe  e Zejd
dG dd deZ								dddZdddZdS )    N)DictOptional)snapshot_download)CsanmtForTranslation)BaseTrainer)TRAINERS)	ModelFile)
get_loggerz2.0zcsanmt-translation)module_namec                       sZ   e Zd Zddedef fddZdd Zdd	 Z	dd
ee deee	f fddZ
  ZS )CsanmtTranslationTrainerNmodelcfg_filec           	         sv  |  |}t  || _t|tj| _|d u rt|tj	}t
 | i | _|   tjdd}d|j_tj|d| _tjtjd d gdd| _tjtjd d gdd| _i | _tj | _t| jfi | j| _| j| j| jd}| j| tjjt | jd d	| _ | j! &}t"#d
| j  t$| j}tj%| j| |&t'  W d    d S 1 sw   Y  d S )NT)Zallow_soft_placement)configsource_wids)dtypeshapenametarget_wids)inputlabelkeep_checkpoint_max)Zmax_to_keepzloading model from )(Zget_or_download_model_dirtfZreset_default_graph	model_dirospjoinr   ZTF_CHECKPOINT_FOLDERZ
model_pathZCONFIGURATIONsuper__init__params_override_params_from_fileZConfigProtoZgpu_optionsZallow_growthSession_sessionplaceholderint64r   r   outputtrainZcreate_global_stepglobal_stepr   r   updateZSaverglobal_variablesmodel_saver
as_defaultloggerinfoget_pretrained_variables_mapZinit_from_checkpointrunZglobal_variables_initializer)	selfr   r   argskwargsZ	tf_configr#   sessZpretrained_variables_map	__class__ s/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/nlp/csanmt_translation_trainer.pyr      sJ   

"z!CsanmtTranslationTrainer.__init__c                 C   s  | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d	 | jd	< | j d d
 | jd
< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< d| jd< d| jd< d| jd< | j d d | jd< | j d d | jd< | j d d d | jd< | j d d d | jd< | j d d | jd< | j d d | jd< | j d d  | jd < | j d d! | jd!< | j d d" | jd"< | j d d# | jd#< | j d d$ | jd$< | j d d% | jd%< | j d d& | jd&< | j d d' | jd'< | j d d( | jd(< | j d d) | jd)< | j d d | jd< | j d d* | jd*< | j d d+ | jd+< | j d d, | jd,< | j d d- | jd-< | j d d. | jd.< | j d d/ | jd/< | j d d0 | jd0< | j d d1 | jd1< | j d d2 | jd2< | j d3 d4 | jd4< | j d3 d5 | jd5< | j d3 d6 | jd6< | j d d7 | jd7< d S )8Nr   Zhidden_sizeZfilter_sizeZ	num_headsZnum_encoder_layersZnum_decoder_layersZlayer_preprocZlayer_postprocZ$shared_embedding_and_softmax_weightsZshared_source_target_embeddingZinitializer_scaleZposition_info_typeZmax_relative_disZnum_semantic_encoder_layersZsrc_vocab_sizeZtrg_vocab_sizeg        Zattention_dropoutZresidual_dropoutZrelu_dropoutZdataset	train_src	train_trg	src_vocabfile	vocab_src	trg_vocab	vocab_trgr$   num_gpusZwarmup_stepsZupdate_cycler   Z
confidenceZ	optimizerZ
adam_beta1Z
adam_beta2Zadam_epsilonZgradient_clip_normZlearning_rate_decayinitializerZlearning_ratetrain_batch_size_wordsZscale_l1Zscale_l2train_max_lennum_of_epochssave_checkpoints_stepsZnum_of_samplesetaZ
evaluationZ	beam_sizeZlp_rateZmax_decoded_trg_lenseed)cfgr   )r.   r4   r4   r5   r   E   s   
















z3CsanmtTranslationTrainer._override_params_from_filec                 O   s  t d t| j| jd }t| j| jd }t| j| jd }t| j| jd }d}d}| j }		 |d7 }|| jd	 krDntj	d
t
|f  t||||| jd | jd | jd dkrg| jd ndd|	|d
}
|
\}}zQ	 |	||g\}}|d7 }| j|| j|i}| jj| j|d}|d }t d|| || jd  dkrtj	dt
|f  | jd }| jj|	|tj d qt tjjy   tj	d|  Y nw q8tj	dt
ttt f  W d    d S 1 sw   Y  d S )NzBegin csanmt trainingr6   r7   r:   r<   r   T   rA   z%s: Epoch %ir?   r@   r=   )batch_size_wordsmax_lenr=   is_trainsessionepoch)	feed_dictZlossz Iteration: {}, step loss: {:.6f}rB   z%s: Saving model on step: %d.z
model.ckpt)r%   zepoch %d end!z'%s: NMT training completed at time: %s.)r*   r+   r   r   r   r   r    r)   r   logging__name__input_fnr-   r   r   r#   formatr(   saver$   Zget_global_steperrorsZOutOfRangeErrortimeasctime	localtime)r.   r/   r0   r6   r7   r:   r<   rK   	iterationZ
tf_sessionZtrain_input_fnfeatureslabelsZfeatures_batchZlabels_batchrL   Zsess_outputsZ	loss_stepZck_pathr4   r4   r5   r$      s   


0"zCsanmtTranslationTrainer.traincheckpoint_pathreturnc                 O   s   dS )a  evaluate a dataset

        evaluate a dataset via a specific model from the `checkpoint_path` path, if the `checkpoint_path`
        does not exist, read from the config file.

        Args:
            checkpoint_path (Optional[str], optional): the model path. Defaults to None.

        Returns:
            Dict[str, float]: the results about the evaluation
            Example:
            {"accuracy": 0.5091743119266054, "f1": 0.673780487804878}
        Nr4   )r.   rY   r/   r0   r4   r4   r5   evaluate   s   z!CsanmtTranslationTrainer.evaluateN)rN   
__module____qualname__strr   r   r$   r   r   floatr[   __classcell__r4   r4   r2   r5   r      s    +EA
r      d         rF   Tc                    s  t jjt jj|t jt jjjt jt jjjdddt jjt jj|t jt jjjt jt jjjdddt j	
| }t j	
|}t j	j||f}|jdd ddd}|jd	d ddd}|jfd
dddd}|	rfdd} fdd} fdd}|fdd}|t j	jj|||d}n|j| d gd gfd}t j	|}t t jj|j | \}}|	r|
|j |dkr|
t   ||fS )N)Z	key_dtypeZ	key_indexZvalue_dtypeZvalue_indexrF   )Znum_oov_bucketsc                 S   s   t | gt |gfS r\   )r   Zstring_splitsrcZtrgr4   r4   r5   <lambda>   s    zinput_fn.<locals>.<lambda>
   )Znum_parallel_callsi@B c                 S   s   | j |j fS r\   )valuesrf   r4   r4   r5   rh     s    c                    s     |  |fS r\   )lookuprf   )r8   r;   r4   r5   rh     s    c                    sL     d  }t t j| d| t j|d| }t jt |t jdS )NrF   r   r   )r   maximumsizecastminimumr"   )Zsrc_dataZtrg_databucket_widthZ	bucket_id)rH   num_bucketsr4   r5   key_func	  s   zinput_fn.<locals>.key_funcc                    s   |j  d gd gfdS )NZpadded_shapes)padded_batch)Z
unused_keyZwindowed_data)rG   r4   r5   reduce_func  s   zinput_fn.<locals>.reduce_funcc                    s8    d  }| d7 }   | |  }t j|t jdS )NrF   rm   )r   rp   r"   )keyrr   ro   )rG   rH   rs   r=   r4   r5   window_size_func  s   z"input_fn.<locals>.window_size_funcc                    s$   t t j| d kt j|d kS )Nrl   )r   logical_andro   rf   )rH   r4   r5   rh     s    )rt   rw   ry   ru   )r   rk   ZStaticVocabularyTableZTextFileInitializerstringZTextFileIndexZ
WHOLE_LINEr"   ZLINE_NUMBERdataZTextLineDatasetZDatasetzipmapZprefetchfilterapplyZexperimentalZgroup_by_windowrv   Zmake_initializable_iteratorZadd_to_collectionZ	GraphKeysZTABLE_INITIALIZERSr>   Zget_nextr-   Ztables_initializer)Zsrc_fileZtrg_fileZsrc_vocab_fileZtrg_vocab_filers   rH   Z
batch_sizerG   r=   rI   rJ   rK   Zsrc_datasetZtrg_datasetZsrc_trg_datasetrt   rw   ry   iteratorrW   rX   r4   )rG   rH   rs   r=   r8   r;   r5   rO      s   
rO   c                    s   t jt j| }|  d u r tfddt  D }nt fddt  D }g }ttt	dd t  t  }i }t j
ddd) |D ]\}}|| }	|	  }
|
| kri||	 |	||< qLW d    |S 1 suw   Y  |S )	Nc                    s6   g | ]}|j d d  v r|j |j d d fqS ):r   r   split).0var)saved_shapesr4   r5   
<listcomp>8  s
    z0get_pretrained_variables_map.<locals>.<listcomp>c                    sL   g | ]"  j d d v rt fddD r j  j d d fqS )r   r   c                 3   s    | ]}| j vV  qd S r\   )r   )r   scoper   r4   r5   	<genexpr>>  s
    
z:get_pretrained_variables_map.<locals>.<listcomp>.<genexpr>)r   r   all)r   ignore_scoper   r   r5   r   <  s     c                 S   s   | j dd S )Nr   r   r   )xr4   r4   r5   rh   D  s    z.get_pretrained_variables_map.<locals>.<lambda> T)Zreuse)r   r$   ZNewCheckpointReaderZlatest_checkpointZget_variable_to_shape_mapsortedr'   dictr}   r~   Zvariable_scope	get_shapeZas_listappend)Zcheckpoint_file_pathr   readerZ	var_namesZrestore_varsZname2varZrestore_mapvar_nameZsaved_var_nameZcurr_varZ	var_shaper4   r   r5   r,   3  s@   



r,   )rb   rc   rd   re   rF   TNNr\   )Zos.pathpathr   rS   typingr   r   Z
tensorflowr   Z modelscope.hub.snapshot_downloadr   Zmodelscope.models.nlpr   Zmodelscope.trainers.baser   Zmodelscope.trainers.builderr   Zmodelscope.utils.constantr   Zmodelscope.utils.loggerr	   __version__compatZv1Zdisable_eager_executionr*   Zregister_moduler   rO   r,   r4   r4   r4   r5   <module>   s6   

 K
U