o
    *j                     @   s$  d dl Z d dlmZ d dlmZmZ d dlZd dlZd dl	Z
d dlmZmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlm Z m!Z! d dl"m#Z#m$Z$ d dl%m&Z& e
j'dkrye
j(j)Z
e
*  e& Z+dgZ,ej-e$j.ej/dG dd deZ0dS )    N)unescape)AnyDict)MosesDetokenizerMosesDetruecaserMosesPunctNormalizerMosesTokenizerMosesTruecaser)SentencePieceProcessor)beam_search_ops)	Pipelines)Model)
OutputKeys)Pipeline)	PIPELINES)ConfigConfigFields)	ModelFileTasks)
get_loggerz2.0AutomaticPostEditingPipeline)module_namec                       s   e Zd Zdef fddZdedeeef fddZdeeef deeef fdd	Zd
eeef deeef fddZ	  Z
S )r   modelc                    s  t  jd*d|i| |}ttj|tj| _	tj|| j	t
j d }tdd tt|dddD | _tdd tt|dddD | _| j	t
j d	d
| _| j	t
ji dd}|rcdn| j	t
ji dd| _| jd
kr}t| jd | _t  tjdd}d|j_tj|d| _tjj| jtj jj!j"g| t# }|$d| _%|$d| _&|$d| _'|$d| _(|$d}|$d}	tj)t*dd |tj+d}
t*dd |	}|
|d| _,t- }t. }| j/||g tjj| jtj jj!j"g| | j	t
j d | _0| j	t
j d | _1| j	t
j d d!t2| j0d"t2| j1d"tj|| j	t
j d# }t3|d$tj|| j	t
j d% }t4 | fd&d| _5fd'd| _6t7 t8| j1d"  fd(d| _9d)S )+zwBuild an automatic post editing pipeline with a model dir.

        @param model: Model path for saved pb file
        r   vocabc                 S   s   g | ]
\}}|  |fqS  strip.0iwr   r   y/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/pipelines/nlp/automatic_post_editing_pipeline.py
<listcomp>0       z9AutomaticPostEditingPipeline.__init__.<locals>.<listcomp>rutf8)encodingc                 S   s   g | ]
\}}||  fqS r   r   r   r   r   r!   r"   2   r#   unk_id	strip_unkT 	unk_tokenz<unk>   )Zallow_soft_placement)configzPlaceholder:0zPlaceholder_1:0zPlaceholder_2:0zPlaceholder_3:0zenc2enc/decoder/transpose:0zenc2enc/decoder/Minimum:0c                 S      | d S Nr   r   xr   r   r!   <lambda>P       z7AutomaticPostEditingPipeline.__init__.<locals>.<lambda>)Zdtypec                 S   r.   r/   r   r0   r   r   r!   r2   Q   r3   )
output_idsoutput_lensZsrc_langZtgt_langZtokenize_escapeF)lang	truecaser)Z	load_fromsentencepiecec              
      s&   d  jj| ddddS N T)
return_strescaper;   joinZencode_as_piecesZtruecasetokenizer0   )spsrc_tokenizer
tok_escaper7   r   r!   r2   i       c              
      s&   d j j| ddddS r9   r>   r0   )mt_tokenizerrA   rC   r7   r   r!   r2   o   rD   c                    s<     j| dddddd dd  dd S )Nu    ▁z@@r:   r*   r,   Tr=   )Z
detokenizeZ
detruecasereplacer   splitr0   )detokenizerdetruecaserr   r!   r2   y   s    
Nr   ):super__init__r   	from_fileospathr?   r   ZCONFIGURATIONcfgr   Zpreprocessordict	enumerateopenr   vocab_reversegetr'   Zpostprocessorr+   lentfZreset_default_graphZConfigProtoZgpu_optionsZallow_growthSession_sessionZsaved_modelloaderloadpythonZtag_constantsZSERVINGZget_default_graphZget_tensor_by_nameinput_src_id_placeholderinput_src_len_placeholderinput_mt_id_placeholderinput_mt_len_placeholdercastZmap_fnZint64outputZglobal_variables_initializerZlocal_variables_initializerrunZ	_src_langZ	_tgt_langr   r	   r
   src_preprocessmt_preprocessr   r   postprocess_fun)selfr   kwargsZ
export_dirZjoint_vocab_filer)   Z	tf_configZdefault_graphZoutput_id_beamZoutput_len_beamZ	output_id
output_leninitZ
local_initZtruecase_modelZsp_model	__class__)rH   rI   rE   rA   rB   rC   r7   r!   rK   %   s   





z%AutomaticPostEditingPipeline.__init__inputreturnc                    s   | dd\}} | |}}t fdd|   D g}t fdd|   D g}dd |D }dd |D }	 j| j| j| j	|	i}
|
S )Nr,   c                       g | ]
} j | jqS r   r   rT   r'   r   r    rf   r   r!   r"      r#   z;AutomaticPostEditingPipeline.preprocess.<locals>.<listcomp>c                    ro   r   rp   rq   rr   r   r!   r"      r#   c                 S      g | ]}t |qS r   rU   r   r1   r   r   r!   r"          c                 S   rs   r   rt   ru   r   r   r!   r"      rv   )
rG   rc   rd   nparrayr   r\   r^   r]   r_   )rf   rl   srcmtZsrc_spZmt_spZinput_src_idsZinput_mt_idsZinput_src_lensZinput_mt_lens	feed_dictr   rr   r!   
preprocess   s    z'AutomaticPostEditingPipeline.preprocessc                 C   sD   | j   | j j| j|d}|W  d    S 1 sw   Y  d S )N)r{   )rX   Z
as_defaultrb   ra   )rf   rl   Zsess_outputsr   r   r!   forward   s   $z$AutomaticPostEditingPipeline.forwardinputsc                    sZ   |d d |d d }}|d |d  }d  fdd|D } |}tj|i}|S )Nr4   r   r5   r,   r:   c                    ro   r   )rS   rT   r+   )r   Zwidrr   r   r!   r"      s    z<AutomaticPostEditingPipeline.postprocess.<locals>.<listcomp>)r?   re   r   ZTRANSLATION)rf   r~   r4   rh   Zoutput_tokensZpost_editing_outputresultr   rr   r!   postprocess   s   

z(AutomaticPostEditingPipeline.postprocess)__name__
__module____qualname__strrK   r   r   r|   r}   r   __classcell__r   r   rj   r!   r   !   s
    Z"*)1rM   htmlr   typingr   r   Zjiebanumpyrw   Z
tensorflowrV   Z
sacremosesr   r   r   r   r	   r8   r
   Z%tensorflow.contrib.seq2seq.python.opsr   Zmodelscope.metainfor   Zmodelscope.models.baser   Zmodelscope.outputsr   Zmodelscope.pipelines.baser   Zmodelscope.pipelines.builderr   Zmodelscope.utils.configr   r   Zmodelscope.utils.constantr   r   Zmodelscope.utils.loggerr   __version__compatZv1Zdisable_eager_executionlogger__all__Zregister_moduletranslationZautomatic_post_editingr   r   r   r   r!   <module>   s4   
