o
    *ÎjE  ã                   @   sD   d dl Z d dlmZ G dd„ dejƒZdd„ Zedkr eƒ  dS dS )é    Nc                       sD   e Zd ZdZ‡ fdd„Zddd„Zdd„ Zd	d
„ Zddd„Z‡  Z	S )ÚMultiheadAttentionz%
    Multi head attention layer.
    c                    s€   || dksJ ‚t t| ƒ ¡  || _|| _|| | _| jd | _t ||d ¡| _	t ||¡| _
tj|d| _tjdd| _d S )Nr   g      à¿é   )Úpéÿÿÿÿ©Údim)Úsuperr   Ú__init__Ú
hidden_dimÚ	num_headsÚhead_dimÚscaleÚnnZLinearÚ
linear_qkvÚ
linear_outZDropoutÚdropout_layerZSoftmaxÚsoftmax)Úselfr
   r   Zdropout©Ú	__class__© úx/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/nlp/space/modules/multihead_attention.pyr	      s   
zMultiheadAttention.__init__Fc                 C   sL   |  | d¡| d¡| j| j¡}|r| dddd¡}|S | dddd¡}|S )Nr   é   é   r   )ÚreshapeÚsizer   r   Úpermute)r   ÚxÚis_keyr   r   r   Ú_split_heads   s
    ÿzMultiheadAttention._split_headsc                 C   s0   |  dddd¡}| | d¡| d¡| j¡}|S )Nr   r   r   r   )r   r   r   r
   )r   r   r   r   r   Ú_merge_heads   s   zMultiheadAttention._merge_headsc                 C   sŒ   t  ||¡}|| j }|d ur'| d¡}| d| jdd¡}| | ¡ tdƒ¡ |  	|¡}|  
|¡}|d ur>	 | | ¡ d¡ t  ||¡}|S )Nr   z-infg        )ÚtorchÚmatmulr   Z	unsqueezeÚrepeatr   Zmasked_fill_ÚboolÚfloatr   r   )r   ÚqueryÚkeyÚvalueÚmaskZscoresZattnÚoutr   r   r   Ú_attn$   s    

þ

	zMultiheadAttention._attnNc           	      C   sÀ   |   |¡}tj|| jdd\}}}|  |¡}| j|dd}|  |¡}|durLd|v rDd|v rDtj|d |gdd}tj|d |gdd}||d< ||d< |  ||||¡}|  |¡}|  |¡}|S )	z$ Forward process of self attention. r   r   T)r   Nr'   r(   r   )	r   r!   Úsplitr
   r   Úcatr+   r    r   )	r   Úinpr)   ÚcacheZqkvr&   r'   r(   r*   r   r   r   ÚforwardB   s   




zMultiheadAttention.forward)F)NN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   r   r    r+   r0   Ú__classcell__r   r   r   r   r      s    
r   c                  C   sr   dd l } tdddƒ}| j ddd¡ d¡}t |¡}| j ddd¡dk d¡}t |¡}|||d d}t|ƒ d S )Nr   é
   r   g      à?r   Zfloat32)r)   r/   )Únumpyr   ÚrandomZrandZastyper!   ZtensorÚprint)ÚnpÚmodelr.   r)   r*   r   r   r   Úmain\   s   

r<   Ú__main__)r!   Ztorch.nnr   ÚModuler   r<   r1   r   r   r   r   Ú<module>   s   U
ÿ