o
    *j#0                     @   s  d dl mZ d dlmZmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlmZ ddlmZmZ dd	 ZG d
d dejZG dd dejZd!ddZG dd dejZG dd deZG dd deZG dd dejZdd ZG dd dejZG dd deZG dd  d eZdS )"    )partialmethod)ListOptionalN)	LayerNormsoftmax_dropout)permute_final_dims   )Linearchunk_layerc                 C   s&   |dk sJ t | }||| dk< |S )Ng     r   )torchZ
zeros_like)maskZneg_inf	attn_mask r   u/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/science/unifold/modules/attentions.pygen_attn_mask   s   
r   c                       sz   e Zd Z	ddedededededef fdd	Z	
	
ddejdejdejdejdeej dejfddZ	dd Z
  ZS )	AttentionTq_dimk_dimv_dimhead_dim	num_headsgatingc                    s   t t|   || _|| j }|| _t||ddd| _t||ddd| _t||ddd| _t||dd| _	d | _
| jrCt||dd| _
|d | _d S )NFglorotbiasinitfinalr   r         )superr   __init__r   r   r	   linear_qlinear_klinear_vlinear_olinear_gnorm)selfr   r   r   r   r   r   Z	total_dim	__class__r   r   r       s   	
zAttention.__init__Nqkvr   r   returnc           	      C   sP  d }| j d ur|  |}| |}|| j9 }| |}| |}||jd d | jdf dd	 }||jd d | jdf dd	 }||jd d | jdf dd}t
||dd}~~t|d| j||d}t
||}~~|dd	 }|jg |jd d dR  }|d urt
|| }tj|| jj}|S )Nr   )r   r   )r%   r!   r&   r"   r#   viewshaper   	transpose
contiguousr   matmulr   trainingsigmoidnnZ
functionallinearr$   weight)	r'   r*   r+   r,   r   r   gattnor   r   r   forward0   s:   





&zAttention.forwardc                 C   s   | j jS N)r$   r   r'   r   r   r   get_output_biasY   s   zAttention.get_output_biasTNN)__name__
__module____qualname__intboolr    r   Tensorr   r>   rA   __classcell__r   r   r(   r   r      s>    	
)r   c                       s8   e Zd Z fddZdejdejdejfddZ  ZS )GlobalAttentionc                    s   t t|   || _|| _|| _t||| ddd| _t||ddd| _t||ddd| _	t||| dd| _
t|| |dd| _t | _|d | _d S )NFr   r   r   r   r   r   )r   rK   r    r   infepsr	   r!   r"   r#   r%   r$   r8   ZSigmoidr7   r&   )r'   Z	input_dimr   r   rL   rM   r(   r   r   r    _   s   
zGlobalAttention.__init__xr   r-   c           
      C   s0  |  | |}| |}| |}tj||d ddtj|ddd| j  }| |}|| j	9 }|
|jd d | jdf }t||dd}~~t|| j dd d d d d f }t|d| j|d}t||}	~~|
|jd d | jdf }|	d	| }	~|	|	jd d d
 }	| |	S )Nr.   r/   dimT)rP   Zkeepdims.r   r   r0   )r.   )r7   r%   r"   r#   r   sum	unsqueezerM   r!   r&   r1   r2   r   r5   r3   r   rL   r   r6   Zreshaper$   )
r'   rN   r   r;   r+   r,   r*   r<   r   r=   r   r   r   r>   o   s2   



"
zGlobalAttention.forward)rD   rE   rF   r    r   rI   r>   rJ   r   r   r(   r   rK   ]   s    $rK   Tc                 C   s\   t | | dd d d d d d f }|r,t | dd| dd d d d d d f }||fS |S N.r.   r/   r   r3   )r   rL   Zgen_col_maskZrow_maskZcol_maskr   r   r   gen_msa_attn_mask   s   "rV   c                       s   e Zd Z		d fdd	Zejj			ddejdeej deej de	d	ejf
d
dZ
ejj			ddejdeej deej dee	 d	ejf
ddZddeej fddZ			ddejdeej deej dee	 d	ejf
ddZdd Z  ZS )MSAAttentionFNc                    sb   t t|   || _t|| _d | _d | _| jr&t|| _t||ddd| _t	|||||| _
d S NFnormalr   )r   rW   r    	pair_biasr   layer_norm_mlayer_norm_zlinear_zr	   r   mha)r'   d_ind_hidr   rZ   d_pairr(   r   r   r       s   

zMSAAttention.__init__mr   r   
chunk_sizer-   c                 C   s(   t | j|||d|t|jd d dS )N)rb   r   r   r/   rc   Znum_batch_dimsr
   _attn_forwardlenr2   )r'   rb   r   r   rc   r   r   r   _chunk   s   	zMSAAttention._chunk 
  c              
   C   s   |  |}|jd | d | }g }t|D ]B}|| }t|jd || }	|d||	d d d d f }
|d urI|d||	d d d d d d f nd }|| j|
|
|
||d qtj|ddS )Nr0   r   .r*   r+   r,   r   r   rO   )r[   r2   rangeminappendr^   r   cat)r'   rb   r   r   rc   Z	num_chunkZoutputsiZchunk_startZ	chunk_endZcur_mZcur_maskr   r   r   _attn_chunk_forward   s   
"z MSAAttention._attn_chunk_forwardc                 C   s   |  |}| j|||||dS )Nrj   )r[   r^   )r'   rb   r   r   r   r   r   rf      s   
zMSAAttention._attn_forwardzr   c                 C   s   d }| j r| |}t| |dd }|d ur%| ||||}|S d}|jd |kr7| |||}|S | j	||||dS )N   r   r   ri   r0   )rc   )
rZ   r\   r   r]   rS   r4   rh   r2   rf   rp   )r'   rb   rq   r   rc   r   Zattn_chunk_sizer   r   r   r>      s$   



zMSAAttention.forwardc                 C   
   | j  S r?   r^   rA   r@   r   r   r   rA         
zMSAAttention.get_output_bias)FNNNN)NNri   r?   )rD   rE   rF   r    r   jitignorerI   r   rG   rh   rp   rf   r>   rA   rJ   r   r   r(   r   rW      sd    
rW   c                       s   e Zd Z fddZ  ZS )MSARowAttentionWithPairBiasc                    s   t t| j|||d|d d S )NT)rZ   ra   )r   r{   r    )r'   d_msara   r`   r   r(   r   r   r         

z$MSARowAttentionWithPairBias.__init__)rD   rE   rF   r    rJ   r   r   r(   r   r{     s    r{   c                	       sN   e Zd Z fddZ		d
dejdeej dee dejf fdd	Z  Z	S )MSAColumnAttentionc                    s   t t| j|||dd d d S )NF)r_   r`   r   rZ   ra   )r   r~   r    )r'   r|   r`   r   r(   r   r   r      r}   zMSAColumnAttention.__init__Nrb   r   rc   r-   c                    s.   | dd}t j|||d}| dd}|S )Nr/   r0   )r   rc   )r3   r   r>   )r'   rb   r   rc   r(   r   r   r>     s   zMSAColumnAttention.forwardrC   )
rD   rE   rF   r    r   rI   r   rG   r>   rJ   r   r   r(   r   r~     s    r~   c                	       s   e Zd Z		d fdd	Zejjdejdejdedejfd	d
Z	dd Z
		ddejdeej dee dejfddZ  ZS )MSAColumnGlobalAttention    eA绽|=c                    s0   t t|   t|| _t|||||d| _d S )N)rL   rM   )r   r   r    r   r[   rK   global_attention)r'   r_   r`   r   rL   rM   r(   r   r   r    (  s   
z!MSAColumnGlobalAttention.__init__rb   r   rc   r-   c                 C   s&   t | j||d|t|jd d dS )N)rb   r   r/   rd   re   r'   rb   r   rc   r   r   r   rh   ;  s   zMSAColumnGlobalAttention._chunkc                 C   s   |  |}| j||dS )NrQ   )r[   r   )r'   rb   r   r   r   r   rf   L  s   
z&MSAColumnGlobalAttention._attn_forwardNc                 C   sN   | dd}| dd}|d ur| |||}n| j||d}| dd}|S )Nr/   r0   r.   rQ   )r3   rh   rf   r   r   r   r   r>   P  s   z MSAColumnGlobalAttention.forward)r   r   rC   )rD   rE   rF   r    r   ry   rz   rI   rG   rh   rf   r   r>   rJ   r   r   r(   r   r   &  s4    r   c                 C   sT   t | | dd d d d d d f }t | dd| dd d d d d d f }||fS rT   rU   )r   rL   Z
start_maskZend_maskr   r   r   gen_tri_attn_maskc  s
   "r   c                       s   e Zd Z fddZejj			ddejdeej deej de	dejf
d	d
Z
		ddejdeej dee	 dejfddZdd Z  ZS )TriangleAttentionc                    sF   t t|   || _t|| _t||ddd| _t|||||| _	d S rX   )
r   r   r    startingr   
layer_normr	   r9   r   r^   )r'   r_   r`   r   r   r(   r   r   r    l  s
   
zTriangleAttention.__init__NrN   r   r   rc   r-   c                 C   s,   t | j|||||d|t|jd d dS )Nrj   r/   rd   )r
   r^   rg   r2   )r'   rN   r   r   rc   r   r   r   rh   y  s   zTriangleAttention._chunkr   c                 C   sz   | j s	|dd}| |}t| |dd }|d ur(| ||||}n
| j|||||d}| j s;|dd}|S )Nr/   r0   rr   rt   rj   )	r   r3   r   r   r9   rS   r4   rh   r^   )r'   rN   r   rc   Ztriangle_biasr   r   r   r>     s   


zTriangleAttention.forwardc                 C   ru   r?   rv   r@   r   r   r   rA     rw   z!TriangleAttention.get_output_biasrx   rC   )rD   rE   rF   r    r   ry   rz   rI   r   rG   rh   r>   rA   rJ   r   r   r(   r   r   j  s:    
r   c                   @      e Zd ZeejddZdS )TriangleAttentionStartingTr   NrD   rE   rF   r   r   r    r   r   r   r   r         r   c                   @   r   )TriangleAttentionEndingFr   Nr   r   r   r   r   r     r   r   rB   )	functoolsr   typingr   r   r   Ztorch.nnr8   Zunicore.modulesr   r   Zunicore.utilsr   commonr	   r
   r   Moduler   rK   rV   rW   r{   r~   r   r   r   r   r   r   r   r   r   <module>   s&   G
6
e=?