o
    "j%                     @   s   d dl mZmZ d dlmZ d dlmZmZmZ d dl	Z	G dd deZ
G dd de
ZG d	d
 d
eZeG dd dZeG dd deZeG dd de
ZeG dd deZeG dd de
ZdS )    )ABCabstractmethod)	dataclass)ListOptionalSequenceNc                   @   s   e Zd ZeejfddZdS )AttentionBiasc                 C      t  NNotImplementedErrorselfshapedtype r   ]/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/incubate/nn/attn_bias.pymaterialize   s   zAttentionBias.materializeN)__name__
__module____qualname__r   paddlefloat32r   r   r   r   r   r      s    r   c                   @   s"   e Zd ZejfddZdd ZdS )LowerTriangularMaskc                 C   s<   |t jur|nt j}t j|td|d}t j|dd|S )N-inf)r   
fill_valuer      Zdiagonalr   Zbfloat16r   fullfloatZtriuZastype)r   r   r   	create_astensorr   r   r   r   %   s
   
zLowerTriangularMask.materializec                 C   s   t |S r
   )!LowerTriangularMaskWithTensorBiasr   Zbiasr   r   r   add_bias,   s   zLowerTriangularMask.add_biasN)r   r   r   r   r   r   r%   r   r   r   r   r   $   s    r   c                       s*   e Zd Zdd Zejf fdd	Z  ZS )r#   c                 C   s
   || _ d S r
   )_biasr$   r   r   r   __init__1   s   
z*LowerTriangularMaskWithTensorBias.__init__c                    s   t  ||| j S r
   )superr   r&   r   	__class__r   r   r   4   s   z-LowerTriangularMaskWithTensorBias.materialize)r   r   r   r'   r   r   r   __classcell__r   r   r)   r   r#   0   s    r#   c                   @   sJ   e Zd ZU ejed< eed< ee ed< dd Ze	dd Z
dd	d
ZdS )
SeqLenInfoseqstart
max_seqlenseqstart_pyc                 c   s"    t | j| jdd  E d H  d S )Nr   )zipr/   r   r   r   r   	intervals>   s    zSeqLenInfo.intervalsc                 C   sN   dg}d}|D ]}t ||}||d |  qtj|tjd}| |||dS )Nr   r   )r.   r-   r/   )maxappendr   	to_tensorint32)clsseqlensr/   r.   seqlenr-   r   r   r   from_seqlensA   s   
zSeqLenInfo.from_seqlensNc                 C   s   | j d |jd kr|jd dksJ |d u r!dgt| j d  }g }d}|D ]}|| j ||  | j |   ||7 }q'dd t||j|ddD S )Nr3   r   r   c                 S   s,   g | ]\}}| |d g|jdd qS )r3      NZreshaper   ).0bsr"   r   r   r   
<listcomp>X   s    z$SeqLenInfo.split.<locals>.<listcomp>Zaxis)r/   r   lenr6   r0   split)r   xbatch_sizesZsplit_chunksitZ
batch_sizer   r   r   rD   M   s   &
zSeqLenInfo.splitr
   )r   r   r   r   Tensor__annotations__intr   r2   classmethodr<   rD   r   r   r   r   r,   8   s   
 

r,   c                       sV   e Zd ZU ejed< ee ed<  fddZe	dd Z
e	dd Zdd
dZ  ZS )PaddedSeqLenInfor;   	seqlen_pyc                 #   s4    t t  | jD ]\\}}}||| fV  q
d S r
   )r0   r(   r2   rM   )r   start_lengthr)   r   r   r2   c   s   zPaddedSeqLenInfo.intervalsc                 C   s   t d)NzQPlease use SeqLenInfo.from_seq_lens() or PaddedSeqLenInfo.from_seq_lens_padded().r   )r9   r:   r   r   r   r<   g   s   zPaddedSeqLenInfo.from_seqlensc                    sd   t  fdd|D sJ ttdt|  d  }| tj|tjd|t|tj|tjd|dS )Nc                 3   s    | ]}| kV  qd S r
   r   )r?   r;   paddingr   r   	<genexpr>o   s    z7PaddedSeqLenInfo.from_seqlens_padded.<locals>.<genexpr>r   r   r4   )r;   rM   r.   r-   r/   )alllistrangerC   r   r7   r8   r5   )r9   r:   rR   r/   r   rQ   r   from_seqlens_paddedm   s   z$PaddedSeqLenInfo.from_seqlens_paddedNc                 C   r	   r
   r   )r   rE   rF   r   r   r   rD   y   s   zPaddedSeqLenInfo.splitr
   )r   r   r   r   rH   rI   r   rJ   r2   rK   r<   rW   rD   r+   r   r   r)   r   rL   ^   s   
 


rL   c                   @   s   e Zd ZU eed< eed< dZeee  ed< e	j
fddZe	j
fddZedd	d
Zedd ZedddZdd Zdd Zdd Zdd ZdS )BlockDiagonalMask	q_seqinfo	k_seqinfoN_batch_sizesc                 C   s   t j||dS N)r   r   )r   Zzerosr   r   r   r   _create_block_mask   s   z$BlockDiagonalMask._create_block_maskc           
      C   s   |d | j jd ksJ |d | jjd ksJ tj|dd  td|d}t| j | j  D ]\\}}\}}|| || g}| |||||||f< q1t	t
|d D ]}	|d}qX||S )Nr3   r   )r   r   r=   r   )rZ   r/   rY   r   r   r    r0   r2   r]   rV   rC   	unsqueezeexpand)
r   r   r   maskq_startq_endk_startk_endZ	sub_shaperO   r   r   r   r      s   
zBlockDiagonalMask.materializec                 C   sR   |d u st |t |ksJ t|}|d u s||kr|}nt|}| ||dS )N)rY   rZ   )rC   r,   r<   )r9   q_seqlen	kv_seqlenrY   rZ   r   r   r   r<      s   

zBlockDiagonalMask.from_seqlensc                 C   sp   dd |D }g }|D ]}t |jd D ]
}||jd  qq| |}||_tjdd |D dd}||fS )Nc                 S      g | ]}|j d  qS r   r   r?   r"   r   r   r   rA          z6BlockDiagonalMask.from_tensor_list.<locals>.<listcomp>r   r   c                 S   (   g | ]}| d dg|jdd qS r   r3   r=   Nr>   r?   rE   r   r   r   rA         ( rB   )rV   r   r6   r<   r[   r   concat)r9   ZtensorsrF   r:   rE   rO   
block_diagZconcated_tensorr   r   r   from_tensor_list   s   
z"BlockDiagonalMask.from_tensor_listc                 C   sR  t |t |ks
J |d u st |t |ksJ dd |D }g g }}tt||D ]D\}\}}	|jd |	jd ks=J ||jd g|jd   ||	jd g|	jd   |d u so|| jd d |	jd d ksoJ q+| ||}
dd |D |
_|
tjdd |D ddtjd	d |D dd|d urtjd
d |D ddfS d fS )Nc                 S   rh   ri   rj   rk   r   r   r   rA      rl   z;BlockDiagonalMask.from_tensor_lists_qkv.<locals>.<listcomp>r   r   r=   c                 S   rh   ri   rj   ro   r   r   r   rA      rl   c                 S   rm   rn   r>   ro   r   r   r   rA      rp   rB   c                 S   rm   rn   r>   ro   r   r   r   rA      rp   c                 S   rm   rn   r>   ro   r   r   r   rA      rp   )	rC   	enumerater0   r   extendr<   r[   r   rq   )r9   Z	tensors_qZ	tensors_kZ	tensors_vrF   Z	q_seqlensZ
kv_seqlensiqkrr   r   r   r   from_tensor_lists_qkv   s4   
,z'BlockDiagonalMask.from_tensor_lists_qkvc                 C      | j || jS r
   )rY   rD   r[   r   r"   r   r   r   split_queries      zBlockDiagonalMask.split_queriesc                 C   rz   r
   )rZ   rD   r[   r{   r   r   r   split_kv   r}   zBlockDiagonalMask.split_kvc                 C   s    | j | ju sJ | j || jS r
   )rY   rZ   rD   r[   r{   r   r   r   rD      s   zBlockDiagonalMask.splitc                 C   s   t | j| j| jdS )N)rY   rZ   r[   )BlockDiagonalCausalMaskrY   rZ   r[   r1   r   r   r   make_causal   s
   zBlockDiagonalMask.make_causalr
   )r   r   r   r,   rI   r[   r   r   rJ   r   r   r]   r   rK   r<   rs   ry   r|   r~   rD   r   r   r   r   r   rX   }   s    
 	
rX   c                   @   s   e Zd ZejfddZdS )r   c                 C   s   t  j||dS r\   )r   r   r   r   r   r   r]      r}   z*BlockDiagonalCausalMask._create_block_maskN)r   r   r   r   r   r]   r   r   r   r   r      s    r   c                   @   s^   e Zd ZU eed< eed< dZeej	 ed< dej
fddZej
fdd	Ze	dd
dZdS )+BlockDiagonalCausalWithOffsetPaddedKeysMaskrY   rZ   Ncausal_diagonalr   c                 C   s@   |t jur|nt j}t j||tdd}t j|d| d|S )Nr   r   r   r   r   r   )r   r   offsetr   r!   r"   r   r   r   r]      s   z>BlockDiagonalCausalWithOffsetPaddedKeysMask._create_block_maskc           
      C   s   |d | j jd ksJ |d | jjd ksJ tj|dd  |tdd}tt| j | j  D ].\}\\}}\}}| j	|| || f| j
d u rMdnt| j
|  |d|||||f< q3tt|d D ]}	|d}qj||S )Nr3   r^   r   r   r   )r   r   r=   )rZ   r/   rY   r   r   r    rt   r0   r2   r]   r   rJ   itemrV   rC   r_   r`   )
r   r   r   ra   rv   rb   rc   rd   re   rO   r   r   r   r      s    

z7BlockDiagonalCausalWithOffsetPaddedKeysMask.materializec                 C   s@   |d u st |t |ksJ t|}t||}| |||dS )N)rY   rZ   r   )rC   r,   r<   rL   rW   )r9   rf   Z
kv_paddingrg   r   rY   rZ   r   r   r   r<      s   
z8BlockDiagonalCausalWithOffsetPaddedKeysMask.from_seqlensr
   )r   r   r   r,   rI   rL   r   r   r   rH   r   r]   r   rK   r<   r   r   r   r   r      s   
 r   )abcr   r   dataclassesr   typingr   r   r   r   r   r   r#   r,   rL   rX   r   r   r   r   r   r   <module>   s"   %]