o
    )j#                     @   s   d dl Z d dlm  mZ d dl mZmZ d dlmZ dd Zdd Z	dd	 Z
G d
d dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZdS )    N)einsumnn)MossFormerConvModulec                 C   s   | d uS N )valr   r   t/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/audio/separation/mossformer_block.pyexists   s   r	   c                 C   s   t | r| S |S r   )r	   )r   dr   r   r   default   s   r   c                 C   s   | | }|dkr
dS || S )Nr   r   )nZmult	remainderr   r   r   padding_to_multiple_of   s   r   c                       &   e Zd Zd fdd	Zdd Z  ZS )	ScaleNormh㈵>c                    s0   t    |d | _|| _ttd| _d S )Ng         )	super__init__scaleepsr   	Parametertorchonesg)selfdimr   	__class__r   r   r      s   

zScaleNorm.__init__c                 C   s.   t j|ddd| j }||j| jd | j S )NT)r   Zkeepdim)min)r   normr   clampr   r   )r   xr!   r   r   r   forward"   s   zScaleNorm.forward)r   __name__
__module____qualname__r   r$   __classcell__r   r   r   r   r          r   c                       s$   e Zd Z fddZdd Z  ZS )ScaledSinuEmbeddingc                    sJ   t    ttd| _ddtd|d |   }| 	d| d S )Nr         ?i'  r      inv_freq)
r   r   r   r   r   r   r   arangefloatZregister_buffer)r   r   r.   r   r   r   r   )   s   
zScaledSinuEmbedding.__init__c                 C   sZ   |j d |j}}tj||d| j}td|| j}tj| |	 fdd}|| j
 S )Nr   )devicezi , j -> i jr   r   )shaper1   r   r/   Ztype_asr.   r   catsincosr   )r   r#   r   r1   tZsinuZembr   r   r   r$   /   s
   
zScaledSinuEmbedding.forwardr%   r   r   r   r   r+   '   s    r+   c                       r   )OffsetScaler   c                    sH   t    tt||| _tt||| _tj	j
| jdd d S )Ng{Gz?)Zstd)r   r   r   r   r   r   gammaZzerosbetainitZnormal_)r   r   headsr   r   r   r   9   s   
zOffsetScale.__init__c                 C   s    t d|| j| j }|jddS )Nz... d, h d -> ... h dr2   )r   r9   r:   Zunbind)r   r#   outr   r   r   r$   ?   s   zOffsetScale.forward)r   r%   r   r   r   r   r8   7   r*   r8   c                       s,   e Zd Zejdf fdd	Zdd Z  ZS )FFConvM皙?c              	      s<   t    t||t||t t|t|| _d S r   )	r   r   r   Z
SequentialZLinearZSiLUr   Dropoutmdl)r   dim_indim_out
norm_klassdropoutr   r   r   r   F   s
   

zFFConvM.__init__c                 C   s   |  |}|S r   )rB   )r   r#   outputr   r   r   r$   L   s   
zFFConvM.forward)r&   r'   r(   r   	LayerNormr   r$   r)   r   r   r   r   r?   D   s    r?   c                       sB   e Zd Zddddddejdf fdd		Zd
d ZdddZ  ZS )MossFormerBlock      r,   Fr@   NTc
                    s   t    t|| }
|| _|| _|	| _|| _t|| _	t
||
||d| _t
||||d| _t|dd| _t
|d |||d| _t | _d S )N)rC   rD   rE   rF      )r<   r-   )r   r   int
group_sizecausalshift_tokensrotary_pos_embr   rA   rF   r?   	to_hiddento_qkr8   qk_offset_scaleto_outZSigmoidgateActivate)r   r   rN   query_key_dimexpansion_factorrO   rF   rQ   rE   rP   Z
hidden_dimr   r   r   r   S   s6   

zMossFormerBlock.__init__c              	   C   s   |}| j r|jddd\}}tj|ddd}tj||fdd}| |jddd\}}| |}| |\}}	}
}| 	|||	|
|||\}}|| | 
||  }|| | }|S )Nr-   r   r2   )r   r   r   r           value)rP   chunkFpadr   r4   rR   rS   rT   cal_attentionrV   rU   )r   r#   Znormed_xZx_shiftZx_passvuZqkquad_qlin_qquad_klin_kZatt_vZatt_ur>   r   r   r   r$   |   s   
zMossFormerBlock.forwardc	                    s  |j d |j d |jjf\}	 }
}ddlm t|r(|d}|| d}tjr<tjj	||||f\}}}}t
 |dkrrtfdd||||||f\}}}}}}t|tj|	 f|
tjd}tj|dfd	d
}tfdd||||||f\}}}}}}t|r|d|d}td||| }t|d }|}t|r|| d}jrtj||ftj|
dd}||d}td||}td||}jrtd||| }|jdd}tj|ddd
}td||}td||| }|jdd}tj|ddd
}td||}ntd||  }td||}td||  }td||}t fdd||f\}}t fdd||f\}}|| || fS )Nr   r=   )	rearrangez... -> ... 1rY   c                    s   t j| ddd fddS )Nr   rY   rZ   )r]   r^   r7   )paddingr   r   <lambda>   s    z/MossFormerBlock.cal_attention.<locals>.<lambda>)r1   dtypeFrZ   c                    s    | dj dS )Nzb (g n) d -> b g n d)r   )rN   rg   )rf   r   r   r   ri      s    zb (g j) -> b g 1 j)jz... i d, ... j d -> ... i jr-   )rj   r1   r   z... i j, ... j d -> ... i dzb g n d, b g n e -> b g d er2   )r   r   r   r   r   r   zb g d e, b g n d -> b g n ezb g n d, b g n e -> b d ezb g n d, b d e -> b g n ec                       | dd d d  f S Nzb g n d -> b (g n) dr   rg   r   rf   r   r   ri          c                    rl   rm   r   rg   rn   r   r   ri      ro   )r3   r1   rN   Zeinopsrf   r	   Zmasked_fillrQ   mapZrotate_queries_or_keysr   r   r   r   boolr]   r^   r   ZrelurF   rO   ZtriuZcumsum)r   r#   rb   rc   rd   re   r`   ra   maskbr1   r   Zlin_masksimZattnZcausal_maskZ
quad_out_vZ
quad_out_uZlin_kvZ	lin_out_vZlin_kuZ	lin_out_uZquad_attn_out_vZlin_attn_out_vZquad_attn_out_uZlin_attn_out_ur   )r   rh   rf   r   r   r_      s|   $





zMossFormerBlock.cal_attentionr   )	r&   r'   r(   r   rH   r   r$   r_   r)   r   r   r   r   rI   Q   s    )rI   c                       s4   e Zd Z							d fdd		Zd
d Z  ZS )MossFormerModulerJ   rK         @Fr@   	scalenormTc
                    s   t    |dv sJ d|dkrtn|dkrtjddlm}
 |
tddt f	d	d
t	|D | _
d S )N)rw   	layernormz/norm_type must be one of scalenorm or layernormrw   rx   r   )RotaryEmbedding    r2   c                    s&   g | ]}t  d 	qS ))	r   rN   rW   rX   rO   rF   rQ   rE   rP   )rI   ).0_	attn_dropoutrO   r   rX   rN   rE   rW   rQ   rP   r   r   
<listcomp>   s    
z-MossFormerModule.__init__.<locals>.<listcomp>)r   r   r   r   rH   Zrotary_embedding_torchry   r    Z
ModuleListrangelayers)r   r   depthrN   rW   rX   rO   r~   Z	norm_typerP   ry   r   r}   r   r      s   



zMossFormerModule.__init__c                 C   s   | j D ]}||}q|S r   )r   )r   r#   Zmossformer_layerr   r   r   r$     s   

zMossFormerModule.forward)rJ   rK   rv   Fr@   rw   Tr%   r   r   r   r   ru      s    $ru   )r   Ztorch.nn.functionalr   Z
functionalr]   r   Z9modelscope.models.audio.separation.mossformer_conv_moduler   r	   r   r   Moduler   r+   r8   r?   rI   ru   r   r   r   r   <module>   s    