o
    *j                     @   sx   d dl Z d dlmZ d dlmZ G dd de jjZG dd dejZ	dd Z
	dd	d
Zdd Zdd ZdddZdS )    N)	is_masterc                   @   s$   e Zd Zedd Zedd ZdS )SparseBinarizerc                 C   sH   t | | }t|dd | }| d}d|d|< |S )N           )intZnumeltorchZargsortZreshapecloneZfill_)ctxmask_scoressparsityZ	num_pruneZprune_indicesmask r   l/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/hooks/compression/utils.pyforward   s
   zSparseBinarizer.forwardc                 C   s   |d fS Nr   )r
   Z
gradOutputr   r   r   backward   s   zSparseBinarizer.backwardN)__name__
__module____qualname__staticmethodr   r   r   r   r   r   r   	   s
    
r   c                       s>   e Zd ZdZ						d fdd	Zdd Zd	d
 Z  ZS )SparseLinearz>
    Fully Connected layer with on the fly adaptive mask.
    pst         ?c           
         s  t t|   || _| jjjd }| jjjd }	| jj| _d | j_| jjd || _d| _	| jdkr|| _
|| _|| _|| _|| _tt|| j
j| jj| jjd| _tt| j
|	j| jj| jjd| _tt|| jj| jj| jjd| _tt| j|	j| jj| jjd| _tt|j| jj| jjd| _tt|	j| jj| jjd| _d| j_| jjd urd| jj_d S d S d S )Nr   r   weightr   r   )devicedtypeF)superr   __init__moduler   shape_parameterspoppruning_methodcur_sparsityweight_rankweight_beta	mask_rankmask_alpha1mask_alpha2nn	Parameterr   Zrandntor   r   weight_UZzerosweight_Vmask_scores_Amask_scores_Bmask_scores_Rmask_scores_CZrequires_gradZbias)
selfr    r$   r&   r'   r(   r)   r*   Zout_featuresZin_features	__class__r   r   r      sd   




zSparseLinear.__init__c                 G   s   | j dkrA| j| j| j | j  }| | j| j | j  | j	| j
d| jd   }t|| j}|| }|| j_| j| S | j| S Nr   r   r   )r$   r   r'   r.   r/   absr)   r0   r1   r*   r2   	unsqueezer3   r   applyr%   r    )r4   Zinputsr   r   r   masked_weightr   r   r   r   T   s   



zSparseLinear.forwardc                 C   s   | j dkrB| j| j| j | j  }| | j| j | j  | j	| j
d| jd   }t|| j}|| }t|j| j_d S d S r7   )r$   r   r'   r.   r/   r8   r)   r0   r1   r*   r2   r9   r3   r   r:   r%   r+   r,   datar    )r4   r   r   r   r;   r   r   r   converte   s   

zSparseLinear.convert)r   r   r   r   r   r   )r   r   r   __doc__r   r   r=   __classcell__r   r   r5   r   r      s    7r   c                 C   s:   | d}|d d D ]}t| |} qt| |d | d S )N.r   )splitgetattrsetattr)modelnamer    	name_listr   r   r   _setattrt   s   
rG   c              	   C   s   t jg}zddlm}	 ||	j|	jg W n	 ty   Y nw |  D ]1\}
}t	||v rTt
|||||||}t| |
| t rT|rL|d|
 d q#td|
 d q#d S )Nr   )mpuconvert z to sparse module.)r+   ZLinearZmegatron_utilrH   extendZRowParallelLinearZColumnParallelLinearImportErrornamed_modulestyper   rG   r   infoprint)rD   r$   r&   r'   r(   r)   r*   loggerZcompress_modulerH   rE   r    Z
new_moduler   r   r   convert_sparse_network{   s6   

rQ   c                 C   s&   |   D ]\}}t|tr||_qd S r   )rL   
isinstancer   r%   )rD   r   rE   r    r   r   r   update_network_sparsity   s
   
rS   c                 C   sx   | || kr
|}|S | |||  kr|}|S || }|| | }	| | | | } d| ||	   }
||| |
d   }|S )Nr      r   )stepZ
total_stepZ	frequencyZinitial_warmupZfinal_warmupZinitial_sparsityZfinal_sparsityr   Zspars_warmup_stepsZspars_schedu_stepsZ	mul_coeffr   r   r   schedule_sparsity_ratio   s   	
rV   c                 C   s   |   D ]G\}}t|trK|  t| ||j t rK|r5|d| dt	d|jj
dk   d qtd| dt	d|jj
dk   d qd S )NrI   zE weight to sparse weight,                             sparsity ratio=r   r   r@   z> weight to sparse,                             sparsity ratio=)rL   rR   r   r=   rG   r    r   rN   r   meanr   itemrO   )rD   rP   rE   r    r   r   r   generate_sparse_model   s   


rY   r   )r   Ztorch.nnr+   Zmodelscope.utils.torch_utilsr   ZautogradFunctionr   Moduler   rG   rQ   rS   rV   rY   r   r   r   r   <module>   s   \
(