o
    "jP                     @   s   d dl Z d dlmZ ddlmZ g Zd dlZd dlmZ d dl	m
Z d dlmZ d dlmZmZ d d	lmZmZ d d
lmZmZ d dlmZmZ d dlmZ G dd deZG dd deZdS )    N)reduce   )MetaOptimizerBase)	framework)base)LayerHelper)corein_dynamic_mode)ClipGradByNormappend_gradient_clip_ops)Momentum	Optimizer)L1DecayL2Decay)create_global_varc                       s   e Zd ZdZdZddgddddddf fdd	Zd	d
 Zdd Zdd Zd"ddZ	d#ddZ
dd Zdd Zd$ddZdd Zdd Zdd Ze d d! Z  ZS )%DGCMomentumOptimizerZ_dgc_u_Z_dgc_v_r   +?NFc                    s  t  rtdt sJ d|d usJ |d usJ t j|||	|
|d d| _|| _t|| _	|dks9J d|| _
|| _|| _d | _d | _d | _|| _|
d urxt|
ts[tdt|tshJ dt| |dkspJ d	|
j|d
  | _| | j\| _| _d S )Nz/In dygraph, don't support DGCMomentumOptimizer.zBPaddle is not compiled with CUDA. DGC is only support GPU for now.)learning_rate
parametersZweight_decay	grad_clipnamedgc_momentumr   zrampup_begin_step must >= 0zjThe type of grad_clip should be 'ClipGradByNorm', because DGCMomentumOptimizer only support ClipGradByNormz9The type of num_trainers should be 'int', but received %sz3The value of num_trainers should be greater than 0!g      )r	   	Exceptionr   Zis_compiled_with_cudasuper__init__type	_momentumbool_use_nesterov_rampup_begin_step_rampup_step	_sparsity_rampup_begin_step_var_global_step_var_dgc_clip_norm_num_trainers
isinstancer
   	TypeErrorint	clip_norm_get_regularization_paramregularizationregular_typeregular_coeff)selfr   momentumrampup_begin_steprampup_stepsparsityparameter_listuse_nesterovnum_trainersr+   r   r   	__class__ w/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.pyr   $   sV   


zDGCMomentumOptimizer.__init__c                 C   sR   d}d}|d ur%|j }t|trd}||fS t|tr!d}||fS td||fS )Nr           r      z+regularization must be None|L1Decay|L2Deacy)Z_coeffr&   r   r   AssertionError)r.   r+   r,   r-   r8   r8   r9   r*   c   s   

z.DGCMomentumOptimizer._get_regularization_paramc                 C   sV   t tdd |jd}|dk s'|jtjjjks'|jtjjjks'|jtjjj	kr)dS dS )Nc                 S   s   | | S Nr8   )xyr8   r8   r9   <lambda>u   s    z2DGCMomentumOptimizer._is_use_dgc.<locals>.<lambda>r   i @  FT)
absr   shaper   r   VarDescVarTypeZSELECTED_ROWSdtypeFP32)r.   	param_vargrad_varZ	var_numelr8   r8   r9   _is_use_dgct   s   z DGCMomentumOptimizer._is_use_dgcc           	      C   s   t |tjjs	J | | j|d }|d usJ |d |d || |d}|d |d}| j| jd}| 	|d |d sAd}nd}|
| j| jd |
d	|d i |
d
t| ji |j||||dd}|S )Nr   r   )ParamGradVelocityLearningRate)ParamOutZVelocityOut)mur4   r/   r   )current_stepnranksGrad_outr0   Tr   inputsoutputsattrsstop_gradient)r&   paddler   ZBlockZ_get_accumulator_u_velocity_acc_str_create_param_lrr   r   rI   updater#   _nranks_varfloatr   	append_op)	r.   blockparam_and_gradZvelocity_accrT   rU   rV   r   Zdgc_momentum_opr8   r8   r9   _append_optimize_op   s@   
z(DGCMomentumOptimizer._append_optimize_opc                 C   s   t d}|j|ddgdd\}}|r>|j|tjjjt|d ddd |j	 j
dd	|gid
|gidt|idd d|_|S )Nglobal_step_counterfloat32r   Tr   rE   rB   persistablevalue	force_cpuinitializer	incrementXOutsteprS   )r   create_or_get_global_variableset_variable_initializerrX   nnrj   ConstantInitializerr]   main_programglobal_blockZ_prepend_oprW   )r.   counter_namebeginrn   helpercounter
is_new_varr8   r8   r9   _add_auto_increment_var   s(   



z,DGCMomentumOptimizer._add_auto_increment_varc                 C   sN   t d}|j|ddgdd\}}|r%|j|tjjjt|ddd d|_|S )Nrb   rc   r   Trd   rf   ri   )	r   ro   rp   rX   rq   rj   rr   r]   rW   )r.   r   rg   rw   rx   ry   r8   r8   r9   _add_nranks_var   s   

z$DGCMomentumOptimizer._add_nranks_varc                 C   s  t j }d|_| jtj dd| _| j	tj
 | jd| _tdgtjjjdtj | jd dd| _t| jj| _|D ]\}}| | j|}| ||sOq=| | j|}tdg|jd|jtj  ddd}tdg|jd|jtj  dd	d}tdg|jd|jtj   dd	d}	tj!}
tj!j"j#}|$ j%D ]8}| &|sq|' |
(  }|j|vrq|)|j |)|j t*|dkr|+|
( | q|,|
(  q|}| j-d ur| .|| j-}| /||||||||	 q=d S )
NTr   )ru   rv   )r   rg   r   g      ?)rB   rE   re   r   rg   rh   r:   F)0rX   staticdefault_main_programZ_enable_dgcrz   r   dgcZkDGCCounterNamer#   r|   ZkDGCNRanksNamer%   r\   r   rC   rD   rF   ZkDGCRampUpBeginStepNamer   r"   r   r7   __name__rw   Z_add_accumulatorrY   rI   _v_velocity_acc_strrE   r   Z	kDGCKNameZkDGCEncodedNameZkDGCGatherNameop_proto_and_checker_makerOpRoleBackwardrt   ops_is_the_backward_op	all_attrskOpRoleVarAttrNameremovelen	_set_attrZ_remove_attrr$   _append_clip_norm_dgc_op)r.   Zparam_and_gradsrs   rG   rH   u_varv_vark_varencoded_var
gather_varop_makerbackwardopZvar_attrclip_varr8   r8   r9   _append_dgc_ops   s   

			




z$DGCMomentumOptimizer._append_dgc_opsc                 C   sB   t j}t jjj}| |jv rt| |  t|krdS dS )NTF)	r   r   r   r   r   Z
attr_namesr(   r   kOpRoleAttrName)r.   r   r   r   r8   r8   r9   r   -  s   
z(DGCMomentumOptimizer._is_the_backward_opc                 C   s   |||d}t di |}|d u rtjjd|jdg}|j|j||j	dd}|j
d|| jd|t| jd	d
|id |S )Nr>   max_normr   dgc_clip_by_norm_op.tmpF)r   r   rE   re   Zdgc_clip_by_norm)rl   rP   )r   r0   rm   )r   rT   rV   rU   )r   )r   rX   r   Zunique_nameZgenerate_with_ignorable_keyjoinr   Zcreate_variabler   rE   r^   r#   r]   r   )r.   r>   r   r   argsrw   outr8   r8   r9   _clip_by_norm6  s$   
	z"DGCMomentumOptimizer._clip_by_normc                 C   sB   |j j  | j|||jdW  d    S 1 sw   Y  d S )Nr   )r_   programZ_backward_role_guardr   r   )r.   rH   r)   r8   r8   r9   r   O  s
   $z&DGCMomentumOptimizer._append_clip_normc	                 C   s   t j  }	tj}
| j}| j}|jd ur| 	|j\}}|	j
d||||| j| jd||||||d| j| j| jt| jt| jt|t|ddd}|
jj}||
 | ||
 |j|jg d S )Nr   )UVrK   rJ   rP   rQ   )ZU_outZV_outZ
EncodeGradkrR   Z
GatherBuff)mr2   r4   r0   r1   r-   r,   TrS   )rX   r}   r~   rt   r   r   r,   r-   Zregularizerr*   r^   r#   r\   r   r!   r   r]   r   r    r(   r   r   r   r   r   r   )r.   rG   r   rH   r   r   r   r   r   r_   r   r,   r-   Zdgc_opr   r8   r8   r9   r   U  sN   
			zDGCMomentumOptimizer._dgc_opc              	   C   s  ddl m} t }t  }||}d}d}g }|D ]\}	}
|	j|kr2|dur-td|	}|
}q||	|
f qd}|dur||g}|jj	
|4 td |   |jd||| |dd|d id	}W d   n1 ssw   Y  W d   n1 sw   Y  |||f|fS )
a  
        Because distribute lookup table only support SGD optimizer for now, not support
        other optimizer and regularization, so we should find the table parameter out,
        and avoid to add regularization and other op for it, and add sgd optimize op
        for it independently.
        :param param_grads(list((Var, Var))): list of (param, grad) pair.
        :param loss: the loss variable.
        :param startup_program: the startup program
        r   )find_distributed_lookup_tableNz1multi dist table var found, only support one now!	optimizerZsgd)rJ   rK   rM   rN   )r   rT   rU   )Z*paddle.distributed.distribute_lookup_tabler   r   r~   rt   r   RuntimeErrorappendr_   r   Z_optimized_guardZ
name_scopeZ_create_global_learning_rater^   rZ   )r.   Zparam_gradsr   r   rt   Z
table_nameZtable_paramZ
table_gradZnew_param_gradspgZsgd_opr`   r8   r8   r9   _process_distribute_lookuptable  sJ   


 z4DGCMomentumOptimizer._process_distribute_lookuptablec           	      C   s   |  | t|dd d}| |\}}}g }g }|D ]\}}| ||s-|||f q|||f q| jd ur@| |}nt|}| || j}|| }t|dd d}| 	|}|d urj|| || |S )Nc                 S   
   | d j S Nr   r   r>   r8   r8   r9   r@        
 z6DGCMomentumOptimizer.apply_gradients.<locals>.<lambda>)keyc                 S   r   r   r   r   r8   r8   r9   r@     r   )
r   sortedr   rI   r   
_grad_clipr   Zappend_regularization_opsr+   Z_create_optimization_pass)	r.   params_gradsZtable_param_and_gradZtable_optimize_opZnot_dgc_params_gradsZdgc_params_gradsparamZgradoptimize_opsr8   r8   r9   apply_gradients  s8   




z$DGCMomentumOptimizer.apply_gradients)r   )r{   r=   )r   
__module____qualname__rY   r   r   r*   rI   ra   rz   r|   r   r   r   r   r   r   imperative_baseZno_gradr   __classcell__r8   r8   r6   r9   r       s2    ?
*
]
	:1r   c                       sv   e Zd Z fddZ fddZdd Zdd Zd	d
 Zdd Z				dddZ	dd Z
dd Z	dddZ  ZS )DGCOptimizerc                    s(   t  | || _d | _g | _g | _d S r=   )r   r   	inner_optdgc_optZmeta_optimizers_white_listZmeta_optimizers_black_list)r.   r   r6   r8   r9   r     s
   
zDGCOptimizer.__init__c                    s   t  |||| d S r=   )r   _set_basic_info)r.   loss
role_makerZuser_defined_optimizeruser_defined_strategyr6   r8   r9   r     s   zDGCOptimizer._set_basic_infoc                 C   s   | j d urd S | j}| jjsd S t|tsd S | jj}t|d dkr(dg|d< t	|j
|j|d |d |d |j|j| j |j|j|jd| _ d S )Nr2   r   r   r0   r1   )r   r/   r0   r1   r2   r3   r4   r5   r+   r   r   )r   r   r   _is_collectiver&   r   r   dgc_configsr   r   Z_learning_rater   Z_parameter_listr   _worker_numr+   r   _name)r.   optZconfigsr8   r8   r9   _init_dgc_opt  s.   


zDGCOptimizer._init_dgc_optc                 C   sR   | j jsdS | jjr't| jtstd dS | j 	 dkr%td dS dS dS )NFz$dgc only works on Momentum optimizerr   zdgc only works on multi cardsT)
r   r   r   r   r&   r   r   loggingwarnr   )r.   r8   r8   r9   
_can_apply  s   

zDGCOptimizer._can_applyc                 C   s   d|_ i |_d S )NFr   r   )r.   dist_strategyr8   r8   r9   _disable_strategy-  s   
zDGCOptimizer._disable_strategyc                 C   s   d|_ ddd|_d S )NTr   r   )r0   r1   r   )r.   r   contextr8   r8   r9   _enable_strategy1  s   zDGCOptimizer._enable_strategyNc                 C   s   |    | j|||||S r=   )r   r   r   )r.   r   startup_programr3   no_grad_set	callbacksr8   r8   r9   r   5  s   
zDGCOptimizer.backwardc                 C   s   |    | jj|dS )N)r   )r   r   r   )r.   r   r8   r8   r9   r   B  s   zDGCOptimizer.apply_gradientsc                 C   s   |    | jj|||dS )N)r   r   )r   r   Z_apply_optimize)r.   r   r   r   r8   r8   r9   apply_optimizeF  s   zDGCOptimizer.apply_optimizec                 C   s&   |    | j||||\}}||fS r=   )r   r   Zminimize)r.   r   r   r3   r   r   r   r8   r8   r9   minimize_implL  s
   zDGCOptimizer.minimize_impl)NNNN)NNN)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r8   r8   r6   r9   r     s     
r   )r   	functoolsr   Zmeta_optimizer_baser   __all__rX   Zpaddle.baser   Zpaddle.base.dygraphr   r   Zpaddle.common_ops_importr   Zpaddle.frameworkr   r	   Zpaddle.nn.clipr
   r   Zpaddle.optimizerr   r   Zpaddle.regularizerr   r   Zpaddle.staticr   r   r   r8   r8   r8   r9   <module>   s$      Q