o
    "j#                     @   s  d dl Z d dlZd dlmZmZmZmZmZmZm	Z	m
Z
 ddlmZmZmZmZ ddlmZmZ edG dd	 d	eZed
G dd deZedG dd deZedG dd deZedG dd deZedG dd deZedG dd deZdS )    N)ExponentialDecayInverseTimeDecayLRSchedulerNaturalExpDecay	NoamDecayexponential_decayinverse_time_decay
noam_decay   )get_optimize_opsget_ps_endpointget_role_idget_trainers   )PassBaseregister_passZadd_lr_decay_table_passc                       sN   e Zd Z fddZdd Zdd Z				dd	d
Zdd Zdd Z  Z	S )AddLrDecayTablePassc                       t    d S Nsuper__init__self	__class__ i/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/passes/ps_server_pass.pyr   (      zAddLrDecayTablePass.__init__c                 C      dS NTr   r   r   r   r   _check_self+      zAddLrDecayTablePass._check_selfc                 C   r   r    r   r   Z
other_passr   r   r   _check_conflict.   r"   z#AddLrDecayTablePass._check_conflict Nc                 C   sT   i }i ||< ||| d< ||| d< ||| d< ||| d< ||| d< ||d< d S )Nfeed_var_namefetch_var_namestartup_programmain_programtensor_table_classZtensor_tabler   )r   attrsr&   r'   r(   r)   r*   Ztensor_table_dictr   r   r   _add_tensor_table1   s   	z%AddLrDecayTablePass._add_tensor_tablec                 C   s  g d}t j }t j }d}t|tr?t j|| td||jd}|j}t	
d|  W d    n1 s9w   Y  nt|trot j|| t|j|jd}|j}t	
d|j  W d    n1 siw   Y  nht|trt j|| t jjd|j }|j}t	
d|  W d    n1 sw   Y  n6t|trt j|| td||jd}|j}t	
d|  W d    n1 sw   Y  ntd	||||fS )
N)r   r   r   r   r%   g      ?Ta  ExponentialDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: 
	 strategy = paddle.distributed.fleet.DistributedStrategy() 
 	 strategy.a_sync = True 
	 strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } 
z(NoamDecay is set, warmup steps is [ %d ]a  NaturalExpDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: 
	 strategy = paddle.distributed.fleet.DistributedStrategy() 
 	 strategy.a_sync = True 
	 strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } 
a  InverseTimeDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: 
	 strategy = paddle.distributed.fleet.DistributedStrategy() 
 	 strategy.a_sync = True 
	 strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } 
zQNot supported current LearningRate strategy, please use follow decay strategy: {})paddleZstaticZProgram
isinstancer   Zprogram_guardr   gammanameloggingwarnr   r	   Zd_modelZwarmup_stepsr   Z	optimizerlrZget_lrr   r   
ValueErrorformat)r   lr_schedulerlr_decay_stepsZschedler_decayZdecay_main_programZdecay_startup_programlr_namer3   r   r   r   _get_lr_scheduler_programE   s   









z-AddLrDecayTablePass._get_lr_scheduler_programc           	      C   sp   |j }t|d dsd S t|d jtsJ dt|d }| |d j|d \}}}| |d|||d d S )NZorigin_main_programr6   zmust be LRSchedulerr7   z@LR_DECAY_COUNTER@ZGlobalStepTable)_attrshasattrr.   r6   r   r   r9   r,   )	r   r)   r(   pass_ctxr+   opsZlr_decay_main_programZlr_decay_startup_programr8   r   r   r   _apply_single_impl   s2   
z&AddLrDecayTablePass._apply_single_impl)r%   NNr%   )
__name__
__module____qualname__r   r!   r$   r,   r9   r>   __classcell__r   r   r   r   r   &   s    
Nr   Zadd_listen_and_serv_passc                       4   e Zd Z fddZdd Zdd Zdd Z  ZS )	AddListenAndServPassc                    r   r   r   r   r   r   r   r      r   zAddListenAndServPass.__init__c                 C   r   r    r   r   r   r   r   r!      r"   z AddListenAndServPass._check_selfc                 C   r   r    r   r#   r   r   r   r$      r"   z$AddListenAndServPass._check_conflictc                 C   s^   |j }d d d d d t|d t|d t|d |d dddd}| jddg ii |d d S )NZ
role_makerZps_mode)Zgrad_to_block_idZsparse_grad_to_paramZlr_decay_block_idZdense_optimize_blocksZsparse_optimize_blocksZendpointZ
pserver_idZFaninZdistributed_modeZrpc_get_thread_numZrpc_send_thread_numZrpc_prefetch_thread_numZlisten_and_servX)typeZinputsZoutputsr+   )r:   r   r   r   Zglobal_blockZ	append_op)r   r)   r(   r<   r+   optr   r   r   r>      s"   



z'AddListenAndServPass._apply_single_implr?   r@   rA   r   r!   r$   r>   rB   r   r   r   r   rD      
    rD   Zadd_rpc_global_flags_passc                       rC   )	AddRpcGlobalFlagsPassc                    r   r   r   r   r   r   r   r      r   zAddRpcGlobalFlagsPass.__init__c                 C   r   r    r   r   r   r   r   r!      r"   z!AddRpcGlobalFlagsPass._check_selfc                 C   r   r    r   r#   r   r   r   r$      r"   z%AddRpcGlobalFlagsPass._check_conflictc                 C      d S r   r   r   r)   r(   r<   r   r   r   r>      r"   z(AddRpcGlobalFlagsPass._apply_single_implrI   r   r   r   r   rK      rJ   rK   Zadd_optimizer_passc                       rC   )	AddOptimizerPassc                    r   r   r   r   r   r   r   r      r   zAddOptimizerPass.__init__c                 C   r   r    r   r   r   r   r   r!      r"   zAddOptimizerPass._check_selfc                 C   r   r    r   r#   r   r   r   r$      r"   z AddOptimizerPass._check_conflictc                 C   rL   r   r   rM   r   r   r   r>      r"   z#AddOptimizerPass._apply_single_implrI   r   r   r   r   rN      rJ   rN   Zadd_geo_optimizer_passc                       rC   )	AddGeoOptimizerPassc                    r   r   r   r   r   r   r   r      r   zAddGeoOptimizerPass.__init__c                 C   r   r    r   r   r   r   r   r!      r"   zAddGeoOptimizerPass._check_selfc                 C   r   r    r   r#   r   r   r   r$      r"   z#AddGeoOptimizerPass._check_conflictc                 C   rL   r   r   rM   r   r   r   r>      r"   z&AddGeoOptimizerPass._apply_single_implrI   r   r   r   r   rO      rJ   rO   Z"build_pserver_startup_program_passc                       rC   )	BuildPserverStartupProgramPassc                    r   r   r   r   r   r   r   r      r   z'BuildPserverStartupProgramPass.__init__c                 C   r   r    r   r   r   r   r   r!     r"   z*BuildPserverStartupProgramPass._check_selfc                 C   r   r    r   r#   r   r   r   r$     r"   z.BuildPserverStartupProgramPass._check_conflictc                 C   rL   r   r   rM   r   r   r   r>     r"   z1BuildPserverStartupProgramPass._apply_single_implrI   r   r   r   r   rP      rJ   rP   Zdelete_unused_in_startup_passc                       rC   )	DeleteUnusedInStartupPassc                    r   r   r   r   r   r   r   r     r   z"DeleteUnusedInStartupPass.__init__c                 C   r   r    r   r   r   r   r   r!     r"   z%DeleteUnusedInStartupPass._check_selfc                 C   r   r    r   r#   r   r   r   r$     r"   z)DeleteUnusedInStartupPass._check_conflictc                 C   rL   r   r   rM   r   r   r   r>     r"   z,DeleteUnusedInStartupPass._apply_single_implrI   r   r   r   r   rQ     rJ   rQ   )r1   r-   Zpaddle.optimizer.lrr   r   r   r   r   r   r   r	   Zps.utils.publicr   r   r   r   Z	pass_baser   r   r   rD   rK   rN   rO   rP   rQ   r   r   r   r   <module>   s(   ( 	 