o
    "jB                     @   sd   d dl Z d dlmZmZmZ ddlmZmZmZ ddl	m
Z
 g ZG dd de
ZG dd	 d	e
ZdS )
    N)default_main_programdefault_startup_programprogram_guard   )OP_ROLE_KEYCollectiveHelperOpRole)MetaOptimizerBasec                       sX   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z		dddZ
  ZS )LocalSGDOptimizerc                    ,   t  | || _dg| _dg| _d| _d S )NAMPOptimizerAdaptiveLocalSGDOptimizer	@SNAPSHOTsuper__init__	inner_optZmeta_optimizers_white_listZmeta_optimizers_black_listsnapshot_keyself	optimizer	__class__ |/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.pyr         
zLocalSGDOptimizer.__init__c                 C   sF   | j jsdS | jjsdS | j  dkrdS t| jtjj	j
tjjjfS NFr   )
role_maker_is_collectiveuser_defined_strategylocalsgd_worker_num
isinstancer   paddler   ZmomentumMomentumsgdSGDr   r   r   r   
_can_apply&   s   zLocalSGDOptimizer._can_applyc                 C      d|_ i |_d S NFr    localsgd_configsr   dist_strategyr   r   r   _disable_strategy8      
z#LocalSGDOptimizer._disable_strategyc                 C      d|_ ddd|_d S )NTr   )k_steps
begin_stepr+   r   r.   contextr   r   r   _enable_strategy<   s   z"LocalSGDOptimizer._enable_strategyc                 C   
   || j  S Nr   r   
param_namer   r   r   snapshot_name@      
zLocalSGDOptimizer.snapshot_namec                 C   j   |  }g }| D ]
}|js|| q
g }|D ]}|j| |j|jdd|jd}|||g q|S NT)nameshapepersistableZstop_gradientdtype	Zglobal_blockZiter_parametersZis_distributedappendZ
create_varr<   r@   rA   rC   r   programblockZnon_dist_paramsparamp2ssnapshotr   r   r   create_snapshot_varsC   "   

z&LocalSGDOptimizer.create_snapshot_varsc                 C   H   t | |D ]
\}}t|| qW d    d S 1 sw   Y  d S r8   r   r#   assignr   startup_programZparam2snapshotrI   rK   r   r   r   init_snapshot_varsW   
   
"z$LocalSGDOptimizer.init_snapshot_varsNc                    s@  j j||d}jjd }jjd }|d u rt }|j}d_tjj}	|		| 
|| 
|jt|j|R tjjjddtjjddg|ddd	tjjddg|ddd	}
tjjd
dg|ddd	fdd  fdd}tjj|
k|  W d    |S 1 sw   Y  |S )NrR   r2   r3      r   beginint64Tr@   rA   valuerC   rB   	last_stepc               	      j  t   } d}D ]C\}}| jd|g|gdd|gittjid | jdd|id|ittjid |d j }| jd	d|gid|gid
|ttjid q	tjD ]}| jdd|id|id
|ttjid qRD ]C\}}| jdd|gid|giddj	  ttjid | jd|g|gdd|gittjid | jdd|gid|gittjid qit
  d S NZelementwise_sub)XYOuttypeZinputsZoutputsattrsc_sync_calc_streamr`   r   c_allreduce_sumring_idZc_sync_comm_streamscale      ?rP   r   Zcurrent_block	append_opr   r   Optimizenringsranger   r!   r#   rP   Z	sub_blockrh   rI   rK   r\   rJ   r   stepr   r   communicate   n   


	z4LocalSGDOptimizer.minimize_impl.<locals>.communicatec                         t jj k  d S r8   r#   staticnncondr   )rs   r2   r\   rr   r   r   begin_localsgd   s   z7LocalSGDOptimizer.minimize_impl.<locals>.begin_localsgd)r   minimizer   r,   r   rH   rn   r   r   update_startup_programrL   rS   rG   r   r#   r   lrautoincreased_step_counterrw   create_global_varrx   ry   )r   lossrR   parameter_listno_grad_set	minimizedZk_steps_valuebegin_step_value
main_blockcollective_helperr3   rz   r   )rs   r2   r\   rJ   r   rr   r   minimize_impl\   s`   

>

^^zLocalSGDOptimizer.minimize_implNNN)__name__
__module____qualname__r   r(   r/   r6   r<   rL   rS   r   __classcell__r   r   r   r   r
      s    	r
   c                       s`   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
	dddZ  ZS )r   c                    r   )Nr   r
   r   r   r   r   r   r   r      r   z"AdaptiveLocalSGDOptimizer.__init__c                 C   sD   | j jsdS | jjsdS | j  dkrdS t| jtjj	tjj
jfS r   )r   r   r   adaptive_localsgdr!   r"   r   r#   r   r$   r%   r&   r'   r   r   r   r(      s   z$AdaptiveLocalSGDOptimizer._can_applyc                 C   r)   r*   r   adaptive_localsgd_configsr-   r   r   r   r/      r0   z+AdaptiveLocalSGDOptimizer._disable_strategyc                 C   r1   )NTr   )init_k_stepsr3   r   r4   r   r   r   r6      s   z*AdaptiveLocalSGDOptimizer._enable_strategyc                 C   r7   r8   r9   r:   r   r   r   r<      r=   z'AdaptiveLocalSGDOptimizer.snapshot_namec                 C   r>   r?   rD   rF   r   r   r   rL      rM   z.AdaptiveLocalSGDOptimizer.create_snapshot_varsc                 C   rN   r8   rO   rQ   r   r   r   rS     rT   z,AdaptiveLocalSGDOptimizer.init_snapshot_varsc              
   C   s   |j dd|gid|giddttjddid |j d	d|gid|gittjid |j d
d|gid|gid
d| j  ttjid d S )Nrg   r`   rb   rh   r   Zuse_calc_streamTrc   rf   ri   rj   )rl   r   r   rm   r   r!   )r   Zprogram_blockr   avg_lossr   r   r   _generate_avg_loss  s.   

z,AdaptiveLocalSGDOptimizer._generate_avg_lossNc                    s  j j|d}jjd jjd }|d u rt }j
d_tjj}|	| 
|| 

jt
j| tjjjddtjjddgtdd	d
tjjddgt|dd	d
}tjjddgddd	d
tjjddgtdjd	d
 tjjddgtddd	d
	tjjddgtddd	d
j   	
fdd}	tjjdk|	 fdd 	
f
ddfdd}
tjj|k|
 W d    |S 1 sw   Y  |S )NrU   r   r3   rV   r   rW   r2   rY   TrZ   r\   r   r   lr_0Zfloat32loss_0c                      s*      t  t d S r8   )r   r#   rP   r   )r   	global_lrr   r   r   r   r   r   r   
initialize  s   z;AdaptiveLocalSGDOptimizer.minimize_impl.<locals>.initializec               	      r]   r^   rk   rp   rq   r   r   rs     rt   z<AdaptiveLocalSGDOptimizer.minimize_impl.<locals>.communicatec                     s     	   tjtt    t dd} tjdgddd}tjdgddd}t| |} t| |} t	|  d S )NrY   )rC   r      )rA   rC   Z
fill_value)
r   r#   castceilsqrtfloatfullminimummaximumrP   )Znext_local_stepsZmax_local_stepsZmin_local_steps)
r   rs   r   r   r2   r   r   r   r   r   r   r   communicate_avg_loss  s:   zEAdaptiveLocalSGDOptimizer.minimize_impl.<locals>.communicate_avg_lossc                      ru   r8   rv   r   )r   r2   r\   rr   r   r   rz     s   z?AdaptiveLocalSGDOptimizer.minimize_impl.<locals>.begin_localsgd)r   r{   r   r   r   rH   rn   r   r   r|   rL   rS   rG   r   r#   r   r}   r~   rw   r   intr   rC   Z_global_learning_raterx   ry   )r   r   rR   r   r   r   r   r   r3   r   rz   r   )r   rs   r   r   r   r2   r\   r   r   r   r   rJ   r   rr   r   r   3  s   


>
 
 !  !z'AdaptiveLocalSGDOptimizer.minimize_implr   )r   r   r   r   r(   r/   r6   r<   rL   rS   r   r   r   r   r   r   r   r      s    	r   )r#   Zpaddle.staticr   r   r   commonr   r   r   Zmeta_optimizer_baser	   __all__r
   r   r   r   r   r   <module>   s    9