o
    "jQ                     @   s   d dl Z d dlmZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZ dgZG dd dZdd Zdd Zdd Zdd Zdd Zdd ZdS )    N)defaultdict)DistributedContext)'get_distributed_operator_impl_container)Programcore)	Parameter   )OperatorDistAttr)BACKWARD_ONLY_DIST_OPS)__no_shape_var_type__is_backward_opis_forward_op
is_loss_opis_optimize_opZlod_tensor_blocking_queuec                   @   sJ   e Zd ZdZdddZdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dS )Partitionera  
    warning:: Partitioner is experimental and subject to change.

    Partitioner convert a program into another program.
    Given a serial program which has been auto completed with shard annotation, the Partitioner
    convert the serial program into a "distributed" program. The Partitioner will  modify the serial
    program in following two ways, which is also the major difference between serial and distributed program:
        1. partition op: replace a serial op into its corresponding dist op infered from the shard annotation
        2. partition var: if a var is sharded, modify the shape of var according to its shard annotation

    Partitioner is supposed to be call by the auto parallel framework, and not supposed to be directly called by user.
    r   c                 C   s@   t |tstdt| || _|| _tt| _d| _	i | _
dS )aT  
        Args:
            dist_context (DistributedContext): used to access the distributed_attr of var & op, every Partitioner object could maintain its own DistributedContext member, and partition program base on that shard scenario.
            rank_id (int): global rank id to which the partitioned distributed program belong.
        z/dist_context be DistributedContext, got %s here N)
isinstancer   	TypeErrortype_dist_context_rank_idr   dict_serial2dist_varname_mapping_dist_varname_suffix_forward_op_id2forward_op)selfdist_contextrank_id r   t/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/auto_parallel/static/partitioner.py__init__7   s   

zPartitioner.__init__c                 C   s~   t |tstdt| | |std| jj}| j|_	| j
|_|d u r)d }n| ||}||_| ||\}}|||fS )Nz5main_program be paddle.framework.Program, got %s herez3Not all vars or ops are annotated in main program !)r   r   r   r   _is_valid_annotated_programRuntimeErrorr   dist_op_contextr   Zvarname_mappingr   r   partition_startup_programZdst_startup_programpartition_main_program)r   serial_main_programserial_startup_programZparams_gradsr#   partitioned_startup_progpartitioned_main_progZpartitioned_params_gradsr   r   r   	partitionK   s6   


zPartitioner.partitionc                 C   s  t |tstdt| tj }| }| }i }i }| D ] }|js)J |j	| j
 }	|	||j	< t| j|||j	|	}
|
||	< q"| jD ]}|j }t|dks`J dt|j d||d  |v srJ d|d  d|j }||j ||d ||d   |d|||d    |  |jd	 }|j| ksJ |j|ksJ ||d }| j|}t }|j|_||j	|j ||j	|j | j|| qH|S )
Nz5dist_context be paddle.framework.Program, got %s herer   z6initializer should output only ONE variable, but got []r   ztry to initialize [z ] which is not a persistable varshape)r   r   r   r   paddle	frameworkglobal_block	list_varspersistablenamer   _partition_varr   opsdescoutput_arg_nameslenstrZ	append_opZ	copy_fromZ_rename_output	_set_attrZ_sync_with_cppvar get_tensor_dist_attr_for_programr	   process_meshZset_output_dims_mappingdims_mappingZset_input_dims_mappingZset_op_dist_attr_for_program)r   r&   r'   r(   	ref_blocktarget_blockZ	var2shapeZtemp_varname_mapr;   new_nametarget_shapeopZoutput_varsZnew_op_descZnew_opZ
output_varZoutput_var_attrZop_attrr   r   r   r$   t   sh   








z%Partitioner.partition_startup_programc              	   C   sl  t j }| jj}||_t| jjjD ]/}|j	| }|dkr$|j	d }n|j
|jd}|j|jks3J ||j ||_| || qd|_t| jjjD ]+}||}	|	jD ] }
|
 D ]}|
|tjjkrv|
|}|
||| q]qWqMg }|D ]4\}}|j| jd v sJ | ||d}|du rd}n|j| jd v sJ | ||d}|||f q}||fS )z_
        1. partition variables
        2. replace local op with corresponding dist op
        r   )
parent_idxN)r.   r/   r   r   r#   Zdst_main_programrangeZblock_stateZnblockblocksZ_create_blockrD   idxZ_set_forward_block_idxforward_block_idxZ
work_blockpartition_blockZcurrent_block_idxblockr5   Z	all_attrsZ	attr_typer   ZAttrTypeZBLOCKZ_block_attr_idr:   r3   r   _get_dist_var_by_serial_varappend)r   r&   Zparams_and_gradsr)   r#   rG   r?   r@   block_idrJ   rC   	attr_nameZrelative_idZpartitioned_params_and_gradspgZdist_pZdist_gr   r   r   r%      sR   





z"Partitioner.partition_main_programc                 C   s  | j j}d}t|jD ]\}}t|r|} nq|dkr!t|j}tt|jD ]}||kr<|j| | j|j| j	 < q(d}t|jD ]\}}| j 
|}t|rlt|j|d  set|j|d  rl|jsl|d7 }|j D ]-}	|	| j|j vs|	| j|j vr|	| j }
||	rt| j |||	|
 |
| j|j |	< qq|j D ]-}|| j|j vs|| j|j vr|| j }
||rt| j ||||
 |
| j|j |< qt|s|jr||\}}t|| j }|j| j fi || qDt|r||\}}t|| j | j}| j jj| }|j| j fi ||d|i qDt|rD||\}}t|| j | j}|j| j fi ||di i qDtdt| d S )Nr-   r   r   grad_var_to_varzEpartitioner only support forward and backward, optimize ops, but got )r   r#   	enumerater5   r   r8   rE   r   r6   original_idget_op_dist_attr_for_programr   r   Zis_recomputeZinput_arg_namesr   rH   rG   r   has_varr4   r7   Zprepare_context_get_dist_op_forward_implementforward_get_dist_op_backward_implementrQ   Zbackwardr   NotImplementedErrorr9   )r   r?   r@   r#   Zlast_fwd_op_idxrG   rC   Zappended_grad_timesop_dist_attrZserial_input_varnameZnew_varnameZserial_output_varnameZkinputsZkoutputsZdist_op_forward_implZdist_op_backward_implrQ   Zdist_op_opt_implr   r   r   rI      s   


	


	




zPartitioner.partition_blockc                    sb   |  j}| } fdd|D } fdd|D }tdd |D }tdd |D }|o0|S )Nc                    s   g | ]} j |qS r   )r   rT   ).0rC   r   r   r   
<listcomp>i  s    z;Partitioner._is_valid_annotated_program.<locals>.<listcomp>c                    s"   g | ]}|j tvr j|qS r   )r   r   r   r<   )r[   r;   r\   r   r   r]   l  s
    

c                 s       | ]}|d uV  qd S Nr   r[   	dist_attrr   r   r   	<genexpr>r      
z:Partitioner._is_valid_annotated_program.<locals>.<genexpr>c                 s   r^   r_   r   r`   r   r   r   rb   u  rc   )r0   r5   r1   all)r   programr5   Zvars_Zop_dist_attrsZvar_dist_attrsZall_ops_annotatedZall_vars_annotatedr   r\   r   r!   e  s   


z'Partitioner._is_valid_annotated_programc                 C   s:   |j j}|j| }| j| |j }||sJ ||S r_   )rJ   rG   rF   r   r3   rU   r;   )r   Z
serial_varr)   rM   Z	block_idxr@   Zdist_var_namer   r   r   rK   {  s   

z'Partitioner._get_dist_var_by_serial_varN)r   )__name__
__module____qualname____doc__r    r*   r$   r%   rI   r!   rK   r   r   r   r   r   )   s    
)A6zr   c              
   C   s   | j }|j}|jj }|g kr|S t|t|ks#J d| d| dg }tt|D ]B}|| dks9|| dkrA|||  q+|| |||   dks`J d|| |||  | j|||||| |||    q+|S )Nzvariable shape [z] and dim_mapping [z] is NOT match !r-   r   zAun-event partition: var_shape[idx]=[{}], mesh[{}], {}, {}, {}, {})r,   r>   r=   r8   rE   rL   formatr3   )r;   ra   Z	var_shapemappingZmeshZ	new_shaperG   r   r   r   _get_dist_shape  s4   
	rl   c                 C   sl   i }|j |d< |j|d< |j|d< |j|d< |j|d< td||j|||j|j|j	|j
|j|jd
|}|S )N	trainableoptimize_attrregularizerdo_model_average	need_clip)
rJ   r   r3   r,   dtype	lod_level
error_clipstop_gradientis_databelong_to_optimizerr   )rm   rn   ro   rp   rq   r   r   rr   rs   rt   ru   rv   rw   )r   src_var	dst_blockdst_varname	dst_shapeZcopied_kwargsparamr   r   r   _partition_parameter  s*   




r}   c                 C   s2   |j |j|||j|j|j|j|j|j|jd
}|S )N)
r   r3   r,   rr   rs   r2   rt   ru   rv   rw   )	
create_varr   rr   rs   r2   rt   ru   rv   rw   )r   rx   ry   rz   r{   r;   r   r   r   _partition_intermediate_var  s   r   c           
      C   s   | |}|jtv rt|dd}|j|j||dd}d}n | |}	t||	}t|tr5t	| ||||}nt
| ||||}t| |}	|	dusKJ | ||	 |S )z.
    partition include: split + replicate
    r2   FT)r   r3   r2   ru   N)r;   r   r   getattrr~   r<   rl   r   r   r}   r   copydeepcopyZ set_tensor_dist_attr_for_program)
r   Z	src_blockry   Zsrc_varnamerz   rx   ZpersistZnew_varrB   ra   r   r   r   r4     s2   






r4   c                 C   s   |j }| j |jv r)|j| j  }|| }||}t|j}||j}|S | j	t
v rE|| }	|	jdks:J t|	j|	j}|S td}
|
dS )Nr   default)r#   r6   rS   Zgrad_op_id_to_op_idrT   r   	impl_typeget_implimpl_idxr   r
   )Zbackward_opr   Zforward_op_id2forward_opr#   Zforward_op_id
forward_opZforward_op_dist_attrdist_op_impl_containerdist_op_implrZ   Zdist_opr   r   r   rX     s6   


rX   c                 C   s$   | | }t|j}||j}|S r_   )rT   r   r   r   r   )r   r   ra   r   r   r   r   r   rV     s   
rV   )r   collectionsr   r.   Z4paddle.distributed.auto_parallel.static.dist_contextr   Z8paddle.distributed.auto_parallel.static.operators.commonr   Zpaddle.frameworkr   r   Zpaddle.staticr   Zdist_attributer	   Zoperators.commonr
   utilsr   r   r   r   r   Z__varname_not_in_block__r   rl   r}   r   r4   rX   rV   r   r   r   r   <module>   s(     `'!