o "j@sddlZddlmZddlZddlmZmZmZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZmZdd lmZd d lm Z m!Z!d d l"m#Z#ej$j%Z%ej$&Z'gd Z(gdZ)e ej*Z+ddZ,e!dGddde Z-ddZ.ddZ/e%j0dfddZ1ddZ2ddZ3dd Z4d!d"Z5d#d$Z6ej7j8j9ej7j8j:fd%d&Z;d'd(Zd-d.Z?d/d0Z@d1d2ZAd3d4ZBd@d6d7ZCd8d9ZDd:d;ZEGdd?d?ZGdS)AN)reduce) ParallelModeis_data_parallel_reduce_opis_parameter_related)new_process_group) _get_comm_group get_logger get_var_numelinsert_dependencies_for_varsis_backward_opis_dep_skip_op is_forward_opis_loss_grad_opis_optimize_op6naive_set_dist_op_attr_for_program_by_mesh_and_mappingset_var_dist_attr get_var_size)core)default_main_programdefault_startup_program) unique_name)PassBase register_pass)AutoParallelStreamType)Zcreate_py_readerZcreate_double_buffer_readerreadslicesplitZassignZsend_v2) ZadamZadamaxZadamwZdecayed_adagradZmomentumZ dgc_momentumZ lars_momentumZmerged_momentumZlambZsgdcCs|jdo d|jdvS)N op_namescopez/auto_parallel/reshard)deschas_attrattropr%q/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/passes/auto_parallel_sharding.py_is_reshard_opLs r'Zauto_parallel_shardingcseZdZfddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Zd+d,Zd-d.ZZS)/ ShardingPasscst|dd|dd|dd|dd|dd|dd|dd|dd|d d|d d|d d|d g|d dt|_g|_i|_d|_d|_g|_ dS)N dist_contextstagesharding_degreedegreeenable_overlapparam_comm_stream_numgrad_comm_stream_numparam_bucket_size_numelgrad_bucket_size_numelpartition_algorenable_hierarchical_comm params_grads global_rankF) super__init__set_attrset dp_groupssharding_infosvarname_to_sharding_infosharding_hybrid_dpouter_dp_groupshared_params_gradsself __class__r%r&r8Xs(               zShardingPass.__init__cCsF|ddur dS|ddvrdS|ddur+t|dtr(|ddkr*dSn|ddurDt|dtrA|ddkrCdSndSt|dd krQdSt|d tr`|d d krbdS|d durkdS|d durtdS|d dur}dS|ddurdS|ddurdS|ddurdS|ddurdSdS)Nr)Fr*)rr+rr,r4rr5r-r.r/r0r1r2r3T)get_attr isinstanceintlenrAr%r%r& _check_selfnsTzShardingPass._check_selfcCsdS)NTr%)rBZ other_passr%r%r&_check_conflictzShardingPass._check_conflictcCs@|d|_t|dp|d|_t|d|_t|d|_|d|_t|d|_t|d|_|d |_ |jd ksI|jd krP|jsPJd t|d |_ t|d |_ |d|_ |d}| | }}||||jD]}||||||||q}|d|j|||dS)Nr)r+r,r*r5r-r.r/r3rz3multiple comm stream need enable_overlap to be Truer0r1r2r4)rG _dist_contextrIsharding_world_sizer*r5r-r.r/r3r0r1r2 global_block_build_sharding_groupsblocks_shard_optimizer_shard_gradient_synchronization_shard_parameterr9r@_optimization_pass)rB main_programstartup_programcontextr4 main_block startup_blockblockr%r%r&_apply_single_implsF       zShardingPass._apply_single_implcCs|||||dSN) _collective_data_parallel_groups_build_sharding_infos)rBrZr4r%r%r&rQs z#ShardingPass._build_sharding_groupscCst|jD]#}t|r|jtvrqt|rqt|j||j}|dur&|j |qt |jdkr8t d t |jdS)NrzuSo far Only and Exactly one data parallel group in network are supported, but got [{}] different data parallel groups) opsr type _skip_opsr'+_inference_data_parallel_group_for_operatorr5rNr;addrJNotImplementedErrorformat)rBrZr$groupr%r%r&r_s$   z-ShardingPass._collective_data_parallel_groupscCsVt|||j}|jD]}|j|jksJd|j|j|j|jdks.Jd|j|j|j|jvs>Jd|j|jt||jksPJdt||j|j|jkrd|_ |j dks`J|j dksgJt|jdksrJd t |j|j|j\}}t |}t ||_n|}||j_t||j||j}|j||jD]}||j|j<qq dS) NzBsharding world size [{}] should not larger than dp world size [{}]rzBsharding world size [{}] should be divisible by dp world size [{}]zBcurrent ranks [{}] does NOT belong to the data parallel group [{}]zDnumber of parameters [{}] is not enough to be shard among [{}] ranksTrErzthybrid sharding and data parallelism are supported only when there is excatly one data parallel group in the network)re_order_programrNr;nranksrOrgr5ranksrJr>r.r/_get_dp_and_sharding_groupsrr?Z_sharding_group ShardingInfor2r<appendparamsr=name)rBrZr4dp_groupr?sharding_group sharding_infoparamr%r%r&r`sb       z"ShardingPass._build_sharding_infoscCs0||||||||||dS)z sharding all optimizer related ops and vars, include: gradient clip ops & vars weight decay ops & vars optimizer ops and states N)_shard_amp_related_op_and_vars_shard_weight_decay_shard_optimizer_ops_and_states_insert_optimizer_broadcasts)rBrZr[r%r%r&rSs   zShardingPass._shard_optimizerc Csf|jdkrdSttt|jD]\}}t||r@|jdkr@|jd}|d|d}||s?|j |dd|j |ddq|j dvrg}|j dD]}|d|d}||rb||qM|rt|j d||j d|q|j d kr|d }|jd} |j| } |j |dd|j|d d| id | jd | jddt|idq|j |ddq|dS)Nrr@Fsync)check_finite_and_unscaleZupdate_loss_scalingXOutr|op_roleZ fill_constantshapedtypevaluerboutputsattrs)r*reversedlist enumeratera_is_param_grad_fp32_cast_opoutput_arg_namesfind_is_parameter_in_local_shard _remove_op _remove_varrbr inputrn set_inputZ set_outputr"vars_insert_op_without_syncrr OP_ROLE_KEY_sync_with_cpp) rBrZidxr$ output_name param_nameZ reversed_x input_namerout_nameZout_varr%r%r&ru*sL            z+ShardingPass._shard_amp_related_op_and_varscCs|jdkrdSgd}t}t}tt|jD]8\}}t|s!q|j|vrP|dd}|d|d}| |sP| ||jdvrP|j D]} | | qHqt tt|jD]\}}t|scqZ||vrn|j |ddqZ|D] } |j| ddqqtt|jD]_\}}t|sq|jd krg} |jD] }||vr| |q|jd| |j d} t|jD],\} }|j|| d d d| gid | gid |jjddddttjid}|j|| }qnq|dS)NrE)Zelementwise_mulsquared_l2_norm clip_by_normr}r@GRAD)rrFrzsumrc_allreduce_sumr~ring_idrz /gradient_clip_model_parallelismuse_calc_streamTrbZinputsrr)r*r:rrra_is_gradient_clip_oprbrrrrerrrrinput_arg_namesrnr rr<Z _insert_oprhidrOpRoleOptimizerN get_tensor_dist_attr_for_programvarr)rBrZZremoved_op_typeZremoved_op_idxZremoved_tmp_varrr$rrrvarnameZ reserved_varsZ sum_op_outputirsnew_op dist_attrr%r%r&_shard_gradient_clip[sn               z!ShardingPass._shard_gradient_clipcCsF|jdkrdSttt|jD] \}}t|sqtd|dS)NrEz$weight decay is NOT supported by now)r*rrrra_is_weight_decay_oprfr)rBrZrr$r%r%r&rvs  z ShardingPass._shard_weight_decaycsBg}ttt|jD]K\}}t|snB|jtvrVd|jvs!Jt| ddks,J| dd| sM| fdd|j D|j |ddq |j|q ttt|jD]\}}t|j dkry|j d|vry|j |ddq`|D]}||r|j|dd||r|j|ddq|||dS)NParamrrcsg|]}|kr|qSr%r%).0rrr%r& s z@ShardingPass._shard_optimizer_ops_and_states..Frz)rrrrarrb_supported_optimizer_typeZ input_namesrJrrextendrrr@rn_get_param_gradhas_varrr)rBrZr[Zshould_removed_optimizer_statesrr$rr%rr&rws@      z,ShardingPass._shard_optimizer_ops_and_statescCs|jdks |jdkr dS|jD]S}|jD]M}||jsJ||js&J|jdd|id|id|jjd| |jdd t t j id }| d d tj|j|}|dusWJt||j|j|jqq|dS) NrEr c_broadcastr}r~rrootrTrr/)r*r0r<rorrp append_oprhr get_var_rankrrr _set_attrr DataParallelrNrr process_mesh dims_mappingr)rBrZr[rsrtrZparam_dist_attrr%r%r&rxs<       z)ShardingPass._insert_optimizer_broadcastscCs"||jvsJ|j|}||Sr^)r=is_in_local_shard)rBrrsr%r%r&rs  z)ShardingPass._is_parameter_in_local_shardcCs2||jvsJ|j|}||}|dusJ|Sr^)r=get_param_grad)rBrrsZp_gr%r%r&rs    zShardingPass._get_param_gradc Cs|jdkrdSdd|jD}ttt|jD]h\}}t||ra|jd}t|}|j |}t ||||j j | ||j}|jrF||sP|j|dddn|d|jj |d d tjt||r|jd} t| }|j |}||s|j|ddq|dS) NrEcSg|]}|jqSr%rrrhr%r%r&rz@ShardingPass._shard_gradient_synchronization..rrFrzrrr)r*r;rrrra_is_param_grad_allreduce_opr_get_base_name_from_grad_namer=_insert_reduce_oprhrrrNr>rrrr?rr_is_param_grad_sum_oprr) rBrZ dp_ring_idsrr$r base_namersZ reduce_oprr%r%r&rTsB           z,ShardingPass._shard_gradient_synchronizationc Cs|jdkrdSdd|jD}|jD]O}||\}}g}|D]}||dkr5|||jkr5||q ttt |j D]r\} } t | rHq?| j D]e} t || |jr_|| s^|| qK| |vrdqK|| } | |jkrq| } n-t| d} || }|j| |j|jdd}|j|}t|j||j|j}| | | t|| | |j| |jj| d|jqKq?ttt |j D]&\} } | j d krq| j d} | j!d}| |vr|j"| dd |j#|dd qttt |j D]Y\} } t$| j!d ksJ| j!d}| j d kr)| d |vr)|j%r!|||jkr!| &d |j%jn|j"| dd q| j d krD||vrD|||jkrD|j"| dd q|D]}|||jkr`|j#|dd |j#|dd qGq|'|'dS)NrFcSrr%rrr%r%r&r1rz1ShardingPass._shard_parameter..rz @BroadCastF)rprr persistablercastrzrrr)(r*r;r<"get_broadcast_vars_and_param_usager local_rankrnrrrrarr_is_param_fp16_cast_op param_namesrrgenerater create_varrrrNrrrr _rename_input_insert_init_and_broadcast_oprhrr"rbrrrrJr?rr)rBrZr[rrsZneed_broadcast_vars param_usageZnot_used_param_namerrr$r root_rankbroadcast_varname input_varZnew_varZ ref_dist_attrZout_var_dist_attrrr%r%r&rU-s           4        zShardingPass._shard_parametercCs|jdkrdSd|_d|_d|_t|jdks!Jdt|j|jd}tj ||@| ||j dkrU|jdkrC| |n"|jdkr]| |WddSWddSWddSWddS1spwYdS) NrZsharding_coalesce_grad_Zsharding_coalesce_param_r6z^gradient synchronization optimization only support one sharding group right now, but got [{}].rrErF)r*grad_coalesce_prefixparam_coalesce_prefixcomm_op_scheduling_priorityrJr<rgpaddleZstaticZ program_guard_gradient_sync_optimizationr0&_fuse_overlap_parameter_comm_stage_two(_fuse_overlap_parameter_comm_stage_three)rBrWrXrsr%r%r&rVs2        "zShardingPass._optimization_passcCsL|jdkr |js dSt}t}|||\}}|||||dS)Nr)r1r-rrPr _group_grads_overlap_grad_comm)rBrsrZr[coalesce_to_group_mapgrad_name_to_group_mapr%r%r&rs  z(ShardingPass._gradient_sync_optimizationcCst}t}t||j\}}tdtd|jt| t| i}|j rcg|_ |j j }t|jD]}|dkrE|j } nt|dd} |j | tjjdq;td|ji|_t| D]\}} t| dksuJt| dkrt|jt|} |j| | jddd |j| | jddd | _|jd d | ji| j| jd d dddd| jt t!j"idn| jd| _td|t#dd| jDt$d|ddd| jDd| || jj%<||j} |j | d} |j | d}|jdd| jid| jid| j&d| j'ddt t!j(id}| |j|<|)d d!t*j+|j r-||j,_-|j.|j,_/qii}t|j0D]\}}t1|r|2dd}|j|}||} d}|j rX|j,j-}t| |jkrqt3|}|j|2d"d}n|j0||j}t1|sJd#|j|2dd}||g|g|fg||<t| jdkr| j4r| jd$}||||g|g|f|||d|g| j|fq5t5| dd%}|D]2}||ddd$D]%\}}}}t6|||||j7t!j(d$gd&d&d'd( }|j r||j,_-|j.|j,_/qېq|8dS))NzSharding Stage2 Optimization:zFParam Bucket size is [{}], [{}] Parameters are fused into [{}] BucketsrTZforce_new_group comm_group comm_streamz/Parameter Communication would use [{}] streams.rrprrZ stop_gradientcoalesce_tensorInputOutputZ FusedOutput copy_data use_alignrrzBucket[{}] size [{}]MB.cSg|]}t|qSr%rrpr%r%r&r#zGShardingPass._fuse_overlap_parameter_comm_stage_two..zBucket[z] parameters: cSrr%rprr%r%r&r'r.rrrr}r~rrrrrZParamOutz:Unexpected: sharding broadcast pre op should be broadcast.r6reverseFZsharding_stage2_broadcast_deprZ is_recomputer{r)9rrPr group_paramr0_loggerinforgrJkeysr-Zparam_comm_group_stream_pairsrhrkranger.rrnrZSHARDING_STREAMrZop_to_stream_idxrrrrstrrr coalesce_varrrrrZForwardrdebugrprrankrrrrrexecution_streamrscheduling_priorityrais_sharding_param_broadcast_opoutput_get_broadcast_first_depend_oprsortedr rNr)rBrsrZr[group_to_param_mapparam_to_group_mapZbroadcast_var_to_group_maprkrrhZ param_groupZcoalesce_var_nameZcomm_stream_idxrrrdep_mapr$r broadcast_varZ prior_varZpre_opZ last_gradindicer prior_vars post_vars depend_opr%r%r&rs2                z3ShardingPass._fuse_overlap_parameter_comm_stage_twocCsdSr^r%)rBrsr%r%r&rrMz5ShardingPass._fuse_overlap_parameter_comm_stage_threec)Cs<|j}|jdkr d|_d}|D]}t|r|}q|durdS|jd}t|j}g}t} dd} d} | t|kr || }t|r|jdksKJd|jd} t | } | | }| | }| ||rq| | vsjJ| ||n||t|j}| ||t|jdkr| d|_d}t|| |r|d7}t|| |s|| |}|jd}||_| | |j| |jr|| rd |_|| djd ksJd|| djd| ksJd |j| d| d7} n| ||r||t|j}| d7} | t|ks:t|jdkr||td td |jt| t|i}i}i}i}g}t|D]`\} }t|jdkr|jt|jt | |j!dd d|_"|||j<|j#}|||<|$|j|jr~|j#}|||<|$|jn|jd|_"|jD]}|||j%<q|||j"j%<q8t|&}t|&}t|}|'|}t|dksJ|'|}t|dksJ|'|}t|dksJt(t)t|jD]\} }| |vr|| }|jd} | |jdj%ksJd|jdj%| |*| |j"j%|+| |j"j%| |vr|j,| dd| |vr|| }|jdj%}!|!|jvs>Jd|!t |dd|jD}"g}#g}$|jD]}%|%j-}&|#$|&|$t|&qM|j.| dd|"i|"|j"ddddd d|j!d|#d|$t/t0j1id}'t2|| | |j|j"|j3t0j1dgdddd }(q|4||fS)!a conditions for gradients to be grouped: 1. group size < grad_bucket_size_numel 2. same dp group (TODO) 3. same src rank 4. same dtype 5. dependency: grad would NOT be used by other ops within group segment main logic: 1. record coalesce group 2. record all dp allreduce/reduce op idx 3. insert coalesce op 4. insert coalesce dependency (avoid allocate memory too early) 5. modify and remove allreduce/reduce op 6. ensure sharding-dp hybrid parallel logic gradients inside same group would be fuse into one coalesce tensor rNrcSs2t|j|j}dd|jD}t||dkS)NcSsh|]}|jqSr%rrrr%r%r& rzHShardingPass._group_grads..op_depend_on_group..r)r:rrrrJ intersection)r$rhZvars_Z var_namesr%r%r&op_depend_on_groupsz5ShardingPass._group_grads..op_depend_on_group c_reduce_sumzZSharding should reduce grad first and than allreduce if Hybrid Sharding with Data-ParallelrETrz@Hybrid Sharding with Data-Parallel should sync same gradient varz-Sharding Gradient Communication Optimization:zIGradient Bucket size is [{}], [{}] Gradients are fused into [{}] Buckets.Frr6z4Unexpected: it is supposed to sync [{}] but got [{}]rzz=Unexpected: op is supposed to generate grad [{}] but got [{}]cSrr%r)rgradr%r%r&rGrz-ShardingPass._group_grads..rrrrrrconcated_shapesconcated_ranksrZsharding_grad_coalesce_depr)5rar1rrVarGroupr:rJrrbrrr acceptablecollectrnrcoalesce_op_idxr coalesce_dep_varnamerereduce_op_indicesr>rallreduce_op_indicesrrrgrrrrrrrrpoprrprrrrrZ_rename_outputrrrrrBackwardr rNr))rBr\rsraZfirst_backward_opr$Zfirst_backward_varname cur_groupZ grad_groupsZgrouped_grad_namesrr grad_namerrZgrad_varjZdep_opZ dep_varnamerrZmodify_reduce_op_mapZcoalesce_op_mapZremove_reduce_op_indicesrhZlast_reduce_op_idxZlast_allreduce_op_idxrZcoalesce_op_setZ modify_op_setZ remove_op_setZconfilctrZfirst_grad_nameZ grad_namesrrZgrad_rZ coalesce_oprr%r%r&rsN                      4                         zShardingPass._group_gradsc)s~|jsdSg|_|jj}t|jD]}|dkr|j}nt|dd}d|}|j||dq|j} i} d} i} t | D]\} }t |r|j dkrKq=| |j}|| |<|j|d}|j|d }| d d}||}|j j|ksuJt|jd kr| |jd |j |fg| | <| d }|jr|jr|d 7}| | ||j |j|f||j_|j|j_|d |j|jr|jr| | d }|j dksJ| d d|ksJ||j_|j|j_| d 7} | d 7} | d 7} q=t| dd}|D],}| |ddd D] \} }}}t|| |||jtjd gdddd }||j_|j|j_qq|jr9d|j dks/J|j}|j}|j!|j!fdd|D}t"#dt"#d|j!dt"#d|dt||j ksmJfdd|D}t|ksJt"#d|dg}g}t|jD]} |t|dd|t|ddqt$t%t |jD]\} }t |r7|j dksJ| |}!||!}"||!}#| d d}|jr|jj}t&|'d}$d}%|$krd}%|$}&|d |#j|d|&|%r7|$}'|j(| d dd|id |id |"jd|'ddt)tjid}(|(d d!t*j+|jr7||(j_|j|(j_q|,dS)"a overlap gradient communication with backward & optimizer computation. 1. assign gradient communications to grad comm stream 2. for coalesce gradient communication: 2.1 insert before communication dependencies 2.2 insert after communication dependencies only when need 3. there is not need to add explicit dependencies for non-coalesce gradient communication P.S. this overlap pass is ONLY adapted for standalone executor (graph based) and stream awared allocator. NrTrZsharding_grad_comm_streamrrrrr~rr6rrFZsharding_grad_comm_deprcsg|] }|kr|qSr%r%rr)nranks_per_noderelative_idx_in_noder%r&r  z3ShardingPass._overlap_grad_comm..z:Sharding Gradient Hierarchical Communication Optimization.zcurrent global rank idx: rzlocal inter node ranks idx: csg|] }|kr|qSr%r%r%)node_idxr&r%r&rr(zlocal intra node ranks idx: rroot_idr}rrrr)-r-Zgrad_comm_group_stream_pairsrhrkrr/rrnrarrrbrrrprJrr>rrrrrrrrrr rNrr r3rOr5rrrrrIr"rrrrr))rBr\rsrrrkrrhstreamrar Zreduce_op_countZgrad_comm_op_to_stream_idxrr$Z stream_idxrrZreduce_varnameZ grad_groupZpost_idxZnext_opr r rrZ global_groupZ global_ranksZinter_node_ranksZintra_node_ranksZinter_node_groupsZintra_node_groups_Zgrad_comm_stream_idxZinter_node_groupZintra_node_groupZdst_rankZin_peerZintra_node_dstZinter_node_dstrr%)r)r&r'r&rss6                   zShardingPass._overlap_grad_comm)__name__ __module__ __qualname__r8rKrLr]rQr_r`rSrurrvrwrxrrrTrUrVrrrrr __classcell__r%r%rCr&r(Vs4 (09 1D (#*y<dr(cCs&|jD] }|jtvr|Sqtd)NzCould not find optimizer op.)rarbr Exception)r\r$r%r%r&rWs  rc Cs||}||} |j|dd|id|id|d|ddt|id} | d d tjt| | j| j |||krU|j|d d|j id |j d |j t|id} t| | j| j |dSdS)z% empty op for initialization rr}r~rrrTrrremptyrrrN) rrrrrrrrrrrprr) r\ insert_idxrrrrrr)r Zbroadcast_var_dist_attrrr%r%r&r_sN   rTc Cs~|dks Jd||j|dd|gid|gid|d|d|t|id }|||} t|| j| j||d d tj |S) Nrz5root id should be a positive int, but now root id is rr}r~rr*rrrr) rrrrrrrrrr) r\r3Z reduce_varrr*r)rrrrr%r%r&rs,   rcCs<d}d}t|||g}t||||}t||||}||fS)Nrr)rJr)Z origin_groupZsharding_group_sizerZdp_axisZ sharding_axisrrqrrr%r%r&rls rlcC|jdo|jddS)Nrz/gradient_clipr r!r" startswithr#r%r%r&r rcCr4)Nrz/regularizationr5r#r%r%r&rr7rcCs`t|sdSt||tjjjtjjjsdS|jd}|d|d}| |s*dS| |j S)NFrry) r _is_desired_cast_oprVarDescVarTypeFP16FP32rrrr is_parameter)r\r$rrr%r%r&rs   rcCs4t|rdSt||s dS|jd}||vrdSdS)NFrT)rr8r)r\r$rorr%r%r&rs  rcCsn|jdkrdSt|jdksJt|jdksJ||jd}||jd}|j|ks3|j|kr5dSdS)NrFrrT)rbrJrrrr)r\r$Z src_var_typeZ dst_var_typerZ output_varr%r%r&r8s r8cCs@d}d|vr|d|d}|Sd|vr|d|d}|S)Nz.cast_fp16@GRADr)r)r"rr%r%r&rsrcCs8t|sdS|jd}t|}||sdS||jS)NFr)rrrrrr=r$r\rrr%r%r&rs   rcCsFt|sdS|jdkr dS|jd}t|}||sdS||jS)NFrr)r rbrrrrr=r>r%r%r&rs    rcCs(|jdko|jdotj|jdvS)Nrr)rbr r!rrr"r#r%r%r&r#s  rc Csd}|jD]>}t||j|sC||}|j}||}|j}t|dkr%q|d} | dkrC|| dkrCt|j |j| |} t | }|Sq|S)Nrr6r) rrr\get_op_dist_attr_for_programrZget_input_dims_mappingrrJrZ process_idsr) Zrank_idr$r)rqrrrZinput_dim_mappingZ mesh_shapeZbatch_size_axisZ group_ranksr%r%r&rd+s,    rdc Csi}d}g}|D]}t|}||7}|||fqddt|D}d}d}|D]\}}||d|d|kr=|d7}|||||7}q)|S)z shard the continouse param into same rank and divide the forward&backward computation into segement, which will favor the fuse pass in later. we assume that the params is already sorted by utilization order. gcSsi|]}|gqSr%r%rxr%r%r& Urz*partition_by_use_order..rg?r)rrnr) ro group_sizemappingZtotal_param_memZ param2memrtZmemZcur_rankZmem_accur%r%r&partition_by_use_orderGs   rEcCsi}t|D]}g||<qdg|}|D]1}|t|}|||tdd|jd}|dks=Jd|jd|d|||7<q|S)zJ use greedy alogrithm to partition parameter as even as possible. rcSs||Sr^r%)rAyr%r%r&lsz*partition_by_greedy_even..rzparam [z#] should larger than 0, but it is [])rindexminrnrrrp)rorCrDZrank_sizesrtrnumelr%r%r&partition_by_greedy_evenas    rM greedy_evenc Cs|dkr t||}nt||}td|D]%\}}td|dtdd|Ddtdd d|Dd q|S) NrNzSharding Parameter Partition:zRank:z, Parameter Size:cSrr%rrr%r%r&r~rz(partition_parameters..z MB.zParams in this rank: cSrr%rrr%r%r&rrr)rMrErritemsr)rorCZalgorrank_to_paramskvr%r%r&partition_parametersus   rScsi|D] \}}||f|j<qg}|jD]}|jD]}|vr)||vr)||qt|tkr4nq|jd}i} t|j} g} t|r|jtvrtt t |jD]"\} }|jtvrwt| ddksiJ|| | dd<| | qUt|t| ksJ|D]} |j dd}|j | | j |||| | q| D] } |j| ddq|t|j| ksJtd |d fd d |DS) Nr6rrrZnop)rbFrzz(Sharding the Order of param being used: rcsg|]}|qSr%r%rZpname_to_pg_pairsr%r&rrz$re_order_program..)rprarrnrJrrbrrrrrrr Z copy_fromZset_op_dist_attr_for_programr?rrrr)r\Z param_gradsr)rgZ use_orderr$rZlast_opZ pname_to_opZnum_opsZremove_op_indicesrZpnamerr%rTr&risL          ricCsi}i}g}t|}|jD]>}||j}|||r"|||n t|}|||||j|_||vr@|||jn|jg||<|||j<q ||fS)zA param are group by: rank id fuse_size dtype )rrorrprrrrn)rsZ fuse_sizerr Zbucketr!rtrr%r%r&rs$      rc@s<eZdZddZddZddZddZd d Zd d Zd S)rmcCs||_dd|D|_t|jtt|jksJddd|D|_dd|jD|_|j|_||_|j |j|_ ||_ t |j|j|j |_i|_|dS)NcSsi|] \}}|j||fqSr%r)rrrUr%r%r&rBsz)ShardingInfo.__init__..z&found duplicated param in params_gradscSsg|]\}}|qSr%r%)rrr,r%r%r&rrz)ShardingInfo.__init__..cSrr%rrr%r%r&rr)rhr4rJr:rorrjrCr5rkrIrr2rSrP param_to_rank_map_param_to_rank)rBrhrr4r2r%r%r&r8s$   zShardingInfo.__init__cCs.|jD]\}}|D]}||j|j<q qdS)z@ mapping parameters to the rank which holds it. N)rPrOrVrp)rBrrortr%r%r&rWs zShardingInfo._map_param_to_rankcCs||jvr |j|SdS)Nr6)rV)rBrr%r%r&rs  zShardingInfo.get_var_rankcCs|||jkSr^)rrrBrr%r%r&rszShardingInfo.is_in_local_shardc Cst}t}i}dd|jD}|jD]}t|rq|jD]}||jvr,||d7<qq|jD]2}t|||js;q1|jd}|jd}|||||||<||d8<|j||j|<q1| D] \} } | dkru|| qh||fS)NcSsi|]}|dqS)rr%r@r%r%r&rB rzCShardingInfo.get_broadcast_vars_and_param_usage..rr) r:rrarrrrrerVrO) rBr\Zbroadcast_varsZ fp16_paramsZ fp16_to_fp32rr$rrrtusager%r%r&rs6         z/ShardingInfo.get_broadcast_vars_and_param_usagecCsB||s td|d||jvrtd|d|j|dS)Nzparam[z] not in current rank.z] not in params_grads)r ValueErrorr4getrXr%r%r&r#s  zShardingInfo.get_param_gradN) r-r.r/r8rWrrrrr%r%r%r&rms rmc@s,eZdZddZddZddZddZd S) rcCsF||_d|_d|_d|_g|_d|_d|_d|_g|_g|_ d|_ dS)Nr6rF) max_siezrrrLrrrrrrr)rBmax_sizer%r%r&r8,s zVarGroup.__init__cCsH|jdkrdS|j|jkrdS||jkrdS|jt||jkr"dSdS)NrTF)rLrrr r\rBrtrr%r%r&r9s   zVarGroup.acceptablecCs0|j|_||_|jt|7_|j|dSr^)rrrLr rrnr^r%r%r&rEszVarGroup.collectcCs t|jSr^)rJrrAr%r%r&__len__Ks zVarGroup.__len__N)r-r.r/r8rrr_r%r%r%r&r+s   r)rN)Hlogging functoolsrrZ8paddle.distributed.auto_parallel.static.operators.commonrrrZ5paddle.distributed.auto_parallel.static.process_grouprZ-paddle.distributed.auto_parallel.static.utilsrrr r r r r rrrrZ7paddle.distributed.fleet.meta_optimizers.sharding.utilsrZpaddle.frameworkrZ paddle.staticrrZ paddle.utilsrZ pass_baserrZ pass_utilsrZop_proto_and_checker_makerrZkOpRoleAttrNamerrcrINFOrr'r(rrr rrlrrrrr9r:r<r;r8rrrrrdrErMrSrirrmrr%r%r%r&sn  4       ? $     4"P