o "Õj·µć@s.ddlmZdadd„ZedkreƒdSdS)é)ŚContextNccCs¬tƒat ”rddlm}| ”dSt ” rAddl}ddl}ddl}ddl }ddl }ddl }ddl m }ddlm}ddlm} m} m} m} m} dd lm}|  ”}tjj d ”s_td ƒ‚zttjjd ƒ }| |”}Wdƒn1swwYWntd ƒ‚| d”}| |j ”|j! "|j! #tjj”|j! $tjj” %d”d›d”}|j&|dd}| |j ”| 'd”}| (|”| )|”tjj* d”rķ|j+ ,d”dkrä|j-ddddddtjj*g}n|j-dtjj*g}ntjj*g}| .tjj/”| 0tjj/”}tjj1sd}n t2tjj1 %d”ƒ}tjj3}t4|t5ƒr%t6| %d”dƒ}nt6|ƒ}||d <||d!<||d |d"<| ,d#d”sFd$d%i|d#<| ,d&d”}|j! "|j! #tjj”|j! $tjj” %d”d›d'”}g}d}|dkr/dd(l7m8}d)tjj9vsJ‚tjj9 :d)” %d”\}} ||| d*}!|! ;d+”|! ”}#|" ?|" @|#””}Wnd,}Y|d,ks»J‚|d# ,d-d”r/| Ad.”d/|›}$|! B|$|› Cd0””sé| Ad”|! B|$|› Cd0””rŁtD|! Ed/”ƒ}%t2|%ƒ}&|&|kr| Ad”|! B|$|› Cd0””tD|! Ed/”ƒ}%t2|%ƒ}&|&|ksłtFd1d2„|%Dƒƒ}| Gd3t2|ƒ›d4|›d”| ,d5d6”}'|'|d5<|'t_Hd7|vrC|'n| ,d7”}(| ,d8d”})d9}*|ƒ}+d},d}-d:tj_I| 0t”}.| ,d;i” ,d|/d#<||/ƒ}0|0 J”}1d}2|1r±| 0|.”ad? K|1d<|1d@|1dA|1dB|1dC|1dD|1dE|1dF|1dG” }3|3tj_L|,d7},dHt5|,ƒ}4|4tj_M| ||1|/ƒ}5|5tj_/tjN GdI K|4|3|1””| GdI K|4|3|1””| Ot”}6|6 P”| tjjLdJ|dKd$tjjM›dLdM\}7}8}9|9d@rtjN QdN|3›”| QdN|3›”d:|1dO<d|1|dKd$<|8|1dP<|9d@rCtjN QdQ|3›”| QdQ|3›”d:|1dO<d|1|dKd$<dR|1dP<|9dS@rb|9d@sbtjN QdT|3›”| QdT|3›”d|1dP<|9su|7|1dO<|7|1|dKd$<|8|1dP<|9d@s|9d@r€n1|,|1dU<|1d<}2|+jRd¢i|1¤Ž|6jSdVdW|+ TdX”|0 J”}:| 0|:”}1|0 R|1”| AdY”|1sƒ|2dur½tdZ K|/”ƒ‚|2|d;d<<|+ TdX”|+ U”|  ”};tjN Gd[|;|›d\”| Gd[|;|›d\”||ƒ}<| Gd]t2|| 0|.”a|*r|(t_HdV}*d<|=vr'|=d<n|d;d<}?|?|=d@|=dC|=dE}@|@|=d`<|?|=d<<da|=vrodb K|?|=d@|=dA|=dB|=dc|=dC|=dD|=dE|=dF|=dG|=d`|=da” }3n#dd K|?|=d@|=dA|=dB|=dc|=dC|=dD|=dE|=dF|=dG|=d`” }3|j! "|j! #tjj”|3”tj_L|,d7},det5|,ƒ}4|4tj_M| ||=|ƒ}A|Atj_/tjN Gdf|4›dg|3›”| Gdf|4›dg|3›”| t|=|ƒat6tjj3 %d”dƒ}B|r÷|d|B…}C||Cvr÷|! ,dh|3›”d}=|  ”}D|=s)|  ”}E|E|D|d5dikrtdj|3›dkƒ‚| AdY”|! ,dh|3›”d}=|=r| Gdl|3›dm”tjN Gdl|3›dm”| X|= Y””}=|dƒ|=dy<tjN Gdz K|4|3|=dy””| Gdz K|4|3|=dy””| tjjLdJ|dKd$tjjM›dLdM\}7}8}9d9}M|9d@}N|Bdkr”dh|,›d{|›}$|NrŸ|! B|$dR Cd0””s| Ad”|! B|$dR Cd0””r~tjN Gd||$›”| Gd||$›”n™t`|6d}ƒrÖ|6jad~krÖ|! B|$d Cd0””sÄ| Ad”|! B|$d Cd0””rµtjN Gd€|$›”| Gd€|$›”nbt`|6d}ƒs|6jbjcdkr|! B|$d Cd0””sü| Ad”|! B|$d Cd0””rķtjN Gd€|$›”| Gd€|$›”n*|! B|$d Cd0””s'| Ad”|! B|$d Cd0””rtjN Gd‚|$›”| Gd‚|$›”tD|! Edh|,›d{”ƒ}Ot2|Oƒ}&|&|Bkre| Ad”tD|! Edh|,›d{”ƒ}Ot2|Oƒ}&|&|BksLdƒd2„|ODƒ}PtjN Gd„|,›d…|P›”| Gd„|,›d…|P›”dR|PvrdV}Md9}Nnd|Pvr”dV}MdV}G|9d@rÅtjN Qd†|3›d‡”| Qd†|3›d‡”d:|=dO<d|=|dKd$<|Nsæ|8ndR|=dP<d9}G|9d@rķtjN Q|3›dˆ”| Q|3›dˆ”d:|=dO<d|=|dKd$<dR|=dP<d9}G|9dS@r|9d@stjN Qd‰|3›d‡”| Qd‰|3›d‡”|NsdndR|=dP<|Gs.|Mr.|7|=dO<|7|=|dKd$<|Ns*|8ndR|=dP<|GsI|MsId:|=dO<d|=|dKd$<|NsEdndR|=dP<|dKd$|=vrZd|=|dKd$<|,|=dU<dŠ|d#vräd‹|d#dŠvräda|=vrä|=|dKd$}Q|d#dŠd‹}R| tjjLdJ|RdŒ}S|d#dŠ ,ddŽg”}Tt6|d#dŠ ,dd”ƒ}U|d#dŠ ,d‘dV”}V|TD]4}W|Vr»|Ud’|W}Xn|UdS|W}X|QrŅ|SrŅt_|S|S|X|Qd.ƒnd}Y|Y|=d“|W›d”|dKd$›<qÆ|G|=dr<|Grļ|-d7}-d}H|H|=ds<t2| ózlaunch..zThe total count of nodes is z and sorted ips are Śmax_time_per_taskiŚ warmup_timeŚmax_search_timeTé’’’’Z model_cfgŚglobal_batch_sizeŚautoZgbsz\GBSSearch/GBS{}_DP{}_MP{}_PP{}_Sharding_degree_{}_stage_{}_MBS{}_Recompute_{}_granularity_{}Z dp_degreeZ mp_degreeZ pp_degreeZsharding_degreeZsharding_stageZmicro_batch_sizeZ use_recomputeZrecompute_granularityZ gbs_tuner_z=Launch task from auto tuner: job_id {}, log_dir {}, config {}z workerlog.0Z metric_cfgz.gpu.log)ŚpathZ metric_fileŚ target_metricZ memory_filez#Read metric failed for parameters: ŚtimeZ max_mem_usagezOut of memory for parameters: ZOOMéz)Read memory usage failed for parameters: Śjob_idF)Śexitz./tuner_gbs_history.csvézTNo valid global batch size found, check memory or valid search time. cur_tuner_cfg{}z!AtuoTuner for GBS search ends in zs.zLaunch z tasks by auto tuner: zNo config can run.Ś acc_stepsZsharding_overlapzdGBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}_Overlap_{}Z vpp_degreezYGBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}Z auto_tuner_zLaunch task: job_id z , log_dir z auto_tuner/ézWait z failedzReceive that task z has ended by etcd.ZOptimizationDirection)ŚmetricŚ directionzCurrent best config: z7Get best config failed. Currently no config can be run.Ś has_errorŚ error_infozaAuto Tuner Schedule: [{}/{}], Pruned nums {}, Error nums {}, Error info {}, Remaining time {} miné<Ś1Z$FLAGS_shard_bypass_dygraph_optimizerŚ0Z exec_timez(Task: job_id {}, log_dir {} ended in {}sś/z Put OOM to ŚsigintéŚOKz Put OK to ŚErrorz Put Error to cSr!r"r#r%r(r(r)r*1r+zStatus of auto_tuner/z/: zRead metric of z failed.z OOM.zRead memory usage of Ś conversionŚ step_time)r2Śfiler3Ścomm_bwédŚ model_size_béŚampéZbw_Ś_z put auto_tuner/z successfully.z-Get best config failed, no config can be run.z;fuser -v /dev/nvidia* |awk '{for(i=1;i<=NF;i++) print $i;}'zkill -9 ZCOLLECTIVE_MASTER_IP)r;r<rzCGet best config failed. Currently there are no appropriate configs.é zAutoTuner ended in Śrun_best)rRzLaunch best cfg: r()mrŚctxZis_legacy_modeZpaddle.distributed.fleetrZis_auto_tuner_modeŚcopyŚjsonŚloggingŚosŚsysr4Zauto_tuner.recorderrZauto_tuner.tunerrZauto_tuner.utilsrr r r r Śr ŚargsZauto_tuner_jsonŚendswithŚ ValueErrorŚopenŚloadŚ getLoggerŚsetLevelŚINFOr2ŚjoinŚdirnameŚbasenameŚsplitŚ FileHandlerŚ FormatterŚ setFormatterŚ addHandlerZtraining_scriptŚenvironŚgetŚ executableŚextendZtraining_script_argsŚdeepcopyZdevicesŚlenŚnnodesŚ isinstanceŚstrŚintZutils.etcd_clientrZmasterŚstripŚdeleteZ delete_prefixŚsocketŚ gethostnameŚ gethostbynameŚgetfqdnŚsleepŚputŚencodeŚlistZ get_prefixŚsortedŚinfor,Z max_restartZ search_onceŚformatŚlog_dirr6ŚloggerŚinitrŚwarningZadd_cfgŚfinalizeZ store_historyZ clean_historyŚalgoZ all_tasksŚloadsr$Z history_cfgsŚpopZget_bestŚhistoryŚidxŚroundŚhasattrrCZpodŚ exit_codeŚdumpsŚgetpidŚpopenŚ readlinesŚsystemŚrangeŚ Exceptionr7rR)crrTrUrVrWrXr4rrrr r r r r Ś start_timeŚfZ tuner_cfgr‚Zauto_tuner_log_pathŚhandlerŚ formatterŚ entrypointZraw_argsrrprZhistory_file_pathZ sorted_ipsŚiprZ master_iprŚclientrvŚhostnamer2ŚipsŚsizer,r-r.Z is_first_taskZrecorderr6Zerror_task_numsZraw_ctxZ gbs_tuner_cfgZ gbs_tunerZ gbs_cur_cfgZbest_gbsrZ task_job_idZ gbs_new_argsŚcr;ZmemŚerrZ gbs_new_cfgZend_timerZcur_cfgZtask_start_timer0r9Śnew_argsZ actual_nnodesZactual_exec_ipsZwait_start_timeZ wait_end_timeZ cur_best_cfgsr=r>Z task_numsZ cur_task_idZnew_cfgZ task_end_timeZ timeout_flagZOOM_flagŚresultŚstatusZsingle_dp_performanceZstep_time_metricrHrJrLrNŚbwZ comm_timeZmulti_dp_performaceZself_pidZ processesŚprocessŚpidrZcollective_master_ipŚdatar'Śer(r(r)rsL       ’€  ž ’  ų     ž        ’   ü’  ’ż  ’  ÷  ’’’’’    ü ’’ ’’’     — k’’  ’’’   ’ ż’žż’ ōõ ’  ’    ’ ų ’ ’   ’ ž’’ ’’ ’žüś’’žüś’       ’’ ’’   ü  ’ ’ ’ ’   ż        ’ż ’ ’’ śś’žüų ž’ žś’ žś’ ž üž   ž       ’ž  € ’ź       ż’  ž   €ż’    ż’’’    rŚ__main__)ŚcontextrrSrŚ__name__r(r(r(r)Śs * ’