o *jO: @s>ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z m Z ddl ZddlZddlmZddlmZddlmZdefddZdedefd d Zd d ZdJdededdfddZdeddfddZdeddfddZdKdede eddfddZ dKde eeffddZ!ddZ"ddZ#ddZ$d d!Z%d"d#Z&dKd$d%Z'dKd&d'Z(d(d)Z)d*d+Z*d,d-Z+e,d.d/Z-d0d1Z.d2d3Z/dKd4d5Z0d6ej1j2defd7d8Z3d9e d:ej4fd;edZ5d?e6ed@edAedBe7ede8e7eej4ff dCdDZ9 dLdEej1j:dFej;dGedej1j:fdHdIZ#s z _is_free_port..)r gethostbyname_ex gethostnameappendr r all)ripsrrr _is_free_ports  $r#cKs`t|dr|jdi|}|Sttjtdkr%tj|fi|}|Stdtjd|S)Ncompilez 2.0.0.devzDCompiling model needs torch version > 2.0.0, your torch version is: z , origin model will be returned.r)hasattrr$rparsetorch __version__print)modelZcompile_optionsrrr compile_model&s r+nccllauncherbackendcKs|tjdddur td|dkrt|fi|dS|dkr)t|fi|dS|dkr7t|fi|dStd|)NT) allow_noneZspawnZpytorchZmpislurmzInvalid launcher type: )mpZget_start_methodZset_start_method_init_dist_pytorch_init_dist_mpi_init_dist_slurm ValueError)r-r.kwargsrrr init_dist4s r7cKs2ttjd}tj|tjdd|i|dS)N LOCAL_RANKr.r)intosenvironr'cuda set_devicedistinit_process_groupr.r6Z local_rankrrrr2As r2cKsxttjd}tj|dtjvrdtjd<dtjvr tdtjdtjd<tjdtjd <tjd d |i|dS) NZOMPI_COMM_WORLD_LOCAL_RANK MASTER_PORT29500 MASTER_ADDRz/The environment variable MASTER_ADDR is not setZOMPI_COMM_WORLD_SIZE WORLD_SIZEZOMPI_COMM_WORLD_RANKRANKr.r) r9r:r;r'r<r=KeyErrorr>r?r@rrrr3Hs    r3cCsttjd}ttjd}tjd}tj}tj||td|d}|dur5t |tjd<ndtjvr;nt drEd tjd<nt t tjd<d tjvrW|tjd <t |tjd <t ||tjd <t |tjd <t j |ddS)aInitialize slurm distributed training environment. If argument ``port`` is not specified, then the master port will be system environment variable ``MASTER_PORT``. If ``MASTER_PORT`` is not in system environment variable, then a default port ``29500`` will be used. Args: backend (str): Backend of torch.distributed. port (int, optional): Master port. Defaults to None. Z SLURM_PROCIDZ SLURM_NTASKSZSLURM_NODELISTzscontrol show hostname z | head -n1NrAir?)r.rZproc_idZntasksZ node_listZnum_gpusaddrrrrr4Us*        r4cCs`tr(ddlm}|dur|rddlm}|}t|}t|}||fSd}d}||fS)zGet dist info of a specified group Args: group: The parallel group, default None, for the global group Returns: A tuple of the current rank and world_size of the group r)is_megatron_initializedN)mpur ) is_distZmodelscope.utils.megatron_utilsrLZ megatron_utilrMZget_data_parallel_groupr>get_rankget_world_size)grouprLrMrank world_sizerrr get_dist_info|s    rTcCsttjddS)Nr8r)r9r:r;getrrrrget_local_ranksrVcC tsdSts dStS)Nr)r> is_availableis_initializedrOrrrrrO rOcCrW)Nr )r>rXrYrPrrrrrPrZrPcCs8tsdSts dSt}|dkrdStdS)zj Helper function to synchronize (barrier) among all processes when using distributed training Nr )r>rXrYrPbarrier)rSrrr synchronizes r\cCstotSN)r>rXrYrrrrrNsrNcCstr t|dkSdS)NrT)rNr>rOrQrrr is_mastersr_csdtdtffdd }|S)Nfuncrcstfdd}|S)Ncstr |i|SdSr])r_)argsr6)r`rQrrwrappersz.master_only..decorate..wrapper) functoolswraps)r`rbr^)r`rdecorateszmaster_only..decorate)r)rQrerr^r master_onlys rfcCs6tstSd}trt}tt|d}|S)zRMake sure each rank has the same temporary directory on the distributed mode. Nr)rNtempfilemkdtempr_r>r[ broadcast)Ztmpdirrrr make_tmp_dirs rjcCst}tjdgdd}||kr&tjtt|tjdd}tj|jdd}t t ||||krAtj | fdtjdd}t t ||t |S)z Broadcasts the inputs to all ranks. Arguments: inputs : Any objects that can be serialized by pickle. src (int): Source rank. Returns: Each rank returns the same value as src. rr<deviceZdtyperl)r>rOr'tensor bytearraypickledumpsuint8shaper[rifullitemloadscpunumpytobytes)ZinputssrcrRZ shape_tensorZ inputs_tensorrrrris$    ricCsN|dur |dkr t|tj|t|tj|dStd|)Nrz0Random seed should be positive, current seed is )randomseednpr'Z manual_seedr<Zmanual_seed_allr5)r|rrrset_random_seeds   r~cCs tdkr tjddStjjS)zj Return a process group based on gloo backend, containing all the ranks The result is cached. r,gloorG)r> get_backendZ new_grouprQZWORLDrrrr_get_global_gloo_groups  rcCst|}|dvs Jt|dkrdnd}t|}t|dkr0td t t|d|tj |}t |j|d}|S)N)rr,rrwr<i@z;Rank {} trying to all-gather {:.2f} GB of data on device {}rk)r>rr'rlrprqlenloggerwarningformatrOZ ByteStorage from_bufferZ ByteTensorto)datarQr.rlbufferZstoragernrrr_serialize_to_tensors      rcstj|d}|dksJdtjgtjjd}fddt|D}tj|||ddd|D}t |}||krStj ||ftj jd}tj |fdd |fS) zz Returns: list[int]: size of the tensor, on each rank Tensor: padded tensor that has the max size r^r zBcomm.gather/all_gather must be called from ranks within the group!rmcs"g|] }tjdgtjjdqS)r rm)r'zerosint64rlr_rnrr 0z*_pad_to_largest_tensor..cSsg|]}t|qSr)r9ru)rsizerrrr5srdim) r>rPr'rnZnumelrrlrange all_gathermaxrrrcat)rnrQrSZ local_size size_listmax_sizepaddingrrr_pad_to_largest_tensor#s*     rcstdkr|gS|durt}t|dkr|gSt||t|\}t|fdd|D}tj||dg}t||D]\} d|}| t |qB|S)a; Run all_gather on arbitrary picklable data (not necessarily tensors). Args: data: any picklable object group: a torch process group. By default, will use a group which contains all ranks on gloo backend. Returns: list[data]: list of data gathered from each rank r Ncs"g|] }tjftjjdqS)rm)r'emptyrrrlrrrnrrrZrzall_gather..r^)rPrr>rrrrziprwrxryr rprv)rrQrZ tensor_listZ data_listrrrrrrCs$   rr*cCs(tdd|Ddh}t|dkS)Ncss|]}t|jVqdSr])rJrl)rprrrrisz$is_on_same_device..rwr )set parametersr)r*Z device_setrrris_on_same_devicehs r forward_fn. chunk_size chunk_dimc s&t|dks J|dttj}|t|kr(td|dt|d|dkr|dj}|D]}|j|krKtd|d|jq5|dj|dkrftd|djd ||dj|tfd d |D}tfd d t|D}tj |d S|S)Nrz" has to be a tuple/list of tensorszforward_chunk_fn expects z arguments, but only z input tensors are givenz/All input tenors have to be of the same shape: z, found shape zThe dimension to be chunked z( has to be a multiple of the chunk size c3s|] }|jdVqdS)rN)chunk)r input_tensor)r num_chunksrrrs  z,apply_chunking_to_forward..c3s|]}|VqdSr]r)rZinput_tensors_chunk)rrrrs  r) rinspect signaturerr5rstuplerr'r) rrrZ input_tensorsZnum_args_in_forward_chunk_fnZ tensor_shaperZinput_tensors_chunksZ output_chunksr)rrrrapply_chunking_to_forwardmsJ   rheadsn_heads head_sizealready_pruned_headscsvt||}t||}|D]tfdd|Dd|<q|dd}tt|| }||fS)Nc3s |] }|kr dndVqdS)r rNr)rhheadrrrsz3find_pruneable_heads_and_indices..rrr ) r'Zonesrsumview contiguouseqZarangerlong)rrrrmaskindexrrr find_pruneable_heads_and_indicess  rlayerrrcCs||jj}|j||}|jdur,|dkr#|j}n |j|}t|j}t |||<t j j |d|d|jdud|jj}d|j_ |j|d|j_ |jdursd|j_ |j|d|j_ |S)Nr r)biasFT)rweightrlZ index_selectdetachclonerlistrrr'nnLinearZ requires_gradZcopy_r)rrrWbnew_sizeZ new_layerrrrprune_linear_layers.    r)r,r])r)=rcrr:rpr{r rHrgtypingrrrrrxr}r'Ztorch.multiprocessingmultiprocessingr1 packagingrrr>rJrr9boolr#r+r7r2r3r4rTrVrOrPr\rNr_rfrjrir~ lru_cacherrrrrModulerZTensorrrrrrrZ LongTensorrrrrrs      '  !   %  /