o "j^@s ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl m Z ddlmZddlmZmZmZddlmZmZmZdd lmZgZGd d d Zd d ZddZddZddZddZ ddZ!ddZ"ddZ#GdddZ$d$d"d#Z%dS)%N)core) get_device)_get_trainers_numget_cluster_and_pod)use_paddlecloud)get_cluster_from_args) DeviceModeblock_windows_and_macos check_backend)_prepare_trainer_env_print_argumentsget_host_name_ip) set_flagsc@seZdZddZdS)ParallelEnvArgscCs(d|_d|_d|_d|_d|_d|_dS)NT)cluster_node_ipsnode_ipr started_port print_configselected_devices)selfrY/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/spawn.py__init__/s  zParallelEnvArgs.__init__N)__name__ __module__ __qualname__rrrrrr.s rcCsLgd}gd}|D]}||vr#||vrtd|tq td|q dS)N) start_methodipsgpusxpusrbackend)rrrrrzThe config option (%s) of `paddle.distributed.spawn` is deprecated. Please use the latest config options stated in the `spawn` API documentation.zFThe config option (%s) of `paddle.distributed.spawn` is not supported.)warningswarnDeprecationWarning ValueError)optionsZsupported_optionsZdeprecated_optionskeyrrr_options_valid_checkGs$r'cCsft}d|vr tSd|vrtSd|vrtS|tvr+t|ddSt d|d)Ngpuxpucpu:rI`paddle.distributed.spawn` does not support parallel training on device `` now.) rrget_cuda_device_countget_xpu_device_countmultiprocessing cpu_countget_available_custom_deviceget_custom_device_countsplit RuntimeErrorZdevicerrr_get_default_nprocshs  r7cCsJt}d|vr dSd|vrdSd|vrdS|tvrdStd|d ) Nr(ncclr)bkclr*glooxcclr,r-)rrr2r5r6rrr_get_default_backendxs  r<cCs>d}dd|dD}t|dkr|d}|St\}}|S)NcSsg|]}|qSr)strip.0xrrr sz _get_node_ip..,r)r4lenr )rrZnode_ips_rrr _get_node_ips  rFc sd|vs |ddkrt|d<t|dt|dg}t}|dd|_|jdur;|dd|_|jdur;d|_|ddkr|dd|_|jdurT|dd|_td d}|dusb|d krnd d t t Dn| d |jdurt |krtdt |fd fdd t d|D|_nP|j d }t ||krtdt ||f|D]}|vrtd|d qn$|ddkrU|dd|_|jdur|dd|_tdd}|dus|d krdd t t Dn| d |jdur't |krtdt |fd fdd t d|D|_n|j d }t ||kr>tdt ||f|D]}|vrRtd|d q@n|ddkrtdd|_d|_|j|_|dddusxJdt |j d d ksJd!td ksJd"nX|dd#krd|_t d}td$|d%d}|dus|d krd&d t t |Dn| d t |krtd't ||fd fd(d t d|D|_|d)d|_|jdurt|j|_|d*d|_|dd|_|jdurt|_|ddkr-tt d|}t|t j!|\} } nt"|\} } | j#D]} |$t%| | |dq6|d+d,|_&|j&rTt'||S)-Nr autorrz 127.0.0.1r8rrZCUDA_VISIBLE_DEVICEScSg|]}t|qSrstrr>rrrrAz,_get_subprocess_env_list..rBzthe number of visible devices(%d) is less than the number of spawn processes(%d), please ensure that the correct `nprocs` argument is passed or the environment variable `CUDA_VISIBLE_DEVICES` is correctly configured.cg|]}t|qSrrJr>Zenv_devices_listrrrArzThe number of selected devices(%s) is not equal to the number of spawn processes(%d), please ensure that the correct `nprocs` and `gpus` arguments are passed.zCThe selected gpu card {} cannot found in CUDA_VISIBLE_DEVICES ({}).r9rZXPU_VISIBLE_DEVICEScSrIrrJr>rrrrArLzthe number of visible devices(%d) is less than the number of spawn processes(%d), please ensure that the correct `nprocs` argument is passed or the environment variable `XPU_VISIBLE_DEVICES` is correctly configured.crMrrJr>rNrrrArOzThe number of selected devices(%s) is not equal to the number of spawn processes(%d), please ensure that the correct `nprocs` and `xpus` arguments are passed.zBThe selected xpu card {} cannot found in XPU_VISIBLE_DEVICES ({}).r:zYour model will be trained under CPUONLY mode by using GLOO,because CPUPlace is specified manually or your installed PaddlePaddle only support CPU Device.Trz.CPUONLY spawn doesn't support use paddle cloudrCzJCPUONLY spawn only support single trainer, that is len(ips)=1, but got %s.z+CPUONLY spawn doesn't support multi-trainerr;ZFLAGS_selected_scSrIrrJr>rrrrAszthe number of visible devices(%d) is less than the number of spawn processes(%d), please ensure that the correct `nprocs` argument is passed or the environment variable `FLAGS_selected_%ss` is correctly configured.crMrrJr>rNrrrA*rOrrrF)(r<r r rgetrrosgetenvrangerr.r4rDr5joinr$formatr/r!r"Zpaddle_cpuonlyrrZget_all_custom_device_typer3rrFrrlistrrZCPUrZtrainersappendr rr ) nprocsr%Zprocesses_env_listargsZ env_devicesZselected_device_listZcard_idZcustom_device_nameZdevices_per_procZclusterZpodZtrainerrrNr_get_subprocess_env_lists                                     r[cCs tjddtjdddS)NZ http_proxyZ https_proxy)rRenvironpoprrrr_remove_risky_envOsr^cCsR|dkr td|din|dkrtd|din |D] }||tj|<qdS)Nr8ZFLAGS_selected_gpusr9ZFLAGS_selected_xpus)rrRr\)env_dictr var_namerrr_set_trainer_envVs racCspztt||||}||WdStyYdSty7ddl}||tdYdSw)NrrC) r^raputKeyboardInterrupt Exception traceback format_excsysexit)funcrZ error_queue return_queuer_r resultrerrr _func_wrapperns   rmc@s&eZdZddZdddZddZdS) MultiprocessContextcCs*||_||_||_ddt|D|_dS)NcSsi|]\}}|j|qSr)sentinel)r?indexprocessrrr s z0MultiprocessContext.__init__..) error_queues return_queues processes enumerate sentinels)rrursrtrrrrs  zMultiprocessContext.__init__NcCst|jdkr dStjj|j|d}d}|D]}|j|}|j|}||j dkr2|}nq|dur>t|jdkS|jD]}| rK| |qA| |dS)NrT)timeout) rDrwr0 connectionwaitkeysr]rurUexitcodeis_alive terminate_throw_exception)rrxready error_indexrorprqrrrrUs*      zMultiprocessContext.joincCsv|j|r(|j|j}|dkr t| j}td||ftd||f|j|}d|}||7}t|)Nrz%Process %d terminated with signal %s.z(Process %d terminated with exit code %d.z ---------------------------------------------- Process %d terminated with the following error: ---------------------------------------------- ) rsemptyrur|signalSignalsnamerdrQ)rrr|rZoriginal_tracemsgrrrrs* z$MultiprocessContext._throw_exception)N)rrrrrUrrrrrrns rnrTFc Kst||dkr t}t||}|dd}|durd}t|}g} g} g} t|D]1} |} |}|jt ||| ||| |dfd}||_ | | | | || |q+t | | | }|sg|S|sp |rk|S)a Start multiple processes with ``spawn`` method for parallel training. .. note:: ``spawn`` now only supports GPU or XPU collective mode. The collective mode of GPU and XPU cannot be started at the same time, so the option `gpus` and `xpus` cannot be configured at the same time. Args: func (function): The target function is called by spawned process. This function need to be able to pickled, so it must be defined at the top level of a module. args (list|tuple, optional): Arguments passed to ``func``. nprocs (int, optional): Number of processed to start. Default: -1. when nprocs is -1, the available device will be obtained from the environment variable when the model is executed: If use GPU, the currently available device ID is obtained from the environment variable CUDA_VISIBLE_DEVICES; If use XPU, the currently available device ID is obtained from the environment variable XPU_VISIBLE_DEVICES. join (bool, optional): Perform a blocking join on all spawned processes. Default: True. daemon (bool, optional): The spawned processes' daemon flag. Default: False. **options(dict, optional): Other initial parallel execution environment configuration options. The following options are currently supported: (1) start_method (string): the way to start a process. The start method can be ``spawn`` , ``fork`` , ``forkserver`` . Because the CUDA runtime does not support the ``fork`` start method, when use CUDA in subprocesses, we should start process by ``spawn`` or ``forkserver`` method. Default: "spawn" ; (2) gpus (string): The training process will run on the selected gpus, such as "0,1,2,3". Default: None; (3) xpus (string): The training process will run on the selected xpus, such as "0,1,2,3". Default: None; (5) ips (string): Paddle cluster nodes ips, such as "192.168.0.16,192.168.0.17". Default: "127.0.0.1" . Returns: ``MultiprocessContext`` object, it hold the spawned processes. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:DISTRIBUTED) >>> import paddle >>> import paddle.nn as nn >>> import paddle.optimizer as opt >>> import paddle.distributed as dist >>> class LinearNet(nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear1 = nn.Linear(10, 10) ... self._linear2 = nn.Linear(10, 1) ... def forward(self, x): ... return self._linear2(self._linear1(x)) >>> def train(print_result=False): ... # 1. initialize parallel environment ... group = dist.init_parallel_env() ... process_group = group.process_group if group else None ... # 2. create data parallel layer & optimizer ... layer = LinearNet() ... dp_layer = paddle.DataParallel(layer, group = process_group) ... loss_fn = nn.MSELoss() ... adam = opt.Adam( ... learning_rate=0.001, parameters=dp_layer.parameters()) ... # 3. run layer ... inputs = paddle.randn([10, 10], 'float32') ... outputs = dp_layer(inputs) ... labels = paddle.randn([10, 1], 'float32') ... loss = loss_fn(outputs, labels) ... if print_result is True: ... print("loss:", loss.numpy()) ... loss.backward() ... adam.step() ... adam.clear_grad() >>> # Usage 1: only pass function. >>> # If your training method no need any argument, and >>> # use all visible devices for parallel training. >>> if __name__ == '__main__': ... dist.spawn(train) >>> # Usage 2: pass function and arguments. >>> # If your training method need some arguments, and >>> # use all visible devices for parallel training. >>> if __name__ == '__main__': ... dist.spawn(train, args=(True,)) >>> # Usage 3: pass function, arguments and nprocs. >>> # If your training method need some arguments, and >>> # only use part of visible devices for parallel training. >>> # If your machine hold 8 cards {0,1,2,3,4,5,6,7}, >>> # this case will use cards {0,1}; If you set >>> # CUDA_VISIBLE_DEVICES=4,5,6,7, this case will use >>> # cards {4,5} >>> if __name__ == '__main__': ... dist.spawn(train, args=(True,), nprocs=2) >>> # Usage 4: pass function, arguments, nprocs and gpus. >>> # If your training method need some arguments, and >>> # only use part of visible devices for parallel training, >>> # but you can't set your machine's environment variable >>> # CUDA_VISIBLE_DEVICES, such as it is None or all cards >>> # {0,1,2,3,4,5,6,7}, you can pass `gpus` to >>> # select the GPU cards you want to use. For example, >>> # this case will use cards {4,5} if your machine hold 8 cards. >>> if __name__ == '__main__': ... dist.spawn(train, args=(True,), nprocs=2, gpus='4,5') rrNspawnr )targetrZ)r'r7r[rQr0Z get_contextrT SimpleQueueProcessrmdaemonstartrXrnrU)rirZrYrUrr%Zprocs_env_listrmprsrtruirjrkrqcontextrrrrsHr         r)rrTF)&r0rRrrgr!Z paddle.baserZ paddle.devicerZpaddle.distributed.cloud_utilsrrZ$paddle.distributed.fleet.cloud_utilsrZpaddle.distributed.fleet.launchrZ%paddle.distributed.fleet.launch_utilsrr r Z%paddle.distributed.utils.launch_utilsr r r Zpaddle.frameworkr__all__rr'r7r<rFr[r^rarmrnrrrrrs4     ! >B