o "j5A@sddlZddlZddlZddlZddlZddlZddlZddlZddl m m m m Zddlm Z m Z ddlmZddlmZddlmZmZmZmZddlmZddlmZmZmZmZddl m!Z!d d l"m#Z#d d l$m%Z%m&Z&m'Z'd d l(m)Z)ddl*m+Z+ddl,m-Z-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5ddl6m7Z7m8Z8ddl9m:Z:ddl;mZ>ddl?m@Z@ddlAmBZBddlCmDZDmEZEGdddZFdS)N)staticutils) _to_name_str)fleet)IrGraph_current_expected_placecorein_dynamic_mode)Metric) InputSpecOperatorVariable global_scope)_convert_float_to_bfloat16) get_logger)CollectionNamesfetchget_collection)Strategy)config_callbacks)Clusterget_default_cluster) Converter)get_cost_from_engine)DistributedContextget_default_distributed_context)DistributedDataLoader"DistributedDataLoaderFromGenerator)DistributedOperator)DistributedSaver) ProgramHelper) Parallelizer)Planner)get_all_process_groupsnew_process_groupc@seZdZdZ      duddZddZddZgfd d Zd d Zd dZ      duddZ dvddZ ddZ ddZ ddZdwddZddZdd Zdvd!d"Zdd#d#dd$dd#ddd#dddd%d&d&gfd'd(Z  #  $   %dxd)d*Z  #    %dyd+d,Z #    -   -  #  #  dzd.d/Z 0     # #   # d{d1d2Z        d|d3d4Zd}d5d6Zd7d8Z  #    -   -  #  d~d9d:Z       # #  dd;d<Zdd=d>Zd?d@ZdAdBZdCdDZ dEdFZ!dGdHZ"dIdJZ#dKdLZ$dMdNZ%dOdPZ&dQdRZ'dvdSdTZ(ddUdVZ)ddWdXZ*dYdZZ+d[d\Z,d]d^Z-d_d`Z.e/dadbZ0e/dcddZ1e/dedfZ2e/dgdhZ3e/didjZ4e/dkdlZ5e/dmdnZ6e/dodpZ7e/dqdrZ8e/dsdtZ9dS)Enginea An High-Level API for auto parallel, which could be used for distributed Training (engine.fit) and Inferenced (engine.predict). Static graph mode is supported natively, Dynamic graph mode is also supported under `@to_static `_ . Args: model (paddle.nn.Layer, optional): The model is an instance of paddle.nn.Layer. loss (Loss|Callable|None, optional): The loss can be a `paddle.nn.Layer` instance or any callable function taken the predicted values and ground truth values as input. It can be None when there is no loss. Default: None. optimizer (Optimizer|None, optional): The optimizer need to be set in training and should be None in eval and predict mode. Default: None. metrics (Metric|list[Metric]|None, optional): If metrics is set, all metrics will be calculated and output in train/eval mode. Default: None. cluster (Cluster|None, optional): The cluster represents the topology information about the used physical devices. Default: None. (Unused for now) strategy (Strategy|None, optional): The strategy is used to configure the parallelization and optimization behaviors. Default: None. Examples: .. code-block:: python >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto >>> from paddle.vision.datasets import MNIST >>> transform = T.Compose([ ... T.Transpose(), ... T.Normalize([127.5], [127.5]) >>> ]) >>> train_dataset = MNIST(mode='train', transform=transform) >>> valid_dataset = MNIST(mode='test', transform=transform) >>> model = paddle.vision.models.LeNet() >>> loss = paddle.nn.CrossEntropyLoss() >>> optimizer = paddle.optimizer.Adam( ... learning_rate=0.001, parameters=model.parameters()) >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) >>> engine = auto.Engine(model, loss, optimizer, metrics) >>> # fit >>> engine.fit(train_dataset, ... epochs=2, ... batch_size=64) >>> # evaluate >>> engine.evaluate(valid_dataset, ... batch_size=64) >>> # predict >>> engine.predict(valid_dataset, ... batch_size=64) >>> # save >>> engine.save("./my_model") >>> # load >>> engine.load("./my_model") Nc CsV|rt|tjjst|std||_|sdndd|D|_|r5t|tjjt fs5t|s5td||_ |rEt|tj j sEtdt ||_|pNg}t |D]}|rft|tsft|jjdqTt ||_|rxt|tsxtd|p|t|_|rt|tstd|pt|_ttj|_d|_|r||_nIt d rz"t d }t!|d } t"#| |_Wdn1swYWnt$y} z|j%d d|_WYd} ~ nd} ~ wwt|j|_t d r|j%d t&j'ddi|_(i|_)t*+|j|_,d|_-tj./|_0tj.1|_2t3|_4t56|_7t58|_9t:|_;i|_dddd|_?g|_@g|_Ag|_Bg|_Cg|_Dd|_Ed|_Fd|_Gd|_Hd|_I|jjJ|_Kd|_L|jjMjNrm|jjMjO|_Ln |jjPjNry|jjPjQ|_L|jjPjNr|jjPjRdkrt ddksJdd|_StjTUdditjTUddid|_VdS)NzI'model must be sub classes of `paddle.nn.Layer` or any callable function.cSsg|]}|jqS)name).0pr)r)o/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/distributed/auto_parallel/static/engine.py sz#Engine.__init__..zW'loss' must be sub classes of `paddle.nn.Layer` or any callable function or a Variable.z@'optimizer' must be object of class `paddle.optimizer.Optimizer`z is not sub class of MetriczP'cluster' must be the object or class `paddle.distributed.auto_parallel.Cluster`zN'strategy' must be object of class `paddle.distributed.auto_parallel.Strategy`ZPADDLE_AUTO_PARALLEL_CONFIGrzILoad json failed, please check json file, engine will run default config.ZPOD_NAMEz0Distribute training by paddle.distributed.launchT)Z is_collectiveFtrainevalpredictr1F1BZCUDA_MODULE_LOADINGZLAZYz._adjust_item_speccst|tjr&t||}|dur|||dS|||dSt|ttj j frOt ||}||durE||dS|||dSt|t j rc|t|gt||dStdt|j)NzYThe sample's dtype returned of dataset should be number, np.ndarray or Tensor, but got {})r5npZndarrayr Z from_numpyappendbatchr reagerZTensorZ from_tensornumbersNumbertyper:formatrD)itemr* batch_sizespecsrrrr)r-_infer_item_spec$s&      z3Engine._prepare_data_spec.._infer_item_speczReceive None input.inputlabel)r5r6ioZIterableDatasetnextiterZDatasetr:rrrDr@rBrGdatasetr enumeratestr_validate_spec) rwdatasplitr inputs_spec labels_specinputslabelssamplerirr*r)rr-_prepare_data_specsJ          zEngine._prepare_data_specc Cs ts|jr td|rKt|tsJdt|t|ts(Jdt|t|t|ks4Jdt||D]\}}|j|jkrJ|j |jq9|rt|ts[Jdt|t|tsiJdt|t|t|ksuJdt||D]\}}|j|jkr|j |jqz||fS)NzOnly support static graph mode.z$inputs should be list, but received z:the number of `inputs_spec` should be equal to `inputs`'s.z$labels should be list, but received z:the number of `labels_spec` should be equal to `labels`'s.) r rn ValueErrorr5listrrziprdesc set_shape) rwrrrrZ input_specrZ label_specrr)r)r-_prepare_data_tensorKsV       zEngine._prepare_data_tensorcCs|j|j}|j|j}|}gd}|jdjdkr4tt|D]}|jdj|vr3|j dddq"| g}t |jD]\}} | j|vrM| |q?g} g} t |D],}|j} | |j|j| | t|| | d} | | t| }||qV| D] } |jd| qtt|D] }||t|7<qt |D]}|j|} |j ||dq| d|j|j<|jjrDtsFd |jjd vsJ|jjd }d}|jjjd kr|d d}n |jjjd kr|d d}|dusJ|}|}|D]}|j|j vr|!|q | D]}|j} | |t|| | d} |jd| q| |"|dSdSdS) N)create_py_readerZcreate_double_buffer_readerreadrrF)sync)rrTtasks fleet_optr4stream)#rarjdist_main_programsrZ global_blockopsrrangerZ _remove_opZ_sync_with_cpprrreversedrZ _prepend_opZ copy_fromr r!Zadd_dist_op_for_programinsertpoprd main_programZ _pipeline_optr@use_new_executorrGrrrsZ get_programr*vars_clone_variableZ set_program)rw feed_list dist_contextdist_main_progdist_main_blockZrelated_reader_opsrZreader_op_indicesidxopZ read_ops_descZnew_reader_opsZ new_op_descZnew_opdist_oprZfwd_taskZfwd_progZ fwd_blockvarZop_descr)r)r-_prepare_readerls                    zEngine._prepare_readercCsi}|durKt|ttfr1t|dkr*t|dtr*|dD]\}}|||<q n!td|t|trD|D]\}}|||<q:ntd||durkt|ts^Jdt|j|D]\}}|||<qb|S)NrrzUnsupported data z'user_feeds must be a dict, but receive ) r5rtuplerdictitemsrrrD)rwrZ user_feedsmodeZfeedsr*valuer)r)r- _prepare_feeds,    zEngine._prepare_feedc s|durt|tsJdt|jggfdd}j|}|j}|dkr2|d|d|dkrL|d}t|D] \}}|dt||q>|dkrW|d|d|pZgD] } t| } t | q[d d t t j D} | psg}|d |fS) Nz)user_fetches must be a list, but receive csPg}|D]}|r t|}|vr|||q|dSN) _is_local_varrrindex) group_namevar_listZ group_indicesrvar_name fetch_indices fetch_namesrwr)r-_process_fetch_groups  z3Engine._prepare_fetch.._process_fetch_groupr3ryrzZmetrics_outputscSsg|]}|dqS)rr))r+rr)r)r-r.z)Engine._prepare_fetch..fetches) r5rrrDraserial_fetch_varsrrrrrrFETCHES) rwZ user_fetchesrrr fetch_varsrzrrZ usr_fetchrZuser_fetches_collectionr)rr-_prepare_fetchs8    zEngine._prepare_fetchcCsi}|dur ||d<|dur|d|d<|dur||d<d} |dkr|| } t| dks.J| D]} || |d<q0| d7} |j|} | jd} | r|jD]7}|| }g}|D] } ||| qV|r|j||}tt |D] \}}||| |<qr| d7} qLn|dkr|| }i}|D] } || |d | <q||d <| d7} t t j }i}|D]\}}||vr||} || ||p|<q||d <|S) Nepochrsteplrrr3ryrzzout%drr)rrarrErupdate accumulaterr@rBr*rrrr)rwoutsrrrrrrlogsZ group_idxZ loss_indicesrr metric_varsr}Zmetrics_indicesZ metric_outresultsrresZoutputs_indicesZlogs_outZcollect_fetchesZ logs_fetchr*rr)r)r-_prepare_loggersZ          zEngine._prepare_loggerTcCs@||||||||||d|j|<dS)NT)_build_plan _parallel _init_comm _initializerc)rwrinit_parametersr)r)r-_prepare_program2s    zEngine._prepare_programc CsZts|jretd|_|jdt|j|j|j |j |j |_ t j|j |Wdn1s7wY|j j|_|j j}|j j}|j j|_|j j|_|j j}|j j|_|j j}tn|j|d}|durrdSg}g}g|_|j }|j! }|j"st#$||yt jddd|j D|_dd|j D|_t%&|j|j}|dkr|jrt'|jtj(j)st*|jsJdt%&|j||j|_|dkr|s|jr|j D]}|+t%&|j,||jqWdn1swYWdn 1s wYn|dkr't'|jt-s Jd t%&|j|_t.}|j/sLt0t1t2|j3d|_4d d|jD|_d d|jD|_|j|jd } tj 5||j|d } |dkri|j dd}t%6|j|j|j7|t8|||j9|j| | |j:|j7|j; |j|<t8|||j9|j| | |j:|j7|j; |j<|<|j7j=|j|_=| |j>|<dS)NTz'Building model with 'to_static' method.cSg|]}|qSr)Z_create_feed_layerr+sr)r)r-r.jrz!Engine._build..cSrr)rrr)r)r-r.mrr3zothe type of `loss` of the Engine arguments should be sub classes of `paddle.nn.Layer` or any callable function.r1z>the type of `loss` of the Engine arguments should be Variable.cSg|]}t|qSr)r@Zset_data_parallelr+rr)r)r-r. cSrr)rrr)r)r-r.r)rr)rryrzZfor_test)?r rnr6Zdisable_staticrJrRr#r;r>rErerfprogram_helperrZ unique_nameguardZ build_programZconcrete_programrstartup_programZ input_varsrgZ label_varsrhZ output_varsZ loss_varsrirZ enable_staticragetr^cloner`rkr program_guardr@rBr5r7r8r9rZcomputer rZhas_annotationr'rrr[Z data_parallelflattenZset_recompute_segmentsrGrrArFrKrTZgradient_scalerU) rwrZserial_main_progZserial_startup_progrrzrr}Z default_ctx feed_varsrr)r)r-r?s                    z Engine._buildcCs|jjstd|dksJ|||||j|_|j|_ddl m }||j |||j |j ||jd|_|j|jjrK|j|j |_dSdS)Nz Please set `tuning.enable=True`.r1r)OptimizationTuner)rrank)rorqrrr_dp_world_sizes dp_world_size _dp_ranksdp_rankZtuner.optimization_tunerrrarerfrZZ_optimization_tunerZtuneZrun_after_tuningZget_best_configrG)rwrrrrr)r)r-_optimization_tunings0     zEngine._optimization_tuningc Cs|jdur ||_n |jjdkr||t||j||j|<|j||j|jd}|j|jd}|j|j }g}||D]}|j |j vrU| |j |j qDg|_g|_|D]}t|j||j|\}} |j ||j | q^dS)Nsemirr)rmrG auto_mode_init_dist_contextr%rarbZplanserial_feed_varsserial_main_programrr*rrrrr@Zget_input_split_inforZ) rwr inputs_var labels_varblockrrZfeed_varrrr)r)r-rs.      z Engine._planFcCsBt||j|j|j|}|s||j|jdS||jdSr)r$rbZ completerraZparallelrZr=Z parallel_all)rwrZ all_ranksZ parallelizerr)r)r-rs zEngine._parallelc Cs|j|}|j}|j}|j|}|j}|j}t|jD]2\}} t| jD](\} } ||j| } | j| jksAJd|| j|| j|| } | | | q$qdS)Nz2'{}' mode op '{}' is different with '{}' op '{}'. ) raZ_original_serial_main_programrmblocksrrrrZget_op_dist_attr_for_programZset_op_dist_attr_for_program)rwrrZorigin_main_progZref_modeZref_dist_contextZref_origin_main_progZ ref_blocksZibrZioprZref_opZref_op_dist_attrr)r)r-r s&   zEngine._init_dist_contextcCsH|jdkr t}|jjdkrt||jdS|D]}|qdSdS)NrZ full_random)r[r&rGrr@Zinitialize_pg_in_full_moderZZ instantiate)rwZall_process_groups process_groupr)r)r-rs    zEngine._init_commc Csjt|_t|jtjjrtjtjj|_|j j r@t |j j |j dt j |j j |j dt |j j |j d|j|}|j|j}|jr|j||j||jrt|jdkr|jD]Z}||jr||jj}t|j}t|j}|r|rqh|t j!j"j#kr|$t%|&|jqh|t j!j"j'kr|$t (|&|jqh|$|&|jqh|j)durtj*+|j|_)g} |j,|j} | -D]} t| j}|r|rq| .| q| r| /| } |j)0| t1|drt1|dr|2||j3|j4|j5|j j6r3|j78d|j,|j} |j)0| dSdS)Nr _state_dict _dist_attrz(NOTE: parameters will be re-initialized.)9 _get_device_placer5r6ruZ CUDAPlacerYZ ParallelEnvZdev_idrGseedrrrandomrarrZrnrrSr;rbuffersrZhas_varr*rdtyperZfind_varZ get_tensorZ_is_initializedrZVarDescZVarTypeZBF16setrnumpyZFP16Zfloat16rXrZExecutordist_startup_programsZ list_varsrZ_prunerunhasattr_set_state_dict_strictrrZreinitrJrR) rwrrrdist_main_programbufferZ dest_typeZ scope_varZ buffer_tensorZ uninitializeddist_startup_progrZprune_startup_progr)r)r-r,s~            zEngine._initializer rc Csd|_|j|js||||\|_|_||jn||jtr?| |}|j |d||| d}|dur= end_step, the nvprof will not apply. Returns: None Examples: .. code-block:: python >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto >>> from paddle.vision.datasets import MNIST >>> transform = T.Compose([ ... T.Transpose(), ... T.Normalize([127.5], [127.5]) >>> ]) >>> train_dataset = MNIST(mode='train', transform=transform) >>> model = paddle.vision.models.LeNet() >>> loss = paddle.nn.CrossEntropyLoss() >>> optimizer = paddle.optimizer.Adam( ... learning_rate=0.001, parameters=model.parameters()) >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) >>> engine = auto.Engine(model, loss, optimizer, metrics) >>> engine.fit(train_dataset, ... epochs=2, ... batch_size=64) r1F) return_listrepochs collate_fnNF)rcapacityiterablerr&steps_per_epochr'rr) enginerr&stepslog_freq save_freqsave_dirverboserzZacc_stepr)Ziter_idstartendfeed fetch_listZuse_program_cache return_numpycSsi|] \}}d||qS)Zval_r))r+r*valr)r)r- ?szEngine.fit..)2rjrcrrerfr _switch_moder@r_validate_batch_size_prepare_dataloaderr"_prepare_dataloader_from_generator_stepsrGrrrqrprr _metrics_nameon_beginrZon_epoch_beginr_validate_batchr6ZprofilerrZ _nvprof_rangeon_batch_beginrXrr use_cacher8Zget_lrr?r on_batch_endr EOFException_resetevaluaterr_reset_metricsZ on_epoch_endon_endrt) rwZ train_dataZtrain_sample_splitrr&r+r/r1r0 valid_datavalid_sample_splitZ valid_freqZ valid_stepsr' callbacksr2Z nvprof_rangelocal_batch_sizeZtrain_dataloadermicro_batch_sizerrcbksrrrrbatches micro_batchrrZval_logsr)r)r-fitvsV                    z Engine.fitc  Csd|_|j|js||||\|_|_||jn||jtr>| |} |j |d| |d} |dur;t | n|} n| |} |j |dd| ||d} | j } | } |jjjr]| |j} |jd|jd\} }t||| |||d}| }|d||d i}t| D]^\}}tr||}nig}z|D]}|d|||jj|j|| |jj|jjd }qWn tjyYn%w| r|| krts| n|!|d|d| ||j}|"d||q|#d||$|S) a Evaluate the loss and metrics of the model on evaluation data. Args: valid_data (Dataset): An instance of paddle paddle.io.Dataset. Default: None. valid_sample_split (int, optional): Each sample of the eval dataset is assumed to be a (input, label) pair by default and has two items. If each sample has more than two items, valid_sample_split specifies how to split these items into input and label. The items before it are input and the left are label. Default: None. batch_size (int, optional): The batch size of valid_data. The user's data will be used directly without batching if set to None. Default: 1. steps (int, optional): It is the total number of steps (batches of samples) to draw before stopping evaluation. If None, evaluation will run until the `valid_data` dataset is exhausted. The evaluation will start from the beginning of the dataset in each run. Default: None. collate_fn(callable, optional): function to generate mini-batch data by merging the sample list, None for only stack each fields of sample in axis 0. Default None. callbacks (Callback|None, optional): A list of `Callback` instances to apply during evaluating. Default: None. (Unused for now) Returns: None Examples: .. code-block:: python >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto >>> from paddle.vision.datasets import MNIST >>> transform = T.Compose([ ... T.Transpose(), ... T.Normalize([127.5], [127.5]) >>> ]) >>> valid_dataset = MNIST(mode='test', transform=transform) >>> model = paddle.vision.models.LeNet() >>> loss = paddle.nn.CrossEntropyLoss() >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) >>> engine = auto.Engine(model, loss, metrics=metrics) >>> engine.evaluate(valid_dataset, batch_size=64) r2Fr%rr'Nr(rr)r*rr+r'r,)r-rr/r2rz)r.rzr5)%rjrcrrerfrr;r@rr<r=rr>r?rGrrrqrprrr@rArrBrCrXrrrDr8rrFrGrrErJrI)rwrKrLrr.r/r'rMr2rNZvalid_dataloaderr+rOrrrPZ eval_stepsrrrrQrRrr)r)r-rHLs9            zEngine.evaluatec Csd|_|j|js||||\|_|_||jn||jtr>| |}|j |d||d} |dur;t | n|} n| |} |j |dd| ||d} | j } |jd|jd\} } g}t|||d}| }|dd |ii}t| D]i\}}tr||}nig}z|D]}|d|||jj|j|| |jj|jjd }qWn tjyYn0w| r|| krts| n||d|d| | |j}|d|||t |d !qt|"d||S) a4 Compute the output predictions on testing data. Args: test_data (Dataset): An instance of paddle paddle.io.Dataset. Default: None. test_sample_split (int, optional): Each sample of the test dataset is assumed to be a (input, label) pair by default and has two items. If each sample has more than two items, test_sample_split specifies how to split these items into input and label. The items before it are input and the left are label. Default: None. batch_size (int, optional): The batch size of test_data. The user's data will be used directly without batching if set to None. Default: 1. steps (int, optional): It is the total number of steps (batches of samples) to draw before stopping predict. If None, predict will run until the `test_data` dataset is exhausted. The predict will start from the beginning of the dataset in each run. Default: None. collate_fn(callable, optional): function to generate mini-batch data by merging the sample list, None for only stack each fields of sample in axis 0. Default None. callbacks (Callback|None, optional): A list of `Callback` instances to apply during testing. Default: None. (Unused for now) Returns: None Examples: .. code-block:: python >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto >>> from paddle.vision.datasets import MNIST >>> transform = T.Compose([ ... T.Transpose(), ... T.Normalize([127.5], [127.5]) >>> ]) >>> valid_dataset = MNIST(mode='test', transform=transform) >>> model = paddle.vision.models.LeNet() >>> engine = auto.Engine(model) >>> engine.predict(valid_dataset, batch_size=64) r3FrTNr(rUr,)r-r2r.r5r)#rjrcrrerfrr;r@rr<r=rr>r?rrrArrBrCrXrrrGrDr8rrFrGrrErrvaluesrJ)rwZ test_dataZtest_sample_splitrr.r'rMr2rNZtest_dataloaderr+rOrrrrPZ test_stepsrrrrQrRrr)r)r-r3sz5         zEngine.predictrcCs|dur |||j|js!||| |\|_|_||jn||j||}|j |d|||||||| | | | |d}|S)NF) r%rshuffle drop_lastr' num_workersuse_buffer_readeruse_shared_memorytimeoutworker_init_fnr&r+places) to_modercrjrrerfrr;r<r=)rwrrrWrXr'rYrZr[r\r]r&r+ sample_splitrr^ dataloaderr)r)r-raPs4     zEngine.dataloaderr(c  Cs|| dur || |j|js!||| |\|_|_||jn||j||} |j ||||d||| || | d }|S)NF) rr)use_double_bufferr*r%use_multiprocessrXrr&r+r') r_rcrjrrerfrr;r<r>)rwrr)rbr*rcrXrr&r+r'r`rrOrar)r)r-dataloader_from_generators.      z Engine.dataloader_from_generatorc CsF|dur |||jstd|j|jrdS||}||}||}||}||_||_|s6|rXd|_| ||||\}}|jdurMt |_|jdurWt |_n&|s\|rtd|_ |jdurit |_|jdurst |_n |jrz|js~Jd|||_|_|||_|_|j|js||j|dS||jdS)Nz7Please set mode to be prepared with `prepare(mode=...)`Tz;Please call the dataloader(...) before calling prepare(...))r_rjrrcr_validate_varsr^r`rkrrr]r_rlrerfrgrhrr;) rwrrrrrrrrr)r)r-preparesT               zEngine.preparec Cs|dur ||||||j}|||j\}}|jr'|j|js'||j|j_|jj |j |||j j |j j d}||ddd|||j} | S)Nr5)r_rrjrrlrdrrvrXrrrGrDr8r) rwrr6r7rZ feed_dictrrrrr)r)r-rs,  z Engine.runc Cs|j|j}|j|j}|j|j}|}|jd}|jd}g}||D]'}|j|jvr8| |j|jq&| ||j } | j |j | | q&|S)Nrr)rarjrrZrrr r*rrr persistablerset_original_id original_id) rwrrr"rr r rrcopy_varr)r)r- get_feed_lists        zEngine.get_feed_listcCsH|j|j}|j|j}|j|j}|}|jd}|jd}g}||D]'}|j|jvr8| |j|jq&| ||j }|j |j | |q&t||Ft|fid|d|d|d|d|d|d |d |d |d | d | d| d| d| d|jjd|jd|j}Wd|S1swY|S)Nrrrr^r%rrWrXr'rYrZr[r\r]r&r+ split_datadata_parallel_world_sizedata_parallel_rank)rarjrrZrrr r*rrrrgrrhrirrrrGrlrr)rwrr%rrWrXr'rYrZr[r\r]r&r+r^rrr"rr r rrrjrar)r)r-r=sn              zEngine._prepare_dataloaderc Csh|j|j} | j|j} | j|j}| }| jd}| jd}g}||D]'}|j|jvr8| |j|jq&| ||j }|j |j | |q&tj}t| |Ltdid|d|d|d|d|d|d |d |d |d |d | d| d| d|jjd|jd|jd|jjjsdn|j}Wdn1swY|||S)Nrrrrr)rbr*r%rcrXr^rr&r+r'rlrmrnZ acc_stepsrr))rarjrrZrrr r*rrrrgrrhrir6rZ cuda_placesrr rGrlrrrrrqrpr)rwrr)rbr*r%rcrXrr&r+r'rrr"rr r rrrjr^rar)r)r-r>Rsp               z)Engine._prepare_dataloader_from_generatorcCs0d|_||||\|_|_||j||dS)Nr1)rjrrerfr)rwZ tune_dataZtune_sample_splitrr)r)r-_tunes  z Engine._tunecCs|durdStr>tt|jdksJdtt|j||jddks7Jdt|t|jd||jdS||jdksNJd||j||jS)NrziDistributedBatchSampler only support one data parallel group, but got [{}] different data parallel groupsrz6batch_size [{}] is not divisible by dp_world_size [{}]z;Requires batch_size:[{}] to be divisible by acc_steps:[{}].)r@rrrrrrrp)rwrr)r)r-r<s(  zEngine._validate_batch_sizecs|durdgS|jjjs|jdkr|Sg}g}|dD]\}}|||tt||jdqg}t |jD]fdd|D}|t t ||q<|S)Nrrcsg|]}|qSr)r))r+Z split_batchrr)r-r.sz*Engine._validate_batch..) rGrrrqrprrrrarrayrrr)rwrZ feed_namesZ split_batchesZ feed_nameZcur_feedZbachesrRr)rpr-rBs  zEngine._validate_batchcCst|}|durVt|D]H\}}t|tstd|jdur*td|d|d|jdkrUt |j }|d|jdksIJd |j d|j|d|j<||_ q |pYgS)Nz9'spec' must be object of class `paddle.static.InputSpec`.zRequires Input[z(].name != None, but receive `None` with .rrz7Requires batch_size[{}] to be divisible by k_steps[{}].) r@rBrr5r r:r*rrprrr)rwrrrrr)r)r-rs,      zEngine._validate_speccCs>t|}|durt|D] \}}t|tstdq |pgS)Nz'var' must be a `Variable`.)r@rBrr5r r:)rwrrrr)r)r-res  zEngine._validate_varscCst|}||jjvSr)rrrr)rwrrr)r)r-rszEngine._is_local_varcCs|jD]}|qdSr)rEreset)rwr}r)r)r-rIs  zEngine._reset_metricscCs4|jrdgng}|jD] }|t|q |S)Nry)r>rEextendr@rBr*)rwZ metrics_namemr)r)r-r@s zEngine._metrics_namecCs&||jvs J|d||dS)Nz3 model is not ready, please call `prepare()` first.)rar_rwrr)r)r-r;s zEngine._switch_modecCs"|dvs Jd|d||_dS)Nr0zmode z. should be one of ['train', 'eval', 'predict'])rjrvr)r)r-r_s   zEngine.to_modec Cs|j|}|j|j}t||}t|||}|j|d}|D]3\} } t | } | |vr1q#| j || j krV|j d| t|| j t| j || | j || <q#||dS)N)strictz!cast {}'s dtype from '{}' to '{}')rarrZr@Z get_dist_attrrconvert state_dictrrrqrrJrRrrZastypeZset_state_dict) rwrrwryZ dist_attrrprogramZ cur_dist_attr converterr*paramZ param_arrayr)r)r-rs(       zEngine._set_state_dictc Cs&|r%|j|jvs J|j|j}|j}|j|j}|jj||||dd Sd|jvs,J|jd}|jd}|jd}|j|j}|j j j r|j j j rddl m}|jd|jd|j j tt|jd d } |t|j} | D]} | | qy| }|jj||||j|d d S) a% Saves the model, parameters, optimizer state to path. If `training` is set to False, only inference model will be saved. Args: path (str): The file prefix to save model. The format is 'dirname/file_prefix' or 'file_prefix'. if empty str. A exception will be raised. training (bool, optional): Whether to save for training. If not, save for inference only. If `training` is set to True, the optimizer state will be saved. Otherwise, only the model and parameters are saved. This function will silently overwrite existing file at the target location. Default: True. Returns: None Examples: .. code-block:: python >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto >>> from paddle.vision.datasets import MNIST >>> transform = T.Compose([ ... T.Transpose(), ... T.Normalize([127.5], [127.5]) >>> ]) >>> train_dataset = MNIST(mode='train', transform=transform) >>> model = paddle.vision.models.LeNet() >>> loss = paddle.nn.CrossEntropyLoss() >>> optimizer = paddle.optimizer.Adam( ... learning_rate=0.001, parameters=model.parameters()) >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) >>> engine = auto.Engine(model, loss, optimizer, metrics) >>> engine.fit(train_dataset, ... epochs=1, ... batch_size=64) >>> engine.save("./my_model") )serial_programr rr3rrr)QuantWeightPasszexport quantized model.zconvert config Tr)rzN)rjrar rrZr\saver rrGZqatrqZ onnx_formatZpaddle.static.quantizationr~rJrRto_dictrrZGraphrrrZall_sub_graphsapplyZ to_programZsave_inference_modelrX) rwr~Ztrainingrr}rrrr~Z test_graphZquant_weight_passZ sub_graphr)r)r-rsH.             z Engine.savecCs(||_|j||\|_|_|j|jfS)a) Load the stored model, parameters and optimizer states. Args: path (str): The prefix of files storing the model states and optimizer states. strict (bool, optional): Whether to skip the loading of mismatch parameter or raise an error when mismatch happens (not found the parameter in file storing model states of or receives a mismatch shape). Default: True. load_optimizer (bool, optional): If True, the stored optimizer states is restored. Otherwise, the optimizer states is initialized from scratch. Default: True. Returns: None Examples: .. code-block:: python >>> import paddle >>> import paddle.vision.transforms as T >>> from paddle.distributed.fleet import auto >>> from paddle.vision.datasets import MNIST >>> transform = T.Compose([ ... T.Transpose(), ... T.Normalize([127.5], [127.5]) >>> ]) >>> train_dataset = MNIST(mode='train', transform=transform) >>> model = paddle.vision.models.LeNet() >>> loss = paddle.nn.CrossEntropyLoss() >>> optimizer = paddle.optimizer.Adam( ... learning_rate=0.001, parameters=model.parameters()) >>> metrics = paddle.metric.Accuracy(topk=(1, 2)) >>> engine = auto.Engine(model, loss, optimizer, metrics) >>> engine.fit(train_dataset, ... epochs=1, ... batch_size=64) >>> engine.save("./my_model") >>> engine.load("./my_model") )rr\rPrr)rwr~rwZload_optimizerr)r)r-rPhs / z Engine.loadcCs|jjdkr|jddS|dur|n|j}|dusJd||jvr1td|t|j | ||durV|j|sV| ||_ | ||_ ||||n(ts\|jr`td|jdtj}|jru|js~|j|s~tdt||\}}|j|fS)a Get and Print cost, including memory of every rank, max memory among all ranks, and the global cost of one step based on communication cost(computation cost is 0 by default). In the future, the flops information of every rank and global cost including computation cost will be added. Args: inputs_spec(InputSpec): The specification of inputs. Default: None. labels_spec(InputSpec): The specification of labels. Default: None. mode (str): The engine mode must be in ["train", "predict", "eval"]. Default: None. Returns: Return the global execution time (ms) and max memory (B). fullzMThe cost will be calcudated in the search process when the auto mode is full.NzPlease set mode.z'The mode {} is not in accepted modes {}z\Please call `prepare()` or `fit()` or `evaluate()` or `predict()` before calling `cost()`.zThe program whose cost to be estimated must be static default program. Otherwise, please call `prepare()`before calling `cost()`.)rGrrJrRrjrcrrrkeysr_rrerfrrr rnr6rr]rrrtime)rwrrrrzZ global_costZ max_memoryr)r)r-costsL          z Engine.costcC|j|j|jSr)rarrZrvr)r)r-get_dist_main_programzEngine.get_dist_main_programcCrr)rarrZrvr)r)r-get_dist_startup_programrzEngine.get_dist_startup_programcC |j|jSr)rar rvr)r)r-get_serial_main_program zEngine.get_serial_main_programcCrr)raserial_startup_programrvr)r)r-get_serial_startup_programrz!Engine.get_serial_startup_programcC|j|j}|j|jSr)rarjrrZrwrr)r)r-r  zEngine.main_programcCrr)rarjrrZrr)r)r-rrzEngine.startup_programcCs |j|jSr)rarjrwr)r)r-rs zEngine.dist_contextcC|j|j}|jSr)rarjr rr)r)r-r  zEngine.serial_main_programcCrr)rarjrrr)r)r-rrzEngine.serial_startup_programcCrr)rarjr rr)r)r-rrzEngine.feed_varscCrr)rarjrrr)r)r-rrzEngine.fetch_varscCs|j|j}|jr |jS|jSr)rarjZ_serial_optimizerrArr)r)r-r? s zEngine.optimizercC|jSr)rgrr)r)r-rz Engine.inputscCrr)rhrr)r)r-rrz Engine.labels)NNNNNN)T)F)NrNr#NNr)NrNNNr)rFTNrTTrNrNrNN) r(TTFTrrNNrN)NNNNNNNT)NNNN) TrFTNrTTrNrNN) NTTFFTrrNN)Nr)TT)NNN):rD __module__ __qualname____doc__rrrrrrrrrrrrr rrrSrHr3rardrfrrkr=r>ror<rBrrerrIr@r;r_rrrPrrrrrpropertyrrrr rrrr?rrr)r)r)r-r(;sH>  G !O* :   M Z  } 3 , 8 A <  U 5?         r()GrVrOrHrrLrrrr6Z-paddle.distributed.auto_parallel.static.utilsrYZ auto_parallelrrr@Zpaddle.base.executorrZpaddle.distributedrZpaddle.frameworkrrrrr Z paddle.metricr Z paddle.staticr r r rZpaddle.static.amp.fp16_utilsrZutils.log_utilsrZ interfacerrrr|rrMrr{rrr{rZcost.estimate_costrrrrZ dist_loaderrr rr!Z dist_saverr"helperr#Zparallelizer_v2r$Z planner_v2r%rr&r'r(r)r)r)r-s@