o #j@sddlZddlZddlmZddlZddlmZddlmZddl m Z m Z m Z ddl mZddlmZddlmZmZmZddlmZmZmZmZdd lmZgZed/d d Zed/d d Zed/ddZda ddZ!ddZ"ddZ#GdddZ$Gddde$Z%ddZ&GdddZ'Gdd d e'Z(Gd!d"d"e'Z)da*d#d$Z+da,d%d&Z-Gd'd(d(e'Z.e j/d0d)d*Z0d+d,Z1d-d.Z2e'Z3e(Z4e)Z5e.Z6dS)1N)NotSupportedError)_C_ops)core framework unique_name)check_variable_and_dtype)DataType)Variable check_typedefault_main_program) LayerHelperin_dynamic_modein_dynamic_or_pir_mode in_pir_mode) templatedoccCstr t||Std it}t|dgddt|dtd|dur0t d |j dg}|j |j ||jdd }|jdd|id|id |id |S) a@ ${comment} Args: x(${x_type}): ${x_comment} max_norm(${max_norm_type}): ${max_norm_comment} name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. Returns: Tensor: out(${out_type}): ${out_comment} Examples: .. code-block:: python >>> import paddle >>> from paddle.nn import clip >>> input = paddle.to_tensor([[2.0, 2.0], [2.0, 2.0]], dtype='float32') >>> reward = clip.clip_by_norm(x=input, max_norm=1.0) >>> print(reward) Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True, [[0.50000000, 0.50000000], [0.50000000, 0.50000000]]) clip_by_normX)float16float32uint16max_normN.tmpF)typenamedtypeZ persistableOutrinputsattrsoutputs)r)r rrr localsrr floatrZgenerate_with_ignorable_keyjoinrZcreate_variablerr append_op)xrrhelperoutr(O/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddle/nn/clip.pyr%s*!   rcCsLtrt|Stdit}|j|jd}|jdd|iid|id|S)aQ ${comment} Args: x(${x_type}): ${x_comment} name(basestring|None): Name of the output. Returns: out(${out_type}): ${out_comment} Examples: .. code-block:: python >>> import paddle >>> import paddle.base as base >>> b = paddle.static.default_main_program().global_block() >>> var = b.create_var( ... name="X", dtype="float32", persistable=True, ... type=base.core.VarDesc.VarType.SELECTED_ROWS) >>> y = paddle.nn.clip.merge_selected_rows(var) merge_selected_rowsrrrrN)r*)rrr*r r!"create_variable_for_type_inferencerr$r%rr&r'r(r(r)r*bs r*cCsrtrt|St|dtd|jtjjj krt dt d it }|j |jd}|jdd|id|iid|S) a? Get tensor data from input with SelectedRows type, and outputs a Tensor. .. code-block:: text input x is SelectedRows: x.rows = [0, 5, 5, 4, 19] x.height = 20 x.value = [[1, 1] [2, 2] [2, 2] [3, 3] [6, 6]] Output is LoDTensor: out.shape = [5, 2] out.data = [[1, 1], [2, 2], [2, 2], [3, 3], [6, 6]] Args: x(SelectedRows): Input with SelectedRows type. The data type is float32, float64, int32 or int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` . Returns: Variable: LoDTensor transformed from SelectedRows. The data type is same with input. Examples: .. code-block:: python >>> import paddle >>> import paddle.base as base >>> from paddle.base import core >>> paddle.enable_static() >>> scope = core.Scope() >>> block = paddle.static.default_main_program().global_block() >>> x_rows = [0, 5, 5, 4, 19] >>> height = 20 >>> x = scope.var('X').get_selected_rows() >>> x.set_rows(x_rows) >>> x.set_height(height) >>> x = block.create_var(name="X", dtype="float32", persistable=True, type=base.core.VarDesc.VarType.SELECTED_ROWS) >>> z = paddle.nn.clip.get_tensor_from_selected_rows(x) r%get_tensor_from_selected_rowszGThe type of 'x' in get_tensor_from_selected_rows must be SELECTED_ROWS.r+rrrrr rN)r.)rrr.r r rrVarDescVarType SELECTED_ROWS TypeErrorr r!r,rr$r-r(r(r)r.s - r.FcGsBt|dksJt|dkrt|dtsJt}|da|StS)Nr)len isinstancebool'_clip_by_global_norm_using_mp_type_flagargs old_valuer(r(r)"_clip_by_global_norm_using_mp_types r<cCsd|jtjjjks|jtjjjkrtr|tjjjS|jt j ks'|jt j kr0tr0|t jS|SN) rrr0r1FP16BF16r<astypeFP32rFLOAT16BFLOAT16r%r(r(r)_cast_to_mp_type_if_enableds rEcCspt|}tr t|Sd}t|dgd|t|fit}||j}d|i}d|i}|j |||d|S)z1 Return the squared L2 norm of a tensor. squared_l2_normr%)rfloat64rrrrrrr ) rErrrFrr r!r,rr$)r%Zop_typer&r'rr r(r(r)_squared_l2_norms   rIc@seZdZddZddZdS)BaseErrorClipAttrcCtr=NotImplementedErrorselfr(r(r)__str__zBaseErrorClipAttr.__str__cCrKr=rL)rOblock grad_namer(r(r)_append_clip_oprQz!BaseErrorClipAttr._append_clip_opN)__name__ __module__ __qualname__rPrTr(r(r(r)rJs rJc@s*eZdZdZd ddZddZddZdS) ErrorClipByValuea1 Clip tensor values to the range [min, max]. Given a tensor ``t`` (see Examples below), this operation clips its value \ to ``min`` and ``max`` inplace. - Any values less than min are set to min. - Any values greater than max are set to max. Args: max (float): The maximum value to clip by. min (float, optional): The minimum value to clip by. if not set by user, \ will be set to ``-max`` by framework. Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> BATCH_SIZE = 128 >>> CLIP_MAX = 2e-6 >>> CLIP_MIN = -1e-6 >>> prog = paddle.static.Program() >>> with paddle.static.program_guard(main_program=prog): ... image = paddle.static.data(name='x', shape=[None, 784], dtype='float32') ... hidden1 = paddle.static.nn.fc(image, size=128, activation='relu') ... hidden2 = paddle.static.nn.fc(hidden1, size=64, activation='relu') ... predict = paddle.static.nn.fc(hidden2, size=10, activation='softmax') ... label = paddle.static.data(name='y', shape=[1], dtype='int64') ... cost = paddle.nn.functional.cross_entropy(input=predict, label=label) ... avg_cost = paddle.mean(cost) >>> prog_clip = prog.clone() >>> prog_clip.block(0).var(hidden1.name)._set_error_clip( ... paddle.nn.clip.ErrorClipByValue( ... max=CLIP_MAX, min=CLIP_MIN)) NcCs0t|}|dur | }nt|}||_||_dSr=)r"maxminrOrYrZr(r(r)__init__,s  zErrorClipByValue.__init__cCd|jdd|jdS)Nz ByValue, min=f, max=rZrYrNr(r(r)rP5zErrorClipByValue.__str__cCsP|j}|d|d|g|d|g|d|j|d|jdS)NcliprrrZrY)descr$set_typeZ set_inputZ set_output _set_attrrZrY)rOrRrSZ clip_op_descr(r(r)rT8s  z ErrorClipByValue._append_clip_opr=)rUrVrW__doc__r\rPrTr(r(r(r)rXs  & rXcs||j|jd}fdd|DD]&}||}t|dd}|dus4t|ts4td|dur>| ||qdS)Nr4csg|]}|vr|qSr(r().0nZ grad_to_varr(r) Esz'error_clip_callback.. error_clipzIVariable's error_clip should be an instance of BaseErrorClipAttr or None.) rcopZop_sizeZoutput_arg_namesZ_var_recursivegetattrr6rJr3rT)rRcontextZop_descZgrad_nZfwd_varrkr(rir)error_clip_callbackAs  rocs\eZdZfddZddZeddZddZd d Z d d Z d dZ ddZ Z S) ClipGradBasecstdSr=)superr\rN __class__r(r)r\SzClipGradBase.__init__cCrKr=rLrNr(r(r)rPVrQzClipGradBase.__str__cCtr=rLrO params_gradsr(r(r) _dygraph_clipYszClipGradBase._dygraph_clipcCrur=rLrvr(r(r) _pir_clip]zClipGradBase._pir_clipcCrur=rLrvr(r(r) _static_clip`rzzClipGradBase._static_clipcCsVtr||Str||S|D]\}}t|dddur%tdnq||S)Ngradient_clip_attrz'set_gradient_clip' will be ineffective, because you have set 'need_clip' in 'ParamAttr'. So, 'set_gradient_clip' is redundant and you can remove it.)r rxrryrmwarningswarnr{)rOrwpgr(r(r)__call__cs    zClipGradBase.__call__cCrKr=rLrOrnparamgradr(r(r)_process_contextsrQzClipGradBase._process_contextcCrKr=rL)rOrrr(r(r)_create_operatorsvrQzClipGradBase._create_operators)rUrVrWr\rPimperative_baseno_gradrxryr{rrr __classcell__r(r(rrr)rpRs  rpcsReZdZdZdfdd ZddZeddZd d Z d d Z d dZ Z S)ClipGradByValuea Limit the value of multi-dimensional Tensor :math:`X` to the range [min, max]. - Any values less than min are set to ``min``. - Any values greater than max are set to ``max``. The multi-dimensional Tensor :math:`X` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). Note: ``need_clip`` of ``ClipGradByValue`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. Args: max (float): The maximum value to clip by. min (float, optional): The minimum value to clip by. if not set by user, it will be set to ``-max`` automatically. In this case, ``max`` must be greater than :math:`0`. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByValue(min=-1, max=1) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() Ncs<t|dur|dksJ| }t||_t||_dS)Ng)rqr\r"rYrZr[rrr(r)r\s   zClipGradByValue.__init__cCr])NzClip Gradient By Value, min = r^r_r`rNr(r(r)rPrazClipGradByValue.__str__cCsbg}|D]*\}}|dur qt|dddur|||fqtj||j|jd}|||fq|S)N need_clipTFr%rZrY)rmappendpaddlerbrZrYrOrwparams_and_gradsrrnew_gradr(r(r)rxs zClipGradByValue._dygraph_clipc Csg}i}tdT|D]I\}}|durq t|dddur%|||fq |jj||gtj||j |j d}Wdn1sCwY|||f|j ||j <q Wdn1s`wYt |||S)N gradient_cliprTFr) r name_scopermrrRprogram_optimized_guardrrbrZrYr_correct_clip_op_role_varrOrwrparam_new_grad_name_dictrrrr(r(r)r{s$   zClipGradByValue._static_clipcCdSr=r(rr(r(r)rrzz ClipGradByValue._process_contextcCstj||j|jd}||fS)Nr)rrbrZrYrOrrrr(r(r)rsz!ClipGradByValue._create_operatorsr= rUrVrWrfr\rPrrrxr{rrrr(r(rrr)rzs'  rcsPeZdZdZfddZddZeddZdd Z d d Z d d Z Z S)ClipGradByNorma Limit the l2 norm of multi-dimensional Tensor :math:`X` to ``clip_norm`` . - If the l2 norm of :math:`X` is greater than ``clip_norm`` , :math:`X` will be compressed by a ratio. - If the l2 norm of :math:`X` is less than or equal to ``clip_norm`` , nothing will be done. The multidimensional Tensor :math:`X` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). The clipping formula is: .. math:: Out = \left\{ \begin{array}{ccl} X & & if (norm(X) \leq clip\_norm) \\ \frac{clip\_norm*X}{norm(X)} & & if (norm(X) > clip\_norm) \\ \end{array} \right. where :math:`norm(X)` represents the L2 norm of :math:`X`. .. math:: norm(X) = ( \sum_{i=1}^{n}|x\_i|^2)^{ \frac{1}{2}} Note: ``need_clip`` of ``ClipGradByNorm`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. Args: clip_norm(float): The maximum norm value. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByNorm(clip_norm=1.0) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() cstt||_dSr=)rqr\r" clip_norm)rOrrrr(r)r\ s zClipGradByNorm.__init__cC d|jS)Nz#Gradient Clip By Norm, clip_norm=%frrNr(r(r)rP zClipGradByNorm.__str__cCs\g}|D]'\}}|dur qt|dddur|||fqt||jd}|||fq|S)NrTFr%r)rmrrrrr(r(r)rxs zClipGradByNorm._dygraph_clipc Csg}tdSi}|D]F\}}|durq t|dddur%|||fq |jj||gt||jd}Wdn1s@wY|j ||j <|||fq Wdn1s]wYt |||S)NrrTFr) rrrmrrRrrrrrrrr(r(r)r{s$    zClipGradByNorm._static_clipcCrr=r(rr(r(r)r1rzzClipGradByNorm._process_contextcCst||jd}||fS)Nr)rrrr(r(r)r4sz ClipGradByNorm._create_operatorsrr(r(rrr)rs 6  rcG>t|dkrtSt|dkrt|dtsJt}|da|SNrr4)r5&_allow_pure_fp16_global_norm_clip_flagr6r7r9r(r(r)!_allow_pure_fp16_global_norm_clip< rcGrr)r5&_allow_pure_bf16_global_norm_clip_flagr6r7r9r(r(r)!_allow_pure_bf16_global_norm_clipJrrcs\eZdZdZ dfdd ZddZedd Zd d Z d d Z ddZ ddZ Z S)ClipGradByGlobalNormao Given a list of Tensor :math:`t\_list` , calculate the global norm for the elements of all tensors in :math:`t\_list` , and limit it to ``clip_norm`` . - If the global norm is greater than ``clip_norm`` , all elements of :math:`t\_list` will be compressed by a ratio. - If the global norm is less than or equal to ``clip_norm`` , nothing will be done. The list of Tensor :math:`t\_list` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). The clipping formula is: .. math:: t\_list[i] = t\_list[i] * \frac{clip\_norm}{\max(global\_norm, clip\_norm)} where: .. math:: global\_norm = \sqrt{\sum_{i=0}^{N-1}(l2norm(t\_list[i]))^2} Note: ``need_clip`` of ``ClipGradyGlobalNorm`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. Args: clip_norm (float): The maximum norm value. group_name (str, optional): The group name for this clip. Default value is ``default_group``. auto_skip_clip (bool, optional): skip clipping gradient. Default value is ``False``. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() default_groupFcs8tt||_||_t|tsJ||_d|_dSr=) rqr\r"r group_namer6r7auto_skip_clip _async_add_n)rOrrrrrr(r)r\s   zClipGradByGlobalNorm.__init__cCr)Nz+Gradient Clip By GlobalNorm, global_norm=%frrNr(r(r)rPrzClipGradByGlobalNorm.__str__cCsg}g}g}g}|D]a\}}|durq t|dddurq |}tr.|r.t|}|}n|jtjjj kr>t|}t |}t |} | j tjjj ksR| j tjjjkrX|| q | j tjjjkrf|| q || q t|t|t|dkr||Sdd} t|dkrdnd} g} t|dkr| |} | | | t|dkr| |}| dkr| |n| || t|dkr| |}| || | } t| } tjg| j |jd }d}|jsd}tj|tj| |d d }n | |krd}tj|| d }|D]B\}}|dur qt|dddur|||fq|r:|j |j kr*||j n|}t||}|||fq|||fq|S) NrTFrcSt|Sr=rstacksumZvar_listr(r(r) async_add_nrtz7ClipGradByGlobalNorm._dygraph_clip..async_add_nrGrshaperZ fill_valuer%y)rmr Zis_selected_rowsr*Z_get_tensor_from_selected_rowsrrr0r1r2r.rIrr>r?rrAr5r@rsqrtfullrrdividemaximummultiplyrOrwrsum_square_listsum_square_list_fp16sum_square_list_fp32rr merge_grad sum_squarer sum_dtypeglobal_norm_varglobal_norm_var_fp16global_norm_var_fp32Zglobal_norm_var_fp64max_global_normrclip_varZ clip_inputrr(r(r)rxs                 z"ClipGradByGlobalNorm._dygraph_clipcCsXg}g}g}g}|D]J\}}|durq t|dddurq |}tr-|r-t|}t|}t|} | jtjks=| jtj krC| | q | jtj krO| | q | | q t |t |t |dkre|Sdd} t |dkrqdnd} g} t |dkr| |} | | | t |dkr| |}| dkr| |n| | | t |dkr| |}| || | } t| } tjg| j|jd }d}|jsd}tj|tj| |d d }n | |krd}tj|| d }|D]A\}}|durqt|dddur| ||fq|r"|j|jkr| |jn|}t||}| ||fq| ||fq|S) NrTFrcSrr=rrr(r(r)r rtz3ClipGradByGlobalNorm._pir_clip..async_add_nrGrrr)rmris_selected_row_typer*r.rIrrrBrCrZFLOAT32r5r@rrrrrrrrrr(r(r)rys                  zClipGradByGlobalNorm._pir_clipc sg}g}g}g}g}fdd}td|D]p\}} | dur"qt|dddur+q| } |jj|| gK| jtjj j krGt | } t | } t | } | jtjj jkrY|| n!| jtjj jkrg|| n| jtjj jkru|| n|| Wdn1swYqt|dkrt|dkrtdt|t|t|dkrt|t|t|dkr|WdS|jj|| gt|dkrd nd } g} t|dkr||}|s|sts| || n| |t|dkr||}|s |s ts| || n| |t|dkr5||}| d kr-| |n| || t|dkrE||}| |t| d krP|| n| d} tj| d } tjd g| jjd }tj|tj|| dd}Wdn 1s|wYi}|D]\}} | durqt|dddur||| fq|jj|| ggt| }|jtjj jkr|jtjj jkr|d}n|jtjj jkr|jtjj jkr|d}n|}t }|j!d||dd|id|| ur |j!dd|id| i|j| jddWdn 1swY| j"||j"<||| fqWdn 1s5wYt#|||S)Ncsjr t|St|Sr=)rrrrZadd_nrrNr(r)_add_n^s z1ClipGradByGlobalNorm._static_clip.._add_nrrTFrz1FP16 and BF16 are not supported at the same time.rGrr4rDrrrZbfloat16elementwise_mulrYrrHcastr)Zin_dtypeZ out_dtyper/)$rrrmrRrrrrr0r1r2r*r.rIrr>rr?rAr5rrr@rrrrrrrrEr Z current_blockr$rr)rOrwrrrZsum_square_list_bf16rrrrrrrrrZglobal_norm_var_bf16rZglobal_norm_var_other_dtyperZ scale_varrZnew_gZ scale_inputrRr(rNr)r{Ws      (        8       $ z!ClipGradByGlobalNorm._static_clipcCs|j|vr#g||j<|j||jd<tjdg|j|jd||jd<n|j||jdks1td|}|jtjj j krDt |}t |}nt rS|rSt |}t |}t|}||j|||_dS)NZ _clip_valuer4r_clipz>All parameters' 'clip_norm' of a same group should be the same)rrrrr ValueErrorrrr0r1r2r*r.rrrIrrn)rOrnrrrZlocal_norm_varr(r(r)rs(     z%ClipGradByGlobalNorm._process_contextcCsdd}|jd}||jvr<||j|j}tj|d}|j|jd}tj|tj||dd}|jdks7J||j|<trLt||j|}||fS|j j d||j|d d |id ||fS) NcSrr=rrr(r(r)rrtz;ClipGradByGlobalNorm._create_operators..async_add_nZ_scalerDrr)r4rrrrH) rrnrrrrrrrrRr$)rOrrrZgroup_scale_nameZgroup_norm_varrZgroup_scale_varr(r(r)rs*     z&ClipGradByGlobalNorm._create_operators)rF)rUrVrWrfr\rPrrrxryr{rrrr(r(rrr)rUs5 _[ rcstdt|tstddurtdjD]}d| vr2d| dvr2tdnq|dur>d }t dd |DrPfd d |D}t d d |Ds]td |D]}t ||_q_dS)a Warning: This API must be used after building network, and before ``minimize`` , and it may be removed in future releases, so it is not recommended. It is recommended to set ``grad_clip`` when initializing the ``optimizer`` , this is a better method to clip gradient. There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . To specify parameters that require gradient clip. Args: grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of some derived class of ``GradientClipBase`` . There are three cliping strategies ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` ). Default value: None, and there is no gradient clipping. param_list (list(Variable), optional): Parameters that require gradient clip. It can be a list of parameter or a list of parameter's name. Default None, meaning that all parameters in the program will be included. program (Program, optional): The program where parameters are located. Default None, meaning that using :ref:`api_paddle_static_default_main_program` . Returns: None Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> def network(): ... image = paddle.static.data(name='image', shape=[ ... None, 28], dtype='float32') ... param_attr1 = paddle.ParamAttr("fc1_param") ... fc1 = paddle.static.nn.fc(image, size=10, weight_attr=param_attr1) ... param_attr2 = paddle.ParamAttr("fc2_param") ... fc2 = paddle.static.nn.fc(fc1, size=10, weight_attr=param_attr2) ... loss = paddle.mean(fc2) ... return loss >>> # network 1: clip all parameter gradient >>> with paddle.static.program_guard(paddle.static.Program(), paddle.static.Program()): ... loss = network() ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByGlobalNorm(clip_norm=2.0)) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 2: clip parameter gradient by name >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByValue(min=-1.0, max=1.0), ... param_list=["fc1_param", "fc2_param"]) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 3: clip parameter gradient by value >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... param_var1 = paddle.static.default_main_program().global_block().var("fc1_param") ... param_var2 = paddle.static.default_main_program().global_block().var("fc2_param") ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByValue(min=-1.0, max=1.0), ... param_list=[param_var1, param_var2]) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 4: use 'set_gradient_clip' and 'optimize(grad_clip=clip)' together >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... clip1 = paddle.nn.ClipGradByValue(min=-1.0, max=1.0) ... clip2 = paddle.nn.ClipGradByNorm(clip_norm=1.0) ... # Set the gradient clipping strategy: clip1 ... paddle.nn.clip.set_gradient_clip(clip1) ... # Set the gradient clipping strategy: clip2 ... sgd = paddle.optimizer.SGD(learning_rate=1e-3, grad_clip=clip2) ... sgd.minimize(loss) ... # 'set_gradient_clip' will not take effect when setting has a conflict, ... # and the gradient clipping strategy will be 'clip2' zCaution! 'set_gradient_clip' is not recommended and may be deprecated in future! We recommend a new strategy: set 'grad_clip' when initializing the 'optimizer'. This method can reduce the mistakes, please refer to documention of 'optimizer'.z<'clip' should be an instance of ClipGradBase's derived classNr op_namescopeZ optimizerz'minimize' has been invoked before, this will make 'set_gradient_clip' be ineffective! Please invoke 'set_gradient_clip' before 'minimize'.css|]}t|tVqdSr=)r6strrgelemr(r(r) sz$set_gradient_clip..csg|] }d|qS)r)rRvarrrr(r)rjsz%set_gradient_clip..css|] }t|tjVqdSr=)r6r Parameterrr(r(r)rszK'param_list' should be a list of Parameter or basestring(parameter's name).)r}r~r6rpr3rr rRopsZ all_attrsattrZall_parametersallcopydeepcopyr|)rbZ param_listrrlrr(rr)set_gradient_clip.s:Z rc Csi}|D]d\}}|dur q|jj||gJtd5t|dd}|dur9|WdWdSt|tsBtd|j |||dWdn1sTwYWdn1scwYqg}i}|D]M\}}|durxqo|jj||g3td|j ||d\}}|j ||j <| ||gWdn1swYWdn1swYqot |||S)Nrr|z8clip attribute should be an instance of GradientClipBase)rnrr)rr)rRrrrrrmr6rpr3rrrrr) Z param_gradsrnrrZ clip_attrresrrrr(r(r)append_gradient_clip_opssJ  "     rc Csg}t|dkr dS|D]I\}}|durq |jj}||vrq |||jjjD]*}|drTd|dvrT|drT|dd}||vrT|||g}| d|q*q dS)NrrrZ op_role_var) r5rRidxrrZ global_blockrZhas_attrrre) rwrZ block_id_listrrZblock_idrl param_nameZ correct_p_gr(r(r)rs2    rr=)NN)7rr}sqlite3rrZpaddle.autogradZautogradrrZ paddle.baserrrZpaddle.base.data_feederrZpaddle.base.libpaddlerZpaddle.common_ops_importr r r Zpaddle.frameworkr r rrZ&paddle.tensor.layer_function_generatorr__all__rr*r.r8r<rErIrJrXrorprrrrrrrZdygraph_not_supportrrrZGradientClipBaseZGradientClipByValueZGradientClipByNormZGradientClipByGlobalNormr(r(r(r)s\       < & ? <(Ze  \ %