o #jX@sddlZddlZddlZddlZddlmZmZmZddl m Z ddl m Z ddl mZmZmZmZddlmZe eejd d ZejjjejjjejjjgZd Zd d ZddZddZ ddZ!ddZ"ddZ#e ddZ$ddZ% d%ddZ& d&dd Z'd'd!d"Z(d%d#d$Z)dS)(N)core framework global_scope) get_logger)signature_safe_contextmanager) _rename_arg_rename_op_inputfind_true_post_opfind_true_prev_op)AutoMixedPrecisionListsBF16z&%(asctime)s-%(levelname)s: %(message)s)fmtZ __use_bf16__cCs4t|}tjddtjgd|j}t||jS)NcSstdtd|dd?S)Nz2sz)convert_float_to_uint16..)Zotypes)npZasarrayZ vectorizeZuint16ZflatZreshapeshape)Zin_listoutrrrconvert_float_to_uint16/s rcCs|tjjjkr dSdS)zx Convert specific variable type to its corresponding string. Args: dtype (VarType): Variable type. Zbf16Zfp32)rVarDescVarTypeBF16)dtyperrr _dtype_to_str8src Csd}|jD]}}|tjjjkr|jdvr|dvrq||D]d}||}|jtvs.|j |kr/q|j |krv|j dt |} |j | } | dusL| j |krm|j| |d|jd} |j|dd |id | i|j | j d d |d 7}t||j | j q|dr|d|qq|tjjjkr|tjjjkr|jD]>} |jdvr| dkrq|| D],} || } | jtvrq| j tjjjkr| jtjjj|dr|dtjjjqq|S)a Insert cast op and rename args of input and output. Args: block (Program): The block in which the operator is. op (Operator): The operator to insert cast op. idx (int): The index of current operator. src_dtype (VarType): The input variable dtype of cast op. dest_dtype (VarType): The output variable dtype of cast op. Returns: num_cast_op (int): The number of cast ops that have been inserted. r) batch_normfused_bn_add_activation layer_normXZ.cast_NFnamerZ persistable stop_gradientcastr$Outin_dtype out_dtypetypeZinputsZoutputsattrsr r-Yr.) input_namesrrrFP32r0inputvar _valid_typesrr(rvarsget create_varr) _insert_oprhas_attr _set_attrr output_namesoutputdesc set_dtype) blockopidx src_dtype dest_dtype num_cast_opsin_name in_var_namein_var cast_nameout_varout_name out_var_namerrr_insert_cast_opEsj           rOc Csd}||}|jtvs|j|kr|S|j|ks%Jdt|jt||jdt|} |j| } | dus=| j|krg|j | |d|j d} |j |dd|id| i|j| jd d |d 7}| j||j |j<|S) Nrz4The real dtype({}) is not equal to the src dtype({})r&Fr'r*r$r+r,r/r ) r6r0r7rformatrr(r8r9r:r)r;rD) rBrCrDrErF target_nameop_var_rename_maprGZ target_varrKZcast_varrrr_insert_cast_post_ops8    rScCsF|jsdS|jD] }||jvrdSq|jD] }||jvr dSqdSNFT) fp32_varnamesinput_arg_namesoutput_arg_names)rC amp_listsrHrMrrr_is_in_fp32_varnamess    rYcCsr|j|vrdSg}|t|j|t|j|D] }d|vr$dSq|r7|dr5t|dvr5dSdSdS)NTZ learning_rateZ op_namescopeF)r0extendlistrVrWr<_bf16_guard_patternattr)rCZunsupported_op_listuse_bf16_guardZin_out_arg_namesr(rrr_need_keep_fp32s  r_ccs:tjtd dVWddS1swYdS)av As for the pure bf16 training, if users set `use_bf16_guard` to True, only those ops created in the context manager `bf16_guard` will be transformed as float16 type. Examples: .. code-block:: python >>> import numpy as np >>> import paddle >>> import paddle.nn.functional as F >>> paddle.enable_static() >>> data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32') >>> conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3) >>> with paddle.static.amp.bf16.bf16_guard(): ... bn = paddle.static.nn.batch_norm(input=conv2d, act="relu") ... pool = F.max_pool2d(bn, kernel_size=2, stride=2) ... hidden = paddle.static.nn.fc(pool, size=10) ... loss = paddle.mean(hidden) )prefixN)rZ name_scoper\rrrr bf16_guards"racCs(|D]}|D] }||vrdSqqdSrTr)post_ops keep_fp32_opspost_oprCrrrare_post_ops_bf16srecCs|j}|D]}t|j|jvrd}g} g} |jD]-} || D]%} || } t||| d}| dus8| jt vrr?r6r r7appendrerrrrr4r@rArr(remover<r]r=) startup_progrXrBZall_opsrcto_bf16_var_namesZ prepend_opsrCZ change_opZ op_post_opsZ op_out_varsrMrNrLrdrrrcast_initializers_to_bf16s<       rmTcCs|durt}|}t}t}t}g}|jD]} || jq|jD]_} | j} | D]H} | jdks:| jdkr;q-t| |j|rH| | q-| j D]w} | jdvrW| dvrWqK| | D]e} d}z| | }Wn-t y}z!td|d| | }|durtd| d WYd}~nd}~ww|dus|jtvrq\|jtjjjkr|jtjjj| | td | j| |jq\qK| jD]w}| jdvr|d krq| |D]e}d}z| |}Wn/t y}z"td|d| |}|durtd|d WYd}~nd}~ww|dus|jtvrq|jtjjjkr0|jtjjjtd | j||jqqd D]}| |r\| |tjjjkr\| |tjjjq@| drj| dd| drv| ddq-|durt|||| ||q%ddt t!|jD}|jD]} | j} d}|t!| kr<| |} d}| |vr| |vrt"| | |tjjjtjjj}||7}nit"| | |tjjjtjjj}||7}| j#D]S}| j$%|}|dus|jtvrq|jtjjjkr-|jtjjjt&| | |}|D]"}||vrq t'| | ||dtjjjtjjj||}||7}q q||d7}|t!| ksqt(|||||S)a  Traverse all ops in the whole model and set their inputs and outputs to the bf16 data type. This function will do some special processing for the batch normalization, which will keep the batchnorm's computations in FP32. Args: program (Program): The used program. amp_lists (AutoMixedPrecisionListsBF16): An AutoMixedPrecisionListsBF16 object. use_bf16_guard(bool): Determine whether to use `bf16_guard` when constructing the program. Default True. Ncreate_py_readerread>r r!r"r#z-- z&, try to get it in the global block --z-- var z is got in the global block --z4-- op type: {}, in var name: {}, in var dtype: {} --r2z6-- op type: {}, out var name: {}, out var dtype: {} --)r-r.r use_mkldnnTmkldnn_data_typebfloat16cSsg|]}tqSr) collections OrderedDict).0_rrr sz&cast_model_to_bf16..rr ))r rfsetblocksrZrgr0r_Zunsupported_listaddr3r5r6 ValueError_loggerdebugr7rrrrr4r@rArrPr>r?r<r]r=rmrangelenrOrWr8r9r rSr )programrkrXr^rfrcrlZto_bf16_pre_cast_opsZ origin_opsrBrgrCrHrIrJerMrNrL attr_namerRrDrGZin_var_cast_numZ pre_cast_numrbrdZ post_cast_numrrrcast_model_to_bf16+s2                                   ,rc Csg}|jD] }||q|r|nt}|r|nt}|D]&}|j|vrEtd|jd||j } t | } | t | |qdS)a Traverse all parameters in the whole model and set them to the BF16 data type. Whereas, this function will keep parameters of batchnorms in FP32. Args: place(base.CPUPlace|base.CUDAPlace): `place` is used to restore the BF16 weight tensors. program (Program): The used program. scope(base.Scope, optional): `scope` is used to get the FP32 weight tensor values. Default is None. to_bf16_var_names(set|list, optional): The data types of vars in `to_bf16_var_names` will be set to BF16. Usually, it is the returned value of `cast_model_to_bf16` API. z ---- cast z to bf16 dtype ----N) ryrZall_parametersrxrr(r|r}Zfind_varZ get_tensorrarrayr) ZplacerscoperlrrBZbf16_var_namesZ var_scopeparamZparam_tdatarrrcast_parameters_to_bf16s   rcCsP|durt}|}|j}t}t}|D]}|jdks"|jdkr#q|jdur3t||r3||q|j|jvr?||q|j|j vrK||q|j|j vrd}d}|j D]D} | r| | D]:} | | } | jdurnqa| j|urt||| } | dur~qan| j} | |vs| j|jvrd}qa| |vs| j|j vrd}qaqX|r||q|r||q q||qd} | t|kr&|| }d}||vrt||| tjjjtjjj}nA||vr|dr|dd|dd n|d r|d tjjjkr|d tjjjt||| tjjjtjjj}n | |d 7} | t|ksdSdS) a/ Traverse all ops in current block and insert cast op according to which set current op belongs to. 1. When an op belongs to the fp32 list, add it to fp32 set 2. When an op belongs to the bf16 list, add it to bf16 set 3. When an op belongs to the gray list. If one of its inputs is the output of fp32 set op or fp32 list op, add it to fp32 set. If all of its previous ops are not fp32 op and one of its inputs is the output of bf16 set op or bf16 list op, add it to bf16 set. 4. When an op isn't in the lists, add it to fp32 op set. 5. Add necessary cast ops to make sure that fp32 set op will be computed in fp32 mode, while bf16 set op will be computed in bf16 mode. Args: main_prog (Program): The main program for training. NrnroFTrrprqrrrr )r rfrgrxr0rUrYrzZ fp32_listZ bf16_listZ gray_listr3r5r6rCr rrOrrrrr4r<r=r])Z main_progrXrBrgZ bf16_op_setZ fp32_op_setrCZ is_fp32_opZ is_bf16_oprHrIrJZprev_oprDrGrrrrewrite_program_bf16s                     r)N)NNT)NN)*rsloggingrnumpyrZ paddle.baserrrZpaddle.base.log_helperrZpaddle.base.wrapped_decoratorrZ fp16_utilsrr r r rXr __name__INFOr|rrZ LOD_TENSORZ SELECTED_ROWSZLOD_TENSOR_ARRAYr7r\rrrOrSrYr_rarermrrrrrrrsB     L$  *  /