o *jB@s<ddlZddlmZddlmZeZGdddeZdS)N)PretrainedConfig)loggingc$sxeZdZdZdddddddddd d d d d d d d d d dd dd d dddddddgd dd d df$fdd ZeddZZS) GPTMoEConfigzgpt-moeidiN i Zgelug?ig-q=TFg{Gz?drg?standardc%( stjdd| i|%||_||_|durd|n||_||_||_||_||_||_ | |_ | |_ | |_ | |_ | |_||_||_||_||_|rL|rLJ||_||_|durc||dks^J|||_||_||_||_||_||_||_||_||_||_||_||_ | |_!|!|_"|"|_#|#|_$|$|_%|j dt&j'(krt&j'(|_)n|j d|_)t*t&j+,dd}&t*t&j+,dd}'|&dkp|&dko|'dk|_-dS)NZlayer_norm_epsr. ).super__init__ vocab_size hidden_sizeffn_hidden_sizenum_hidden_layersnum_attention_heads hidden_actintermediate_sizehidden_dropout_probattention_probs_dropout_probmax_position_embeddingstype_vocab_sizelayernorm_epsilonbias_gelu_fusionfp32_residual_connectionsequence_parallelfp16bf16apply_query_key_layer_scalingattention_softmax_in_fp32 kv_channelsmasked_softmax_fusionattention_dropoutbias_dropout_fusion(apply_residual_connection_post_layernormhidden_dropoutinit_method_stdeod_idtokens_to_generatetop_ktop_p num_experts use_tuteltop_k_linear_strategyuse_expert_residual_network load_ds_ckpts model_dirtorchcudaZ device_countZmoe_expert_parallel_sizeint __version__splitZno_persist_layer_norm)(selfrrrrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4kwargsZ TORCH_MAJORZ TORCH_MINOR __class__rl/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/nlp/gpt_moe/configuration.pyrsf(   zGPTMoEConfig.__init__cCs|jrtjS|jr tjStjS)N)r r5Zhalfr!Zbfloat16float)r:rrr> params_dtypeys zGPTMoEConfig.params_dtype)__name__ __module__ __qualname__Z model_typerpropertyr@ __classcell__rrr<r>rsR^r)r5Z transformers.configuration_utilsrZtransformers.utilsrZ get_loggerloggerrrrrr>s