o 0jS@sddlZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z ddl Z ddlmZddlmZmZmZddlmZddlmZdd lmZdd lmZmZdd lmZd d lm Z d dl!m"Z"ddl#m$Z$edrrddl%Z%ej&edGddde$Z'dS)N)AnyDictListOptionalTupleUnion)logging)function_requires_depsis_dep_availablepipeline_requires_extra) custom_open)ImageBatchSampler) ReadImage) HPIConfigPaddlePredictorOption) benchmark)BaseChat)LayoutParsingResult)PP_ChatOCR_Pipelineopencv-contrib-pythonZiec/seZdZdZdgZdddddddddedeed eed eeeefd ee d e d ee eeefe fde ddffddZ ddZdeddfddZdeddfddZdeddfddZdeddfddZdedefddZ                     dtde eeeejeejfd ee d!ee d"ee d#ee d$ee d%ee eefd&ee d'ee eeeefefd(eed)eed*eed+eed,eed-eed.eed/eed0eed1eed2eed3eed4eedef.d5d6Zd7ed8eddfd9d:Zd;edeefdeedeeeeeffd?d@Z A B  dud7edCedDedEe dFedef dGdHZ! dvdIed8edFeddfdJdKZ"dvd;edFedefdLdMZ#dNe eeefdeefdOdPZ$e%dQ dvde eejfdNe eeefdefdRdSZ&dTe'dUedNedVedWeddf dXdYZ(dFedZe dIedNeed[edCedefd\d]Z)dTe'dNeed^ed_edef d`daZ*   A            b  dwdNe eeefd7edZe dIedCedceddedeedfedgedhediedjedkedledmednedoedFedef(dpdqZ+dxdrdsZ,Z-S)yPP_ChatOCRv4_PipelinezPP-ChatOCRv4 PipelinezPP-ChatOCRv4-docNFT)deviceengine engine_config pp_optionuse_hpip hpi_configinitial_predictorconfigrrrrrr r!returnc stjd ||||||d| |d|_||_|dd|_|dd|_d|_d|_d|_ d|_ |rH| || || |||tdd|_td d |_d |_dS) aInitializes the PP-ChatOCRv4 pipeline. Args: config (Dict): Configuration dictionary containing various settings. device (Optional[str], optional): The device to use for prediction. Defaults to `None`. engine (Optional[str], optional): Inference engine. Defaults to `None`. engine_config (Optional[Dict[str, Any]], optional): Engine-specific config. Defaults to `None`. pp_option (Optional[PaddlePredictorOption], optional): Paddle predictor options. Defaults to `None`. use_hpip (bool, optional): Whether to use HPIP. Defaults to `False`. hpi_config (Optional[Union[Dict[str, Any], HPIConfig]], optional): HPIP configuration. Defaults to `None`. initial_predictor (bool, optional): Whether to initialize the predictor. Defaults to `True`. )rrrrrr pipeline_nameuse_layout_parserTuse_mllm_predictNr)Z batch_sizeZBGR)formati)super__init__r$r"getr%r&layout_parsing_pipelinechat_bot retriever mllm_chat_botinintial_visual_predictorinintial_chat_predictorinintial_retriever_predictorinintial_mllm_predictorrZ batch_samplerr img_readertable_structure_len_max) selfr"rrrrrr r!kwargs __class__r(s/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/paddlex/inference/pipelines/pp_chatocr/pipeline_v4.pyr*2s4        zPP_ChatOCRv4_Pipeline.__init__cCs|jdur |jdSdSN)r,close)r6r(r(r:r<ms zPP_ChatOCRv4_Pipeline.closecCs<|dd|_|jr|didddi}|||_dS)a Initializes the visual predictor with the given configuration. Args: config (dict): The configuration dictionary containing the necessary parameters for initializing the predictor. Returns: None r%TZ SubPipelinesZ LayoutParserZpipeline_config_errorz)config error for layout_parsing_pipeline!N)r+r%Zcreate_pipeliner,)r6r"Zlayout_parsing_configr(r(r:r0qs   z/PP_ChatOCRv4_Pipeline.inintial_visual_predictorcCs2ddlm}|didddi}|||_dS)a  Initializes the retriever predictor with the given configuration. Args: config (dict): The configuration dictionary containing the necessary parameters for initializing the predictor. Returns: None rcreate_retriever SubModulesZ LLM_RetrieverZretriever_config_errorzconfig error for llm retriever!N)r>r+r.)r6r"r>retriever_configr(r(r:r2s z2PP_ChatOCRv4_Pipeline.inintial_retriever_predictorcCsddlm}|didddi}|||_ddlm}|didid d d i}|||_|didid d d i}|||_dS)a Initializes the chat predictor with the given configuration. Args: config (dict): The configuration dictionary containing the necessary parameters for initializing the predictor. Returns: None rcreate_chat_botr?ZLLM_ChatZchat_bot_config_errorzconfig error for llm chat bot!)create_prompt_engineeringPromptEngneeringZKIE_CommonTextpe_config_errorzconfig error for text_pe!Z KIE_Tablezconfig error for table_pe!N)r@rCr+r-rDtext_petable_pe)r6r"rCchat_bot_configrDZtext_pe_configZtable_pe_configr(r(r:r1s0       z-PP_ChatOCRv4_Pipeline.inintial_chat_predictorcCstddlm}m}|dd|_|jr8|didddi}|||_|did id d d i}|||_d S)a Initializes the predictor with the given configuration. Args: config (dict): The configuration dictionary containing the necessary parameters for initializing the predictor. Returns: None r)rCrDr&Tr?Z MLLM_Chatmllm_chat_bot_configzconfig error for mllm chat bot!rEZEnsemblerFzconfig error for ensemble_pe!N)r@rCrDr+r&r/ ensemble_pe)r6r"rCrDrJZensemble_pe_configr(r(r:r3s"     z-PP_ChatOCRv4_Pipeline.inintial_mllm_predictorlayout_parsing_resultcCsi}|d}tt|D],}||d}||d}|dvrq d|}||vr-|||<q ||d|7<q |d}g} g} g} |D]} | | dd | d d } | | | | d qEi}||d <| |d<| |d<| |d<|S)z Decodes the visual result from the layout parsing result. Args: layout_parsing_result (LayoutParsingResult): The result of layout parsing. Returns: dict: The decoded visual information. parsing_res_listZ block_labelZ block_content)tableZformulaz words in z table_res_listZ pred_html Ztable_ocr_predZ rec_textsZneighbor_textsnormal_text_dicttable_text_listtable_html_listtable_nei_text_list)rangelenappendjoin)r6rLrQrMZpnolabelcontentkeyrOrRrSrTZ table_resZsingle_table_text visual_infor(r(r:decode_visual_results4      z*PP_ChatOCRv4_Pipeline.decode_visual_resultinputuse_doc_orientation_classifyuse_doc_unwarpinguse_textline_orientationuse_seal_recognitionuse_table_recognitionlayout_threshold layout_nmslayout_unclip_ratiolayout_merge_bboxes_modetext_det_limit_side_lentext_det_limit_typetext_det_threshtext_det_box_threshtext_det_unclip_ratiotext_rec_score_threshseal_det_limit_side_lenseal_det_limit_typeseal_det_threshseal_det_box_threshseal_det_unclip_ratioseal_rec_score_threshcks|jdkrtdddiV|jdur td||j|jj|fid|d|d|d |d |d |d |d | d| d| d| d| d|d|d|d|d|d|d|d|d|D]}||}||d}|VqgdS)a This function takes an input image or a list of images and performs various visual prediction tasks such as document orientation classification, document unwarping, general OCR, seal recognition, and table recognition based on the provided flags. Args: input (Union[str, list[str], np.ndarray, list[np.ndarray]]): Input image path, list of image paths, numpy array of an image, or list of numpy arrays. use_doc_orientation_classify (bool): Flag to use document orientation classification. use_doc_unwarping (bool): Flag to use document unwarping. use_textline_orientation (Optional[bool]): Whether to use textline orientation prediction. use_seal_recognition (bool): Flag to use seal recognition. use_table_recognition (bool): Flag to use table recognition. layout_threshold (Optional[float]): The threshold value to filter out low-confidence predictions. Default is None. layout_nms (Optional[bool], optional): Whether to use layout-aware NMS. Defaults to `False`. layout_unclip_ratio (Optional[Union[float, Tuple[float, float]]], optional): The ratio of unclipping the bounding box. Defaults to `None`. If it's a single number, then both width and height are used. If it's a tuple of two numbers, then they are used separately for width and height respectively. If it's None, then no unclipping will be performed. layout_merge_bboxes_mode (Optional[str], optional): The mode for merging bounding boxes. Defaults to `None`. text_det_limit_side_len (Optional[int]): Maximum side length for text detection. text_det_limit_type (Optional[str]): Type of limit to apply for text detection. text_det_thresh (Optional[float]): Threshold for text detection. text_det_box_thresh (Optional[float]): Threshold for text detection boxes. text_det_unclip_ratio (Optional[float]): Ratio for unclipping text detection boxes. text_rec_score_thresh (Optional[float]): Score threshold for text recognition. seal_det_limit_side_len (Optional[int]): Maximum side length for seal detection. seal_det_limit_type (Optional[str]): Type of limit to apply for seal detection. seal_det_thresh (Optional[float]): Threshold for seal detection. seal_det_box_thresh (Optional[float]): Threshold for seal detection boxes. seal_det_unclip_ratio (Optional[float]): Ratio for unclipping seal detection boxes. seal_rec_score_thresh (Optional[float]): Score threshold for seal recognition. **kwargs: Additional keyword arguments. Returns: dict: A dictionary containing the layout parsing result and visual information. Fz1The models for layout parser are not initialized.errorNzGThe layout parsing pipeline is not initialized, will initialize it now.r_r`rarbrcrdrerfrgrhrirjrkrlrmrqrnrorprrrs)rLr\) r%r rtr,warningr0r"predictr])r6r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsr7rLr\Zvisual_predict_resr(r(r:visual_predictsx @          z$PP_ChatOCRv4_Pipeline.visual_predictr\ save_pathcCs^t|ts |g}n|}t|d}|tj|dddWddS1s(wYdS)a- Save the visual info list to the specified file path. Args: visual_info (dict): The visual info result, which can be a single object or a list of objects. save_path (str): The file path to save the visual info list. Returns: None wF ensure_ascii N) isinstancelistopenwritejsondumps)r6r\rxvisual_info_listfoutr(r(r:save_visual_info_listrs   z+PP_ChatOCRv4_Pipeline.save_visual_info_list data_pathcCsBt|d}|}t|}Wd|S1swY|S)z Loads visual info list from a JSON file. Args: data_path (str): The path to the JSON file containing visual info. Returns: list[dict]: A list of dict objects parsed from the JSON file. rN)rreadlinerloads)r6rfindatarr(r(r:load_visual_info_lists   z+PP_ChatOCRv4_Pipeline.load_visual_info_listrc Csg}g}g}g}|D]5}|d}|D] }||dd||<q|d} |d} |d} |||| || || q ||||fS)a Merge visual info lists. Args: visual_info_list (list[dict]): A list of visual info results. Returns: tuple[list, list, list, list]: A tuple containing four lists, one for normal text dicts, one for table text lists, one for table HTML lists. one for table neighbor texts. rQr|r@rRrSrT)replacerWextend) r6rall_normal_text_listall_table_text_listall_table_html_listall_table_nei_text_listZsingle_visual_inforQr[rRrSrTr(r(r:merge_visual_info_lists(    z,PP_ChatOCRv4_Pipeline.merge_visual_info_list ,min_characters block_sizeflag_save_bytes_vectorrAcCsrt|ts |g}n|}|durddlm}||}n|jdur*td||j|j}| |} | \} } } } i}g}t | D]\}}| D]\}}||d|dg7}qHq@t | | | D]\}}}t |||jkrx|d|d|g7}q_d |}d |d <t ||krd |d <|j|d <||d<|j||d|d<|r||d|d<d|d <|Sd|d <||d<|S)a Build a vector representation from visual information. Args: visual_info (dict): The visual information input, can be a single instance or a list of instances. min_characters (int): The minimum number of characters required for text processing, defaults to 3500. block_size (int): The size of each chunk to split the text into. flag_save_bytes_vector (bool): Whether to save the vector as bytes, defaults to `False`. retriever_config (dict): The configuration for the retriever, defaults to `None`. Returns: dict: A dictionary containing the vector info and a flag indicating if the text is too short. Nrr=8The retriever is not initialized,will initialize it now.：r|utable： r@Frflag_too_short_text model_namer)rvectorT)r}r~r@r>r.r rur2r"r enumerateitemsziprVr5rXrZgenerate_vector_databaseencode_vector_store_to_bytes)r6r\rrrrArr>r.all_visual_inforrrr vector_info all_itemsirQtypetext table_html table_texttable_nei_textZ all_text_strr(r(r: build_vectorsb          z"PP_ChatOCRv4_Pipeline.build_vectorrc Cstj|}tj|st||dur ddlm}||}n|jdur0t d| |j |j}t |}|dsM|dsM||d|d<d|d<t|d}|tj|d d d WddS1sjwYdS) Nrr=rrrrTryFrzr|)ospathdirnameexistsmakedirsr@r>r.r rur2r"copydeepcopyrr rrr) r6rrxrA directoryr>r.Zvector_info_datarr(r(r: save_vectors6          z!PP_ChatOCRv4_Pipeline.save_vectorcCsd}|durddlm}||}n|jdur!td||j|j}t|dG}|}t |}d|vs?d|vs?d|vrOt dd d iWdS|drh| |d|d<d |d<Wd|SWd|S1sswY|S) Nrr=rrrrrzInvalid vector info.rtz%Invalid vector info when load vector!F) r@r>r.r rur2r"rrrrrtdecode_vector_store_from_bytes)r6rrArr>r.rrr(r(r: load_vector)s>           z!PP_ChatOCRv4_Pipeline.load_vectorkey_listcCsZ|dkrgSt|trdd|D}|St|tr+tdd|}|ddd}|SgS)z Formats the key list. Args: key_list (str|list[str]): A string or a list of strings representing the keys. Returns: list[str]: A list of formatted keys. r@cSsg|]}|ddqS) rP)r.0r[r(r(r: Xsz4PP_ChatOCRv4_Pipeline.format_key..z [\t\n\r\f\v]u，,)r}r~strresubrsplit)r6rr(r(r: format_keyJs   z PP_ChatOCRv4_Pipeline.format_keyrc CsP|jdkrtdddiS||}t|dkrddiSt|tr+tdddiSt|tr>|d r>td dd iS|j d urNt d | |j |d ur]ddl m}||}n|j }||gD]?}td|d}t|d}i} |D]!} t| d} |j| |dd} | d urddiS| | | <q~d| iSd S)a Generates MLLM results based on the provided key list and input image. Args: input (Union[str, np.ndarray]): Input image path, or numpy array of an image. key_list (Union[str, list[str]]): A single key or a list of keys to extract information. chat_bot_config (dict): The parameters for LLM chatbot, including api_type, api_key... refer to config file for more details. Returns: dict: A dictionary containing the chat results. FzMLLM prediction is disabled.Zmllm_resz"Error:MLLM prediction is disabled!r Error:输入的key_list无效！z-Input is a list, but it's not supported here.z3Error:Input is a list, but it's not supported here!z.pdfz/MLMM prediction does not support PDF currently!z5Error:MLMM prediction does not support PDF currently!Nz.rrrzThe vector model name (z+) does not match the retriever model name (z&). Please check your retriever config.r2)Ztopkrrr@rr|zMThe input text content is too long, the large language model may truncate it.)r@r>r.r rur2r"rrZsimilarity_retrievalrVrXrr)r6rArrrrrr>r.Zquestion_key_listr related_textrrrQrrr(r(r:get_related_normal_textsL           z-PP_ChatOCRv4_Pipeline.get_related_normal_textocr_llm_predict_dictmllm_predict_dictcCsi}|D]}d}d}d} ||vr||}||vr||} |dkrl| dkrl|j||| } || } | d} | d} | durNd|vrG| g|d<n|d| | durW|| } || vri| |}d|vrf| }n|}n|}n||vru||}n||vr}||}|dkr|||<q|S)a Ensemble OCR_LLM and LMM predictions based on given key list. Args: key_list (list[str]): List of keys to retrieve predictions. ocr_llm_predict_dict (dict): Dictionary containing OCR LLM predictions. mllm_predict_dict (dict): Dictionary containing mLLM predictions. Returns: dict: A dictionary with final predictions. r@rZrNB)rKgenerate_promptrrWr)r6r-rrrfinal_predict_dictr[rvZocr_llm_predictZ mllm_predictrrrrtmpr(r(r:ensemble_ocr_llm_mllmsR   z+PP_ChatOCRv4_Pipeline.ensemble_ocr_llm_mllm integrationtext_task_descriptiontext_output_formattext_rules_strtext_few_shot_demo_text_content!text_few_shot_demo_key_value_listtable_task_descriptiontable_output_formattable_rules_str table_few_shot_demo_text_content"table_few_shot_demo_key_value_listmllm_predict_infomllm_integration_strategyrIc& Cs||}|}t|dkrddiSt|ts|g}n|}|jdur.td||j |dur=ddl m }||}n|j}| |}|\}}}}i}gd}t|dkr}| ||||||}t|dkr}|jj|||||| | d } ||| |||t|dkrt|||D]<\}!}"}#t|!||jkr|!fD]*}$t|dkrt|#dkr|$d |#}$|jj|$|| | | ||d } ||| |||qq|jr|d kr|dur|d kr|||||}%d|%iS|d kr|}%d|%iSdd|diS|}%d|%iS)a Generates chat results based on the provided key list and visual information. Args: key_list (Union[str, list[str]]): A single key or a list of keys to extract information. visual_info (dict): The visual information result. use_vector_retrieval (bool): Whether to use vector retrieval. vector_info (dict): The vector information for retrieval. min_characters (int): The minimum number of characters required for text processing, defaults to 3500. text_task_description (str): The description of the text task. text_output_format (str): The output format for text results. text_rules_str (str): The rules for generating text results. text_few_shot_demo_text_content (str): The text content for few-shot demos. text_few_shot_demo_key_value_list (str): The key-value list for few-shot demos. table_task_description (str): The description of the table task. table_output_format (str): The output format for table results. table_rules_str (str): The rules for generating table results. table_few_shot_demo_text_content (str): The text content for table few-shot demos. table_few_shot_demo_key_value_list (str): The key-value list for table few-shot demos. mllm_predict_dict (dict): The dictionary of mLLM predicts. mllm_integration_strategy (str): The integration strategy of mLLM and LLM, defaults to "integration", options are "integration", "llm_only" and "mllm_only". chat_bot_config (dict): The parameters for LLM chatbot, including api_type, api_key... refer to config file for more details. retriever_config (dict): The parameters for LLM retriever, including api_type, api_key... refer to config file for more details. Returns: dict: A dictionary containing the chat results. rZchat_resrNz;The LLM chat bot is not initialized,will initialize it now.rrB)ru未知u未找到关键信息Noner@)Ztask_descriptionZ output_formatZ rules_strZfew_shot_demo_text_contentZfew_shot_demo_key_value_listu 表格周围文字：Zllm_onlyrZ mllm_onlyz,Error:Unsupported mllm_integration_strategy z9, only support 'integration', 'llm_only' and 'mllm_only'!)rrrVr}r~r-r rur1r"r@rCrrrGrrrr5rHr&r)&r6rr\rrrrrrrrrrrrrrrrIrAZ key_list_orirrCr-rrrrrrrrrrrrZ table_inforr(r(r:chatRs 1                   zPP_ChatOCRv4_Pipeline.chatcOstddS)NzPP-ChatOCRv4-doc Pipeline do not support to call `predict()` directly! Please invoke `visual_predict`, `build_vector`, `chat` sequentially to obtain the result.)r rt)r6argsr7r(r(r:rvszPP_ChatOCRv4_Pipeline.predict)NNNNNNNNNNNNNNNNNNNNN)rrFNr;)TNrNNNNNNNNNNNrNN)r#N).__name__ __module__ __qualname____doc__entitiesrrrrrboolrrr*r<dictr0r2r1r3rr]rnpZndarrayfloatrintrwrrr~rrrrrr rrrrrrrv __classcell__r(r(r8r:r+s   ;),      k ' P !"!  > . B E      r)(rrrrrtypingrrrrrrnumpyrutilsr Z utils.depsr r r Zutils.file_interfacer Zcommon.batch_samplerrZ common.readerrmodelsrrZutils.benchmarkrZcomponents.chat_serverrZlayout_parsing.resultrZ pipeline_baserrZ time_methodsrr(r(r(r:s,