o
    *ÎjµN  ã                   @   s  d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlZd dlZd dlZd dlm  mZ d dlZd dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( e(ƒ Z)e$j*ej+dG dd„ de"ƒƒZ,dS )é    N)Údatetime)ÚDictÚOptional)ÚTrainers)ÚNeRFReconPreprocessor)ÚBlenderDatasetÚColmapDataset)Ú	NeRFModel)ÚObjectSegmenter)ÚPSNR)ÚBaseTrainer)ÚTRAINERS)Ú	ModelFile)Ú
get_logger)Úmodule_namec                       sª   e Zd ZdZ												ddedef‡ fdd„Zdd„ Zd	d
„ Zdd„ Z	dde	e de
eef fdd„Zdd„ Zdd„ Zddd„Zdd„ Zddd„Z‡  ZS ) ÚNeRFReconAccTrainera  initialize the acceleration version of nerf reconstruction model for object.

    Args:
        model (str): the model path.
        cfg_file (str): cfg json file
        data_type (str): only support 'blender' or 'colmap'
        use_mask (bool): whether use mask of objects, default True
        max_step (int): max train steps, default 30000
        train_num_rays (int): init number of rays in training, default 256
        num_samples_per_ray (int): sampling numbers for each ray, default 1024
        max_train_num_rays (int): max number of rays in training, default 8192
        test_ray_chunk (int): chunk size for rendering, default 1024
        dynamic_ray_sampling (bool): whether use dynamic ray sampling when training, default True
        max_size (int): max size of (width, height) when training, default 800
        n_test_traj_steps (int): number of testing images, default 120
        log_every_n_steps (int): print log info every n steps, default 1000
        work_dir (str): dir to save ckpt and other results
        render_images (bool): whether to render test image after training
        save_mesh (bool): whether to save the reconstructed mesh of object, default False
        save_ckpt (bool): whether to save the checkpoints in data_dir, default False
        network_cfg (dict): args of network config
        match_type (str): colmap feature matching type, only for colmap data
        frame_count (str): extract number of frames, only for video input
        use_distortion (bool): whether run colmap undistortion
    NÚmodelÚcfg_filec                    sú  |   |¡}|| _|d u rt |tj¡}tƒ  |¡ tj	 
¡ s"tdƒ‚i | _|  ¡  | ¡ D ]	\}}|| j|< q-|d ur@|| jd< |d urI|| jd< |d urR|| jd< |d ur[|| jd< |d urd|| jd< |d urm|| jd< |	d urv|	| jd< |
d ur|
| jd	< |d urˆ|| jd
< |d ur‘|| jd< |d urš|| jd< | jd | _| jdkr²| jdkr²td | j¡ƒ‚| jd | _| jd | _| jd | _| jd | _| j| j | _| jd | _| jd | _| jd | _| jd | _| jd | _| jd | _| jd	 | _| jd | _| jd
 | _| jd | _| jd | _ | jd | _!| jd | _"| jd | _#t$ %d | j¡¡ t&j' (| j¡s<t& )| j¡ t*| j| j| j!| j"| j#d| _+| jrb| jdkrbt&j' | jd¡}t,|ƒ| _-| jdkryd| _.d| j d< d| _/d| j d< n5| jdkr®d | _.| j| _| j| _d | j d< | jr¡d| _/d| j d< t$ %d!¡ nd"| _/d"| j d< t$ %d#¡ t$ %| j ¡ t0| j | j| jd$ 	¡ | _1tj2j3| j1 4¡ d%d&d'| _5tj	j6 7d(¡| _8tj2j9j:| j5| jd) | jd* d+ | jd, d- gd.d/| _;t<ƒ | _=|  >d0¡ d S )1NzGPU is requiredÚ	data_typeÚuse_maskÚmax_stepÚtrain_num_raysÚmax_train_num_raysÚlog_every_n_stepsÚwork_dirÚrender_imagesÚ	save_ckptÚframe_countÚuse_distortionÚblenderÚcolmapz%data type {} is not support currentlyÚnum_samples_per_rayÚtest_ray_chunkÚdynamic_ray_samplingÚmax_sizeÚn_test_traj_stepsÚ	save_meshÚnetwork_cfgÚ
match_typez	params:{})r   r   r(   r   r   z
matting.pb)é   r)   g      ø?ZradiusÚwhiteÚ
backgroundg      à?zrun nerf with mask dataÚrandomzrun nerf without mask data)r!   r"   g{®Gáz„?gVçž¯Ò<)ÚlrÚepsi   é   é   é   é	   é
   g…ëQ¸Õ?)Z
milestonesÚgammaé*   )?Zget_or_download_model_dirZ	model_dirÚospÚjoinr   ZCONFIGURATIONÚsuperÚ__init__ÚtorchÚcudaZis_availableÚ	ExceptionÚparamsÚ_override_params_from_fileÚitemsr   Úformatr   r   r   r!   Útrain_num_samplesr   r"   r#   r$   r%   r   r   r&   r   r   r'   r(   r   r   ÚloggerÚinfoÚosÚpathÚexistsÚmakedirsr   Úpreprocessorr
   Ú	segmenterÚimg_whr+   r	   r   ZoptimZAdamÚ
parametersÚ	optimizerÚampZ
GradScalerÚgrad_scalerZlr_schedulerZMultiStepLRÚ	schedulerr   Ú
criterionsÚset_random_seed)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   ÚargsÚkwargsÚkeyÚvalueZsegment_path©Ú	__class__© ún/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/cv/nerf_recon_acc_trainer.pyr9   =   sØ   












ÿû





ýýÿþúzNeRFReconAccTrainer.__init__c                 C   s$   t  |¡ tj  |¡ t |¡ d S ©N)r,   ÚseedÚnpr:   Zmanual_seed)rR   r\   rY   rY   rZ   rQ   Â   s   
z#NeRFReconAccTrainer.set_random_seedc                 C   s€  | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d	 | jd	< | j d d
 | jd
< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< | j d d | jd< d S )NÚtrainr   r   r   r   r   r#   r   r   r   r   r   r!   r"   r$   r%   r&   r'   rH   r(   r   r   )Úcfgr=   )rR   rY   rY   rZ   r>   Ç   s>   
ÿ
ÿ
ÿ
ÿ
ÿÿz.NeRFReconAccTrainer._override_params_from_filec                 O   s„  t  d¡ i }| jdkr d|vrtdƒ‚|d }||d< d|d< | jdkrod|v r7|d }| j|d< ||d< n8d|v rk|d }tj |d¡}tj |¡rgt	 	d	 
|¡¡}t|ƒd
kr^tdƒ‚||d< d|d< ntdƒ‚tdƒ‚|  |¡}|d }t  d 
|¡¡ | jdkrí| jrítj tj |d¡¡r¢tj |d¡}	tj |d¡}
ntj |d¡}	tj |d¡}
tj|
dd t	 	d	 
|	¡¡t	 	d 
|	¡¡ }|D ]}t |¡}| j |¡}tj |
tj |¡¡}t ||¡ qÉt  d¡ | jdkrt|d| j| j| jd| _t|d| j| jd| _n$| jdkr0t|d| j| j| j| jd| _t|d| j| j| j| jd| _d
}t ¡ }|| jk rt t| jƒƒD ]¹}| j! "¡  | j| }| j! #|¡ |d  $¡ }|d  $¡ }|  !|¡}|d d
kroqCd }| j%r•| j&t'|d ƒ }t(| j| ƒ}t)t(| jd! |d"  ƒ| j*ƒ| _| j +| j¡ t, -|d# |d$  ||d$  ¡}||7 }|  .|d# |¡}| j/ 0¡  | j1 2|¡ 3¡  | j/ 4¡  | j5 4¡  || j6 d
kr÷t ¡ | }t  d%|d&›d'|› d(|d)›d*| jd+›d,|d)›d-¡ |d.7 }qC|| jk s<| j7r(tj | jd/¡}t8 9| j| j! :¡ | j/ :¡ d0œ|¡ t  d1 
|¡¡ | j;r;tj | jd2¡}|  <| j|¡ t  d3¡ d S )4Nz"Begin nerf reconstruction trainingr   Údata_dirz.Please specify data_dir of nerf_synthetic dataÚ Úvideo_input_pathr    Zimagesz{}/*.*gr   zno images found in images dirz images dir not found in data_dirz;Please specify video_path or images path for colmap processz3nerf reconstruction preprocess done, data_dir is {}Ú
preprocesszpreprocess/imageszpreprocess/masksZmasksT©Úexist_okz{}/*.*Gzsegment images done!r^   )Úroot_fpÚsplitrJ   Únum_raysÚcolor_bkgd_augÚtest)rf   rg   rJ   rh   )rf   rg   rJ   r$   rh   ri   )rf   rg   rJ   r$   rh   r%   ÚraysÚpixelsZnum_samplesg        gÍÌÌÌÌÌì?gš™™™™™¹?Úcomp_rgbZ
rays_validzelapsed_time=z.2fz	s | step=z | loss=z.4fz | train/num_rays=Údz |PSNR=ú é   z
model.ckpt)Zglobal_stepZnetwork_state_dictZoptimizer_state_dictz"save checkpoints done, saved as {}z
render.mp4zNeRF reconstruction finish)=rB   rC   r   r<   r   rD   rE   r7   rF   Úglobr@   ÚlenrH   r   rG   Úcv2ÚimreadrI   Zrun_maskÚbasenameÚimwriter   rJ   r   r+   Ztrain_datasetÚtest_datasetr   r$   r%   Útimer   Úranger   r^   Zupdate_stepr;   r#   rA   ÚsumÚintÚminr   Zupdate_num_raysÚFZsmooth_l1_lossrP   rL   Z	zero_gradrN   ÚscaleZbackwardÚsteprO   r   r   r:   ÚsaveZ
state_dictr   Úrender_video)rR   rS   rT   Zprocessor_inputr`   rb   Z
images_dirZ
image_listZprocessor_outputZ	image_dirZsave_mask_dirZimg_listZimg_pathÚimgÚmaskZoutpathr   ZticÚiÚdatark   rl   ÚoutZlossÚtempr   Zloss_rgbÚpsnrZelapsed_timeZsave_ckpt_nameÚsave_video_pathrY   rY   rZ   r^   å   s  

ÿ



ÿ
ÿÿÿ

ÿ
û
üú	ú	


þ
ÿ


ÿþýÿØ*ýûÿzNeRFReconAccTrainer.trainÚcheckpoint_pathÚreturnc                 O   s   t dƒ‚)aÍ  evaluate a dataset

        evaluate a dataset via a specific model from the `checkpoint_path` path, if the `checkpoint_path`
        does not exist, read from the config file.

        Args:
            checkpoint_path (Optional[str], optional): the model path. Defaults to None.

        Returns:
            Dict[str, float]: the results about the evaluation
            Example:
            {"accuracy": 0.5091743119266054, "f1": 0.673780487804878}
        z#evaluate is not supported currently)ÚNotImplementedError)rR   rŠ   rS   rT   rY   rY   rZ   Úevaluate}  s   zNeRFReconAccTrainer.evaluatec                 C   s‚  | j  ¡  t ¡ ® d}t tt| jƒƒ¡D ]S}| j| }|d  ¡ }|d  ¡ }|d }| j  	|¡}	||  
|	d |¡7 }|\}
}|	d  ||
d¡}tj |d¡}tj|dd	 tj ||d
›d¡}|  ||¡ q|  ||¡ t d |t| jƒ ¡¡ t d |¡¡ | jr¯| j  ¡ }tj |d¡}|  ||d |d ¡ t d |¡¡ W d   ƒ d S W d   ƒ d S 1 sºw   Y  d S )Nr   rk   rl   Úimage_whrm   r0   ÚrenderTrd   rn   z.pngztest psnr: {}z#save render video done. saved as {}z
render.objÚv_posÚ	t_pos_idxz"save render mesh done. saved as {})r   Úevalr:   Zno_gradÚtqdmry   rr   rw   r;   Z	inferencerP   ÚviewrD   rE   r7   rG   Ú
save_imageÚ
save_videorB   rC   r@   r&   Z
isosurfaceÚsave_obj)rR   Úsave_dirr‰   rˆ   r„   r…   rk   rl   rŽ   r†   ÚWÚHr‚   Zsave_img_dirZsave_img_pathZmeshZsave_mesh_pathrY   rY   rZ   r     s>   


ÿ
ÿä"èz NeRFReconAccTrainer.render_videoc                 C   s\   |  dd¡ ¡  ¡ }|d  tj¡}t |tj¡}t	j
 |¡}t	j|dd t ||¡ d S )Nr   rp   g     ào@Trd   )ZclipÚcpuÚnumpyZastyper]   Zuint8rs   ZcvtColorZCOLOR_RGB2BGRrD   rE   ÚdirnamerG   rv   )rR   Úfilenamer‚   r˜   rY   rY   rZ   r•   ±  s   zNeRFReconAccTrainer.save_imageé   c                 C   sz   t   d |¡¡}t|dd„ d}dd„ |D ƒ}|d j\}}}t |tjdŽ |||fd	¡}	|D ]}
|	 |
¡ q/|	 ¡  d S )
Nz{}/*.pngc                 S   s   t tj | ¡d d… ƒS )Néüÿÿÿ)r{   rD   rE   ru   )ÚfrY   rY   rZ   Ú<lambda>¼  s    z0NeRFReconAccTrainer.save_video.<locals>.<lambda>)rU   c                 S   s   g | ]}t  |¡‘qS rY   )rs   rt   )Ú.0r¡   rY   rY   rZ   Ú
<listcomp>½  s    z2NeRFReconAccTrainer.save_video.<locals>.<listcomp>r   Zmp4vT)	rq   r@   ÚsortedÚshapers   ZVideoWriterZVideoWriter_fourccÚwriteÚrelease)rR   rž   Zimg_dirZfpsZ	img_pathsZimgsrš   r™   Ú_Úwriterr‚   rY   rY   rZ   r–   ¹  s   ÿ
ÿzNeRFReconAccTrainer.save_videoc           
   
   C   s  t |dƒy}|D ]}| d |d |d |d ¡¡ q|d ur=t|ƒt|ƒks)J ‚|D ]}| d |d d|d  ¡¡ q+tt|ƒƒD ]2}| d¡ td	ƒD ]!}	| d
t|| |	 d ƒ|d u rbdn	t|| |	 d ƒf ¡ qN| d¡ qCW d   ƒ d S 1 sw   Y  d S )NÚwzv {} {} {} 
r   rp   r/   z
vt {} {} 
g      ð?zf r0   z %s/%sra   Ú
)Úopenr§   r@   rr   ry   Ústr)
rR   rž   r   r‘   Úv_texÚ	t_tex_idxr¡   Úvr„   ÚjrY   rY   rZ   Ú	write_objÆ  s(   " 
ÿÿÿù"÷zNeRFReconAccTrainer.write_objc                 C   s†   |  ¡  ¡ }|  ¡  ¡ }tj |¡}tj|dd |d ur8|d ur8|  ¡  ¡ }|  ¡  ¡ }|  |||||¡ d S |  |||||¡ d S )NTrd   )r›   rœ   rD   rE   r   rG   r³   )rR   rž   r   r‘   r¯   r°   r˜   rY   rY   rZ   r—   Ù  s   zNeRFReconAccTrainer.save_obj)NNNNNNNNNNNNr[   )rŸ   )NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r®   r9   rQ   r>   r^   r   r   Úfloatr   r   r•   r–   r³   r—   Ú__classcell__rY   rY   rW   rZ   r   !   sF    óÿþ  ÿÿ

ý!
r   )-rq   rD   Zos.pathrE   r6   r,   rx   r   Útypingr   r   rs   rœ   r]   r:   Ztorch.nn.functionalÚnnZ
functionalr}   r“   Zmodelscope.metainfor   Z#modelscope.models.cv.nerf_recon_accr   Z;modelscope.models.cv.nerf_recon_acc.dataloader.nerf_datasetr   r   Z0modelscope.models.cv.nerf_recon_acc.network.nerfr	   Z5modelscope.models.cv.nerf_recon_acc.network.segmenterr
   Z1modelscope.models.cv.nerf_recon_acc.network.utilsr   Zmodelscope.trainers.baser   Zmodelscope.trainers.builderr   Zmodelscope.utils.constantr   Zmodelscope.utils.loggerr   rB   Zregister_moduleZnerf_recon_accr   rY   rY   rY   rZ   Ú<module>   s2   