o
    *j!                     @   s.  d dl Z d dlZd dlmZ d dlmZmZ d dlZd dl	Z
d dlZd dlmZ d dlm  mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lm Z  d d
l!m"Z"m#Z# d dl$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+m,Z, e% Z-ej.e#j/ej0dG dd deZ1dS )    N)AnyDict)Image)shot_detector)tqdm)Models)
TorchModel)MODELS)Config)	ModelFileTasks)
get_logger   )get_contextual_relation_networkget_shot_encoder)get_pred_boundary
pred2scenescene2video)module_namec                       s   e Zd Zdef fddZdeeef deeejf fddZ	dd	 Z
d
d Zdd ZdejdejfddZdeeef fddZdd Zdd Z  ZS )MovieSceneSegmentationModel	model_dirc              	      sN  t  j|g|R i | t|tj}tj|dd}t|tj}t	
|| _dd }t| j| _|d| j| t| j| _|d| j| | jjjj}| jjjj| d }	t|	d| _|d	| j| t | _| jjdi | jjj ttjd
tjdt dt! tj"g dg ddg| _#| jj$j%j}
| jj$j%j|
 j&| _&d| _'dS )zstr -- model file root.cpu)Zmap_locationc                 S   s@   |  }| D ]\}}| d | }|| ||< q|| d S )N.)Z
state_dictitemsZload_state_dict)prefixmodelZ
src_paramsZ	own_statenameparamZsrc_name r   t/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/cv/movie_scene_segmentation/model.pyload_param_with_prefix-   s
   zDMovieSceneSegmentationModel.__init__.<locals>.load_param_with_prefixshot_encodercrnZhidden_size   head_sbd   )sizeinterpolation   )g
ףp=
?gv/?gCl?)gZd;O?gy&1?g?)meanZstdgh㈵>Nr   )(super__init__ospjoinr   ZTORCH_MODEL_FILEtorchloadZCONFIGURATIONr
   	from_filecfgr   r!   r   r"   r   Zcontextual_relation_networkr   paramsnnZLinearr$   r   initpreprocessorshot_detectTFZComposeZResizer   ZBICUBICZ
CenterCropZToTensorZ	Normalizetest_transformdatasetsampling_methodneighbor_sizeeps)selfr   argskwargsZ
model_pathr2   Zconfig_pathr    Zcrn_nameZhdimr:   	__class__r   r   r+   #   sB   

z$MovieSceneSegmentationModel.__init__inputsreturnc                 C   s   | d}|d }| |}tj| | dd}|dk}|dk}d\}}	|t||	  | | | j  }
|	t||	  | | | j  }|
| }||  }tj	|dd}t
||d	}|S )
NZvideolabelnone)Z	reductionr   r   )r   r   dim)predloss)popshared_stepFZcross_entropyZsqueezefloatsumr<   r.   Zargmaxdict)r=   rB   datalabelsoutputsrI   lposZlnegppr3   ZwpZwnwZprobsrer   r   r   forwardQ   s    

""z#MovieSceneSegmentationModel.forwardc                    s  t d | jjj}| jjj}|d  |d }t }t	
|| }g }i }| j  tt|D ]j}	|	| }
|	d | |k rE|	d | n|}||
| }|d d }|d d } fddt||d D }| j|||}| |||}t||}| |}tj|dd	}||d d df     q1|d
t|i |dt|i t|d
 |ksJ | j  |S )NzBegin scene detect ......shot_timecode_lstshot_idx_lstr   r   c                    s   i | ]}| | qS r   r   ).0irX   r   r   
<dictcomp>}   s    z9MovieSceneSegmentationModel.inference.<locals>.<dictcomp>rF   rH   Zsid)loggerinfor1   pipelineZbatch_size_per_gpur"   Zattention_maskdevicelenmathceilr   startr   rangeZget_frame_imgget_batch_inputr.   stacktorK   rL   Zsoftmaxextendr   detachnumpyupdatenparangerelease)r=   batchbsrb   rY   shot_numZcntZ
infer_predZinfer_resultr\   rf   endZbatch_shot_idx_lstshot_start_idxZshot_end_idxZbatch_timecode_lstZbatch_shot_keyf_lstrB   Zinput_rR   Zprobr   r]   r   	inferencef   sD   



 

$
z%MovieSceneSegmentationModel.inferencec                 C   sd   t   | |}t|jdksJ W d    n1 sw   Y  | j|d d\}}| |}|S )N   )mask)r.   Zno_gradextract_shot_representationrc   shaper"   r$   )r=   rB   	shot_repr_ZpooledrH   r   r   r   rK      s   


z'MovieSceneSegmentationModel.shared_stepc                 C   sr   |    }| jjjd }t| t|j	d D ]}dt
|d d}t||}t|||  qd S )Nz	/featuresr   Zshot_   z.npy)rM   r   rm   r1   r9   Zimg_pathosmakedirsrg   r{   strzfillr,   r-   ro   save)r=   _reprfeatpthidxr   r   r   r   save_shot_feat   s   
z*MovieSceneSegmentationModel.save_shot_featc           
         sr   t  jdks	J  j\}}}}}}tj d|d  fddt|D }t|jdd}	tj|	d|d}	|	S )	z& inputs [b s k c h w] -> output [b d]    zb s k c h w -> (b s) k c h w)sc                    s"   g | ]}  d d |f qS )N)r!   )r[   Z_krB   r=   r   r   
<listcomp>   s   " zKMovieSceneSegmentationModel.extract_shot_representation.<locals>.<listcomp>r   rF   z(b s) d -> b s d)rc   r{   einopsZ	rearrangerg   r.   ri   r)   )
r=   rB   br   kchrU   Zkeyframe_reprr|   r   r   r   rz      s   z7MovieSceneSegmentationModel.extract_shot_representationc                 K   sx   t d |d }|d }| jjj}t||}t||\}}}	}
| jjjr4t|d ||}t	d|  t
|||	|
fS )NzGenerate scene .......r   	shot2keyfZinput_video_pthzSplit scene video saved to )r_   r`   r1   ra   Zsave_thresholdr   r   Zsave_split_scener   printrc   )r=   rB   r?   Z	pred_dictr   ZthresZ	anno_dictZscene_dict_lstZ
scene_listrt   Zshot_dict_lstZre_dirr   r   r   postprocess   s   



z'MovieSceneSegmentationModel.postprocessc                    s   g }t |D ]\}} fdd|D }tj|dd}|| qtj|dd}g }t |D ]\}}|| }	||	 }
||
 q-|S )Nc                    s   g | ]}  |qS r   )r8   )r[   Z	one_framer=   r   r   r      s    
z?MovieSceneSegmentationModel.get_batch_input.<locals>.<listcomp>r   rF   )	enumerater.   ri   append)r=   Zshot_keyf_lstrv   rY   Zsingle_shot_featr   one_shotZ	shot_featshot_idxZ	shot_idx_	_one_shotr   r   r   rh      s   
z+MovieSceneSegmentationModel.get_batch_inputc           	      C   s   t d | jj|fi | jjj\}}}t d g }t|D ]%\}}t|d t	| j
 | j
d  }t|d|d d }|| q ||||fS )NzBegin shot detect......zShot detect done!Zshot_idr   r   Znum_shot)r_   r`   r   r6   r1   r5   r   intro   rp   r;   Zclipr   )	r=   rB   rX   annor   rY   r   r   r   r   r   r   
preprocess   s   

z&MovieSceneSegmentationModel.preprocess)__name__
__module____qualname__r   r+   r   r   r.   ZTensorrW   rw   rK   r   rz   r   rh   r   __classcell__r   r   r@   r   r      s    $.,

r   )2rd   r   Zos.pathpathr,   typingr   r   r   rm   ro   r.   Ztorch.nnr3   Ztorch.nn.functionalZ
functionalrL   Ztorchvision.transformsZ
transformsr7   ZPILr   Zshotdetect_scenedetect_lgssr   r   Zmodelscope.metainfor   Z'modelscope.models.base.base_torch_modelr   Zmodelscope.models.builderr	   Zmodelscope.utils.configr
   Zmodelscope.utils.constantr   r   Zmodelscope.utils.loggerr   Z	get_modelr   r   Zutils.save_opr   r   r   r_   Zregister_moduleZmovie_scene_segmentationZresnet50_bertr   r   r   r   r   <module>   s4   