o
    *j5                     @   sT  d Z ddlZddlZddlZddlmZ ddlm  mZ	 ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZ ddlmZ e ZG d	d
 d
ejZejejejdG dd deZG dd deZG dd dejZ G dd dejZ!G dd dejZ"G dd dejZ#G dd dejZ$G dd deZ%dddZ&dS ) z VideoInpaintingProcess
The implementation here is modified based on STTN,
 originally Apache 2.0 License and publicly available at https://github.com/researchmm/STTN
    N)Models)Model)
TorchModel)MODELS)	ModelFileTasks)
get_loggerc                       s.   e Zd Z fddZdd Zd	ddZ  ZS )
BaseNetworkc                    s   t t|   d S N)superr	   __init__)self	__class__ w/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/cv/video_inpainting/inpainting_model.pyr      s   zBaseNetwork.__init__c                 C   sN   t | tr	| d } d}|  D ]}|| 7 }qtdt| j|d f  d S )Nr   zoNetwork [%s] was created. Total number of parameters: %.1f million. To see the architecture, do print(network).i@B )
isinstancelist
parametersZnumelprinttype__name__)r   Z
num_paramsparamr   r   r   print_network   s   
zBaseNetwork.print_networknormal{Gz?c                    s@    fdd}|  | |  D ]}t|dr|  qdS )z
        initialize network's weights
        init_type: normal | xavier | kaiming | orthogonal
        https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/9451e70673400885567d08a9e97ade2524c700d0/models/networks.py#L39
        c                    s  | j j}|ddkr7t| dr| jd urtj| jjd t| dr3| j	d ur5tj| j	jd d S d S d S t| dr|ddksJ|ddkrd	krYtj
| jjd  nLd
krhtjj| jj d n=dkrwtjj| jjdd n.dkrtjj| jjddd ndkrtjj| jj d ndkr|   ntd t| dr| j	d urtj| j	jd d S d S d S d S d S )NZInstanceNorm2dweightg      ?biasg        ZConvZLinearr   Zxavier)gainZxavier_uniformZkaimingr   Zfan_in)amodeZ
orthogonalnonez-initialization method [%s] is not implemented)r   r   findhasattrr   nninitZ	constant_datar   Znormal_Zxavier_normal_Zxavier_uniform_Zkaiming_normal_Zorthogonal_Zreset_parametersNotImplementedError)m	classnamer   	init_typer   r   	init_func/   s>   
z+BaseNetwork.init_weights.<locals>.init_funcinit_weightsN)applychildrenr$   r.   )r   r,   r   r-   r)   r   r+   r   r.   (   s   

zBaseNetwork.init_weights)r   r   )r   
__module____qualname__r   r   r.   __classcell__r   r   r   r   r	      s    r	   )module_namec                       s   e Zd Zd fdd	Z  ZS )VideoInpaintingr   c                    s   t  j|||d| t | _tj rd}nd}tjd|t	j
|d}| j|d  | j  || _| jdkrUtj rU| jd| j td	| j d S d
| _td d S )N)	model_dir	device_idcudacpuz{}/{})Zmap_locationZnetGr   zcuda:{}zUse GPU: {}r   zUse CPU for inference)r   r   InpaintGeneratormodeltorchr8   Zis_availableloadformatr   ZTORCH_MODEL_BIN_FILEZload_state_dictevalr7   tologgerinfo)r   r6   r7   argskwargsZdeviceZpretrained_paramsr   r   r   r   V   s*   

zVideoInpainting.__init__)r   )r   r1   r2   r   r3   r   r   r   r   r5   R   s    r5   c                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	r:   Tc                    sR  t t|   d}d}g d}g }t|D ]}|t||d qtj| | _ttj	dddddd	tj
d
ddtj	dddddd	tj
d
ddtj	dddddd	tj
d
ddtj	d|dddd	tj
d
dd| _tt|ddddtj
d
ddtj	dddddd	tj
d
ddtdddddtj
d
ddtj	dddddd	| _|r|   d S d S )N      ))0      )      )rJ      )rK      )hidden   @   rL      kernel_sizestridepadding皙?TZinplace   rR   rT   )r   r:   r   rangeappendTransformerBlockr%   
SequentialtransformerConv2d	LeakyReLUencoderdeconvdecoderr.   )r   r.   ZchannelZ	stack_num	patchsizeblocks_r   r   r   r   n   s<   	zInpaintGenerator.__init__c                 C   s   |  \}}}}}||| d||}| ||| |||}|  \}	}}}tj|dd}| ||||dd }| |}
t|
}
|
S )NrP         ?scale_factorxr)   bcrj   )	sizeviewr`   Finterpolater]   rb   r<   tanh)r   Zmasked_framesmasksrk   trl   hwenc_featre   outputr   r   r   forward   s    

zInpaintGenerator.forwardc           	      C   sZ   |  \}}}}|||||}tj|dd}|  \}}}}| ||d|dd }|S )Nrf   rg   rP   ri   rj   )rm   rn   ro   rp   r]   )	r   featrr   rs   rl   rt   ru   re   rv   r   r   r   infer   s   zInpaintGenerator.inferT)r   r1   r2   r   rx   rz   r3   r   r   r   r   r:   l   s    !r:   c                       s*   e Zd Z		d fdd	Zdd Z  ZS )ra   rN   r   c                    s$   t    tj|||d|d| _d S )NrP   rQ   )r   r   r%   r^   conv)r   Zinput_channelZoutput_channelrR   rT   r   r   r   r      s   
zdeconv.__init__c                 C   s    t j|dddd}| |}|S )NrL   ZbilinearT)rh   r!   Zalign_corners)ro   rp   r|   r   rj   r   r   r   rx      s
   
zdeconv.forward)rN   r   r   r1   r2   r   rx   r3   r   r   r   r   ra      s
    ra   c                   @   s   e Zd ZdZdd ZdS )	Attentionz/
    Compute 'Scaled Dot Product Attention
    c                 C   sR   t ||ddt|d }||d tj|dd}t ||}||fS )Nr   g    edim)	r<   matmul	transposemathsqrtrm   Zmasked_fillro   Zsoftmax)r   querykeyvaluer)   ZscoresZp_attnZp_valr   r   r   rx      s   zAttention.forwardN)r   r1   r2   __doc__rx   r   r   r   r   r      s    r   c                       s(   e Zd ZdZ fddZdd Z  ZS )MultiHeadedAttentionz1
    Take in model size and number of heads.
    c                    s~   t    || _tj||ddd| _tj||ddd| _tj||ddd| _ttj||dddtj	ddd| _
t | _d S )NrP   r   rX   rN   rU   TrV   )r   r   rc   r%   r^   query_embeddingvalue_embeddingkey_embeddingr\   r_   output_linearr   	attention)r   rc   d_modelr   r   r   r      s    
zMultiHeadedAttention.__init__c              
   C   sp  |  \}}}}|| }	|t| j }
g }| |}| |}| |}t| jtj|t| jddtj|t| jddtj|t| jddD ]\\}}}}}|| || }}|	||	d||||}|
ddddddd 	||	| | || }|d	d
kdd|	| | d}|	||	|
||||}|
ddddddd 	||	| | |
| | }|	||	|
||||}|
ddddddd 	||	| | |
| | }|	||	|
||||}|
ddddddd 	||	| | |
| | }| ||||\}}|	||	|||
||}|
ddddddd 	||
||}|| qGt|d}| |}|S )NrP   r   r   rN      rL   rK   rF   r   g      ?)rm   lenrc   r   r   r   zipr<   chunkrn   Zpermute
contiguousmean	unsqueezerepeatr   rZ   catr   )r   rj   r)   rk   rl   Zbtre   rt   ru   rs   Zd_krw   Z_query_key_valuewidthheightr   r   r   Zout_wZout_hmmyr   r   r   rx      sp   





&
zMultiHeadedAttention.forwardr   r1   r2   r   r   rx   r3   r   r   r   r   r      s    r   c                       s$   e Zd Z fddZdd Z  ZS )FeedForwardc              
      sV   t t|   ttj||ddddtjdddtj||dddtjddd| _d S )	NrN   rL   )rR   rT   ZdilationrU   TrV   rP   rX   )r   r   r   r%   r\   r^   r_   r|   )r   r   r   r   r   r     s   
zFeedForward.__init__c                 C   s   |  |}|S r
   )r|   r}   r   r   r   rx     s   
zFeedForward.forwardr~   r   r   r   r   r     s    r   c                       s*   e Zd ZdZd fdd	Zdd Z  ZS )r[   zS
    Transformer = MultiHead_Attention + Feed_Forward with sublayer connection
    rW   c                    s&   t    t||d| _t|| _d S )N)r   )r   r   r   r   r   feed_forward)r   rc   rM   r   r   r   r   !  s   
zTransformerBlock.__init__c                 C   sT   |d |d |d |d f\}}}}||  |||| }|| | }||||dS )Nrj   r)   rk   rl   ri   )r   r   )r   rj   r)   rk   rl   r   r   r   rx   &  s   $zTransformerBlock.forward)rW   r   r   r   r   r   r[     s    r[   c                       s.   e Zd Z				d fdd	Zdd Z  ZS )	DiscriminatorrN   FTc                    s<  t t|   || _d}tttj||d ddd| d|tjdddttj|d |d	 ddd| d
|tjdddttj|d	 |d ddd| d
|tjdddttj|d |d ddd| d
|tjdddttj|d |d ddd| d
|tjdddtj|d |d dddd| _	|r| 
  d S d S )NrO   rP   )rN   r   r   )rP   rL   rL   )in_channelsZout_channelsrR   rS   rT   r   rU   TrV   rL   )rR   rS   rT   r   rK   rQ   )r   r   r   use_sigmoidr%   r\   spectral_normZConv3dr_   r|   r.   )r   r   r   Zuse_spectral_normr.   nfr   r   r   r   /  s   5zDiscriminator.__init__c                 C   sD   t |dd}|d}| |}| jrt |}t |dd}|S )Nr   rP   rL   )r<   r   r   r|   r   Zsigmoid)r   ZxsZxs_try   outr   r   r   rx   p  s   


zDiscriminator.forward)rN   FTTr~   r   r   r   r   r   -  s    Ar   Tc                 C   s   |rt | S | S r
   )Z_spectral_norm)moduler!   r   r   r   r   z  s   r   r{   )'r   r   numpynpr<   Ztorch.nnr%   Ztorch.nn.functionalZ
functionalro   Ztorchvision.modelsmodelsZmodelscope.metainfor   Zmodelscope.modelsr   Zmodelscope.models.baser   Zmodelscope.models.builderr   Zmodelscope.utils.constantr   r   Zmodelscope.utils.loggerr   rA   Moduler	   Zregister_moduleZvideo_inpaintingr5   r:   ra   r   r   r   r[   r   r   r   r   r   r   <module>   s6    :A<M