o
    *jY                     @   s`  d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlmZ	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lm Z m!Z! d dl"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4m5Z5 e(j6ej7dG dd de&Z8G dd dZ9dS )    N)Dict)EasyDict)DistributedDataParallel)Trainers)	Evaluater)	inference)FeatureLoss)build_ddp_modelbuild_local_model)cosine_scheduler	ema_model)build_dataloaderbuild_dataset)BaseTrainer)TRAINERS)save_checkpoint)DEFAULT_MODEL_REVISION	ModelFile
ThirdParty)
get_logger)MeterBuffer)get_ranksynchronize)module_namec                       st   e Zd Zddddefdededededef
 fdd	Zd
d Zdd Z	ddede	ee
f fddZdd Z  ZS )ImageDetectionDamoyoloTrainerNTmodelcfg_fileload_pretrain
cache_pathmodel_revisionc           
         s   |dur+| tj}|dur|tj | |||| _|du r*tj| jt	j
| _n|dur3|dus7J d|durE|| _|durE|| _t | j | j}	tj| j|	jjj|	jj_|rwd|v rj|d |	j_ntj| j| jjj|	j_d| jv r| |	}	d|v r|d |	j_d|v r|d |	j_d|v r|d |	j_d|v r|d |	j_d	|v r|d	 |	j_d
|v r|d
 |	j_d|v r|d |	j_d|v r|d |	jj_d|v r|d |	j_ d|v r|d |	j!_"d|v r|d |	j!_#|	jj| _t$| j| _%|	| _dS )a   High-level finetune api for Damoyolo.

        Args:
            model: Model id of modelscope models.
            cfg_file: Path to configuration file.
            load_pretrain: Whether load pretrain model for finetune.
                if False, means training from scratch.
            cache_path: cache path of model files.
            model_revision: the git version of model on modelhub.
            gpu_ids: the id list of gpu.
            batch_size: total batch size.
            max_epochs: maximum number of training epoch.
            train_image_dir: the directory of training image.
            val_image_dir: the directory of validation image.
            train_ann: the path of train set annotation file.
            val_ann: the path of val set annotation file.
            num_classes: class number.
            base_lr_per_img: learning rate per image.
                The final learning rate is base_lr_per_img*batch_size.
            pretrain_model: the path of pretrained model.
            work_dir: the directory of work folder.
            exp_name: the name of experiment.
            third_party: in which third party library this function is called.
        Nz;cfg_file and cache_path is needed, if model is not providedZpretrain_modelZ	frameworkgpu_ids
batch_size
max_epochstrain_image_dirval_image_dir	train_annval_annnum_classesbase_lr_per_imgwork_direxp_name)&getr   ZKEYpopZget_or_download_model_dirr   ospathjoinr   ZCONFIGURATIONr   super__init__cfgr   ZbackboneZstructure_filetrainfinetune_pathweights_config_transformr    r!   total_epochsdatasetr#   r$   r%   r&   headr'   r(   miscs
output_dirr*   len
world_size)
selfr   r   r   r   r   argskwargsZthird_partyr2   	__class__ x/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/trainers/cv/image_detection_damoyolo_trainer.pyr1   )   sr    






z&ImageDetectionDamoyoloTrainer.__init__c                 C   s8   t j| tjdd||d t|d d }|| d S )NZncclztcp://127.0.0.1:12344)Zinit_methodrankr=   )torchcudaZ
set_devicedistZinit_process_groupDamoyoloTrainerr3   )r>   
local_rankr=   r2   trainerrC   rC   rD   _train   s   z$ImageDetectionDamoyoloTrainer._trainc                 C   sR   t | jjjdkrtj| j| j| j| jfdd d S t| jd d }|jdd d S )N   T)Znprocsr?   r/   r   rJ   )	r<   r2   r3   r    mpZspawnrL   r=   rI   )r>   rK   rC   rC   rD   r3      s   

z#ImageDetectionDamoyoloTrainer.traincheckpoint_pathreturnc                 O   s(   |d ur	|| j j_t| j }|  d S N)r2   testrP   r   evaluate)r>   rP   r?   r@   Z	evaluaterrC   rC   rD   rT      s   

z&ImageDetectionDamoyoloTrainer.evaluatec                 C   s  t i }|jj|_|jjj|j_|jj|j_|j|_|j	|_	|j|_|j
|_|jj|j_|jj
|j_|jjj|j_|jjj|j_|jjj|j_t|jj|jjj |j_|jjj|jj |j_|jjj|j_|jjj|j_|jj|j_|d d= |d d= |d d= |d d= |S )Nr3   r:   lr_scheduler	optimizer
dataloader)easydictr3   r:   rW   Zworkers_per_gpunum_workersr)   r;   r   r8   Z
evaluationrS   ZpreprocessoraugmentrU   warmup_start_lrmin_lr_ratiowarmup_epochsr<   r    Zbatch_size_per_gpur!   rV   lrr(   momentumweight_decayr"   r7   )r>   configZ
new_configrC   rC   rD   r6      s6   




z/ImageDetectionDamoyoloTrainer._config_transformrR   )__name__
__module____qualname__r   strboolr1   rL   r3   r   floatrT   r6   __classcell__rC   rC   rA   rD   r   &   s4    [


	r   c                   @   sT   e Zd ZdddZdddZdd Zd	d
 Zdd ZdddZdddZ	dd Z
dS )rI   Nc                 C   s   || _ || _|| _|jj| _|jj| _d| _t| j jj	dkr#d| _
nd| _
t|jjd| _tj|jj|jj| _t dkrGtj| jdd ttj| jd| _| jd	| j | jd
| j  d S )NrG   rM   TF)Zwindow_sizer   exist_okztrain_log.txtzargs info: {}zcfg value:
{})r2   tea_cfgr?   r:   r;   r*   devicer<   r3   r    distributedr   print_interval_itersmeterr-   r.   r/   	file_namer   makedirsr   loggerinfoformat)r>   r2   r?   rk   rC   rC   rD   r1      s    


zDamoyoloTrainer.__init__Fc                 C   s   t ||jj|jjd|jjjd}t ||jj|jjdd}t	
t|d |jj }t||jj|jj| j|jj|jjdd|d	}t||jj|jj|jjdd|d}|||fS )	NT)is_trainmosaic_mixupF)ru   r       )r!   start_epochr7   rY   ru   size_divrm   )r!   rY   ru   ry   rm   )r   r8   r#   r%   r3   rZ   rv   r$   r&   mathceilr<   r!   r   rx   r7   r:   rY   rS   )r>   r2   rm   Ztrain_datasetZval_datasetiters_per_epochtrain_loader
val_loaderrC   rC   rD   get_data_loader   sN   

	zDamoyoloTrainer.get_data_loaderc	           	      C   sl   || _ || _|| _ || | _|| | _|| | _|| | _| j| j| j k| _|| | _|| | _|| _	d S rR   )
r|   r7   
start_itertotal_iterswarmup_itersno_aug_itersZno_augeval_interval_itersckpt_interval_itersrn   )	r>   r|   rx   r7   r]   no_aug_epochseval_interval_epochsckpt_interval_epochsrn   rC   rC   rD   setup_iters  s   






zDamoyoloTrainer.setup_itersc           	      C   sP  g g g }}}| j  D ]9\}}t|dr#t|jtjr#||j t|tjs-d|v r4||j	 qt|drFt|j	tjrF||j	 q| j
r| j D ]9\}}t|dret|jtjre||j t|tjsod|v rv||j	 qOt|drt|j	tjr||j	 qOtjj|d|dd}|||d |d|i || _| jS )	NbiasZbnweightgMbP?T)r^   r_   Znesterov)paramsr`   r   )r   Znamed_moduleshasattr
isinstancer   nn	ParameterappendZBatchNorm2dr   distillfeature_lossrF   ZoptimZSGDZadd_param_grouprV   )	r>   r_   r`   Zbn_groupZweight_groupZ
bias_groupkvrV   rC   rC   rD   build_optimizer  sB   zDamoyoloTrainer.build_optimizerc              
      sb  t  j j _ jrtj j _ jd ured _	d _
t  j j _ j  tjtj jdd}d|v rD jj|d dd nd|v rR jj|d dd t jjj jjjdd j _nd	 _	d  _
  jjj jjj _ jjjd ur jd
 jjj   j jjj d _ d _!n. jjj"d ur j# jjj"dd}| _ | _! jd$ j  nd _ d _! jd  jjj%r߈ jd t& j jjj' _&nd  _& ( j j\ _) _*} +| j! jjj, jjj- jjj. jj/j0 jj/j1 jj/j2 t3 jjj4 jjj5 jjj6 j7 j8 j9 jjj: _;d jjj<v  _= jr<t> j| _n jd _ jd  j  t?? }t?? }t@ j)D ]\}\}}	}
 jA| } j;B|} jjCD ]}||d< qr| j} fdd|	D }	t?? } j	r j||	dd\}}|d }tD   j||	dd}W d    n	1 sw   Y  dtEF|tEjG tH j)  d d d }| || }||7 }||d< n
 ||	}|d } jI  |J   j
d urtjKjL jM  j
dd  jN   j&d ur j&O| j |}t?? }dd |P D } jQjOd3|| || |d | |d  j7 j8 krb j=rb jd!  j)jRSd	  jT _U jT _Vd	 _=|d  j2 dkr j7|d  } jQd" jW| }d#$tXjYtZ|d$}d%$ j d  j,|d  jT  jT} jQ[d&}d'\d(d |P D } jQ[d)}d'\d*d |P D } jd+$||| jQd j]d,$|j^j_d |j^j_d- |   jQ`  |d  jV dkr jad. j d  |d/ |d  jU dkrt?bd0  c| jjdje  j  tf  |d  jT dkr& j d  _ q[ jad1|d2 d S )4NT   Zmap_locationZweights_onlyr   )strict
state_dictcwd)Z	distillerFzfinetune from r   )Zneed_optimizerzResume Training from Epoch: {}zStart Training...z8Enable ema model! Ema model will be evaluated and saved.rv   rG   zTraining start...r^   c                    s   g | ]}|  jqS rC   )torl   ).0targetr>   rC   rD   
<listcomp>  s    z)DamoyoloTrainer.train.<locals>.<listcomp>)ZstuZ
total_loss)tearM      gdistill_loss)Zmax_normZ	norm_typec                 S   s   i | ]	\}}||  qS rC   )item)r   _name_vrC   rC   rD   
<dictcomp>  s    z)DamoyoloTrainer.train.<locals>.<dictcomp>)	iter_timeZ
model_timer^   z--->turn OFF mosaic aug now!r   zETA: {})secondszepoch: {}/{}, iter: {}/{}lossz, c                 S      g | ]\}}d  ||jqS )z
{}: {:.1f}rt   Zavgr   r   r   rC   rC   rD   r         timec                 S   r   )z{}: {:.3f}sr   r   rC   rC   rD   r     r   z{}, {}, {}, lr: {:.3e}z, size: ({:d}, {:d}), {}   zepoch_%d_ckpt.pthrN   g~jth?zlatest_ckpt.pth)	ckpt_namerJ   rC   )gr
   r2   rl   r   rm   r   ZSyncBatchNormZconvert_sync_batchnormrk   r   Z	grad_clipZ	tea_modelevalrF   loadr?   tea_ckptload_state_dictr   ZneckZout_channelsr   r   r   r3   r_   r`   rV   r4   rr   rs   Zload_pretrain_detectorepochrx   resume_pathresume_modelrt   Zemar   Zema_momentumr   r}   r~   r   r7   r]   r   r:   r   r   rn   r   r(   r!   r\   r   r   r   r[   rU   rZ   rv   r	   r   	enumerater   Zget_lrZparam_groupsZno_gradrz   cospir<   Z	zero_gradZbackwardutilsZclip_grad_norm_
parametersstepupdateitemsro   Zbatch_samplerZ
set_mosaicr|   r   r   Z
global_avgdatetime	timedeltaintZget_filtered_meterr/   ZlatestZtensorsshapeZclear_meters	save_ckptsleeprT   r8   r&   r   )r>   rJ   r   resume_epochZitersZiter_start_timeZiter_end_timeZ	data_iterZinpstargetsZidsZcur_iterr^   Zparam_groupZmodel_start_timeZoutputsZfpn_outsr   Zfpn_outs_teaZdistill_weightr   Zoutputs_arrayZ
left_itersZeta_secondsZeta_strZprogress_strZ
loss_meterZloss_strZ
time_meterZtime_strrC   r   rD   r3   9  s^  



















zDamoyoloTrainer.trainc                 C   s   |dkrL| j d ur| j j}nt| jtr| jj}n| j}tj| j|}| j	
d| d| jd i}| jr@|j| j d t||| j|dd d S d S )Nr   zSave weights to {}r   rM   )r   T)r   filenamerV   metaZ	with_meta)r   r   r   DDPmoduler-   r.   r/   rp   rr   rs   rt   r   r   r   r   r   r   rV   )r>   r   rJ   Zupdate_best_ckptZ
save_modelr   rC   rC   rD   r     s&   



zDamoyoloTrainer.save_ckptc                 C   s   |}t j|| jdd}d|v r| j|d  nd|v r$| j|d  |rPd|v r2| j|d  | jrPd|v rD| j|d d  nd|v rP| j|d  d|v r\|d d }|S d|v rd|d }|S )	NTr   r   r   rV   r   r   r   )rF   r   rl   r   r   rV   r   r   )r>   r   Zload_optimizerZckpt_file_pathZckptr   rC   rC   rD   r     s0   
zDamoyoloTrainer.resume_modelc                 C   sv   | j d ur
| j j}n| j}t|tr|j}tj| j| j	d}|dkr*tj
|dd | jD ]}t||| j|d q-d S )Nr   r   Tri   )rl   output_folder)r   r   r   r   r   r-   r.   r/   r;   r*   rq   r~   r   rl   )r>   rJ   r&   Z	evalmodelr   Zdata_loader_valrC   rC   rD   rT   )  s$   



zDamoyoloTrainer.evaluaterR   )F)rb   rc   rd   r1   r   r   r   r3   r   r   rT   rC   rC   rC   rD   rI      s    

(% 
D
rI   ):r   rz   r-   r   typingr   rF   Ztorch.distributedrm   rH   Ztorch.multiprocessingmultiprocessingrO   Ztorch.nnr   rX   r   Ztorch.nn.parallelr   r   Zmodelscope.metainfor   ZCmodelscope.models.cv.tinynas_detection.damo.apis.detector_evaluaterr   ZCmodelscope.models.cv.tinynas_detection.damo.apis.detector_inferencer   ZKmodelscope.models.cv.tinynas_detection.damo.base_models.losses.distill_lossr   Z>modelscope.models.cv.tinynas_detection.damo.detectors.detectorr	   r
   Z1modelscope.models.cv.tinynas_detection.damo.utilsr   r   Z:modelscope.msdatasets.dataset_cls.custom_datasets.damoyolor   r   Zmodelscope.trainers.baser   Zmodelscope.trainers.builderr   Zmodelscope.utils.checkpointr   Zmodelscope.utils.constantr   r   r   Zmodelscope.utils.loggerr   Zmodelscope.utils.metricr   Zmodelscope.utils.torch_utilsr   r   Zregister_moduleZtinynas_damoyolor   rI   rC   rC   rC   rD   <module>   s:    