o *j@s.ddlZddlZddlZddlmZddlZddlZddlmZddl Z ddlm Z ddl m Z ddl mZddlmZdejd<d d gZdhd d ZddZddZddZddZdiddZddZdiddZddZdjdd Zdjd!d"Zd#d$Zd%d&Zd'd(Z did)d*Z!did+d,Z"d-d.Z#d/d0Z$d1ej%d2d2fd3d4Z&dkd5d6Z'did7d8Z(d9d:Z)e*d1d1ge*d;d;gdd?Z+d@dAZ,dBdCZ-dDdEZ.dhdFdGZ/dhdHdIZ0dhdJdKZ1dhdLdMZ2dldQdRZ3dmdSdTZ4dmdUdVZ5dndYdZZ6dnd[d\Z7d]d^Z8d_d`Z9dodbdcZ:dpddd Z;dmdedfZcCst|j}|tj|tjkr|d9}t|gdgdgdgdgd}|tjkr1|}n|d}||S)zVsame as matlab ycbcr2rgb Input: uint8, [0, 255] float, [0, 1] r)oݩr?r@r@)rg] +Yg}3?)gy?gN jr)gxkgOn`@gL7A`Mq)r6r7rrr!r9r#)rr<r=rrr ycbcr2rgbOs       rAcCr))zxbgr version of rgb2ycbcr only_y: only return Y channel Input: uint8, [0, 255] float, [0, 1] r)r,r+r*r-r1r0r.r2r5r:rrr bgr2ycbcrfr?rBcCsz|dkr|dkrdd|D}dd|DS|dkr,|dkr,dd|D}dd|DS|d kr;|d kr;d d|DS|S) Nr graycSg|] }t|tjqSr)rrZCOLOR_BGR2GRAY.0rrrr z#channel_convert..cSg|] }tj|ddqSr r rrrErrrrGrHycSsg|]}t|ddqS)T)r;)rBrErrrrGscSrIrJrKrErrrrGrHr RGBcSrDr)rrZCOLOR_GRAY2BGRrErrrrGrHr)Zin_cZtar_typeZimg_listZ gray_listZy_listrrrchannel_convertsrNcCs|j|jks td|jdd\}}|||||||f}|||||||f}|tj}|tj}t||d}|dkrLtdSdtdt |S)N+Input images must have the same dimensions.r rinfr) shape ValueErrorr7rfloat64meanfloatmathlog10sqrt)img1img2borderhwZmserrrcalculate_psnrs   r_c Cs|j|jks td|jdd\}}|||||||f}|||||||f}|jdkr9t||S|jdkr|jddkrog}tdD]}|t|dddd|f|dddd|fqKt|S|jddkrtt |t |SdStd)zMcalculate SSIM the same outputs as MATLAB's img1, img2: [0, 255] rONr r r zWrong input image dimensions.) rRrSrssimrangeappendrarrayrUsqueeze)rZr[r\r]r^Zssimsirrrcalculate_ssims"     6rfcCs:d}d}|tj}|tj}tdd}t||}t|d|ddddf}t|d|ddddf}|d}|d} ||} t|dd|ddddf|} t|dd|ddddf| } t||d|ddddf| } d| |d| ||| || | |}|S) Ng(\@gzGBM@ ?r ) r7rrTrZgetGaussianKernelouter transposeZfilter2DrU)rZr[ZC1ZC2kernelZwindowZmu1Zmu2Zmu1_sqZmu2_sqZmu1_mu2Z sigma1_sqZ sigma2_sqZsigma12Zssim_maprrrr`s,   &&&    r`cCsnt|}|d}|d}d|d|d|dk|d|d|d|d|dk|dk|S)Nr r rhg@r )torchabsZtype_as)xZabsxZabsx2Zabsx3rrrcubics  0rtcCs|dkr |r ||}td||}||ddd|}t||d}t|d} ||d|| td| d| d| || } ||d|| | } |dkrc|rc|t| |} nt| } t| d|d} | | || } t| dkd}tj |dddds| dd| d} | dd| d} tj |dddds| dd| d} | dd| d} | } | } | d}| |}| |d} | | t|t|fS)Nr ?r rgư>)Zrel_tolri)rqZlinspacefloorrWceilviewexpandrtsumisclosenarrow contiguousminmaxint)Z in_lengthZ out_lengthscalern kernel_width antialiasingrsuleftPindicesZdistance_to_centerweightsZ weights_sumZweights_zero_tmpZ sym_len_sZ sym_len_errrcalculate_weights_indicess<     rcCsR|dkrdnd}|r|d|\}}}|t||t||}}} d} d} t|||| | |\} } }}t|| || | |\}}}}t|||||}|d|| ||ddd|ddf}t |dddd }| d|}|dd| ||dd| dddf}t |dddd }| d|}|d||| |t|||}| d} t |D].}t| |d}t |D]}||||| ddfdd| ||||ddf<qqt|||||}|d|| ||ddddd|f}t |dddd }| d|}|dd| ||dddd| df}t |dddd }| d|}|d||| |t||| }|d} t | D],}t||d}t |D]}||dd||| f||||dd|f<qqs|r||S) Nr TFrrprtr ri)dim unsqueeze_sizerWrwrrq FloatTensorr|copy_arangelong index_selectrarrmmvsqueeze_)rrr need_squeezein_Cin_Hin_Wout_Cout_Hout_Wrrn weights_H indices_H sym_len_Hs sym_len_He weights_W indices_W sym_len_Ws sym_len_Weimg_aug sym_patchinv_idx sym_patch_invout_1reidxj out_1_augout_2rrrimresizesx                   rcCs`t|}|dkr dnd}|r|d|\}}}|t||t||}}} d} d} t|||| | |\} } }}t|| || | |\}}}}t|||||}| d|| ||d|ddddf}t |dddd }| d|}| dd| ||| dddddf}t |dddd }| d|}| d||| |t|||}| d} t|D].}t| |d}t|D]}|||| dd|fdd| |||dd|f<qqt|||||}| d|| ||ddd|ddf}t |dddd }| d|}| dd| ||dd| dddf}t |dddd }| d|}| d||| |t|| |}|d} t| D],}t||d}t|D]}|dd||| |f|||dd||f<qqx|r||S) Nr TFrprtrr ri)rqZ from_numpyrrrrWrwrrr|rrrrrarrmrrnumpy)rrrrrrrrrrrrnrrrrrrrrrrrrrrerrrrrrr imresize_np]sz                 rcCsB|jdd\}}t|}|d|||d|||dfS)zq Args: img: numpy image, WxH or WxHxC sf: scale factor Return: cropped image Nr .)rRrcopy)rsfr^r]Zimrrr modcrop_nps &rcCs|jd}td|dd|df}t|D])}t|D]"}|d|d||d|d||f|||f|7<qq|d}||| || f}||S)zLCalculate the X4 kernel from the X2 kernel (for proof see appendix in paper)rr r )rRrzerosrarz)kk_sizeZbig_krccropZ cropped_big_krrranalytic_kernels   B rc Csttt|t| gt|t|ggtddg}t|d|dg|d|d gg}t|dgd|gg}tt||tj|}tddg||d}|S)aM generate an anisotropic Gaussian kernel Args: ksize : e.g., 15, kernel size theta : [0, pi], rotation angle range l1 : [0.1,50], scaling of eigenvalues l2 : [0.1,l1], scaling of eigenvalues If l1 = l2, will get an isotropic Gaussian kernel. Returns: k : kernel ?rr )rUcovr)rr8rccossinlinalginvgm_blur_kernel) ksizethetal1l2vVDSigmarrrranisotropic_Gaussians  (rc Cs~|dd}t||g}t|D]$}t|D]}||d}||d}tjj||g||d|||f<qq|t|}|S)N@rur )rUr)rrrastatsmultivariate_normalZpdfrz) rUrrcenterrrLrscyZcxrrrrs      rc Cs|jdd\}}|dd}td|dtd|d}}|r)||}||} n||}||} t|d|d}t| d|d} |jdkrQt||||| }|jdkr{t|jdD]} t|||dddd| f|| |dddd| f<q]|S) zshift pixel for super-resolution with different scale factors Args: x: WxHxC or WxH sf: scale factor upper_left: shift direction Nr r rurrr ri)rRrrr"rrra) rsrZ upper_leftr]r^shiftZxvZyvx1y1rerrr shift_pixels     8rcCs|jdd\}}|jddd|jddd}}tjjj|||||fdd}|d|dd}|dd|jd|jd}|dd|jd|jd}tjjj||ddd ||d }||||jd|jd}|S) z2 x: image, NxcxHxW k: kernel, Nx1xhxw Nr r riZ replicate)padmoder r)ZbiasZstridepaddinggroups)rRrqnnZ functionalrrepeatrxZconv2d)rsrnrp1Zp2rrrblurs&rrp333333?g$@cCs|tj||}|tj||}tjtj}| tjj||d}t||g} tt|t| gt|t|gg} | | | j} tj | ddddddf} |dd|d} | dddddf} t t |dt |d\}}t ||gddddddddf}|| }|dddd}tdt|| |d|}|t|}|S)z" # modified version of https://github.com/assafshocher/BlindSR_dataset_generator # Kai Zhang # min_var = 0.175 * sf # variance of the gaussian kernel will be sampled between min_var and max_var # max_var = 2.5 * sf r Nrur rr ro)rrandomrandpidiagrcrrTrrmeshgridrastackrmexprdrz)rZ scale_factorZmin_varZmax_var noise_levelZlambda_1Zlambda_2rnoiseZLAMBDAQZSIGMAZ INV_SIGMAZMUXYZZZZZZZ_tZ raw_kernelrnrrr gen_kernels&   ($rc Cs||g}|ddd|dddg}|}tt|d |ddt|d |dd\}}|||| d||}t|}d||ttj|k<| }|dkrb||}|S)Nrrrr r ) rrrrscipyZfinforVepsrrz) ZhsizesigmaZsizZstdrsrLargr]Zsumhrrrfspecial_gaussianIs 0 rcCsbtdt|dgg}||d}d||d}|||g|d|d|g|||gg}t|}|S)Nrr )rr~rrc)alphah1h2r]rrrfspecial_laplacianXs  $ rcOs0|dkr t|i|S|dkrt|i|SdS)NgaussianZ laplacian)rr)Z filter_typeargskwargsrrrfspecialas rcCst|d|d}|S)z Args: x: HxWxC image, [0, 1] sf: down-scale factor Return: bicubicly downsampled LR image r )r)r)rsrrrrbicubic_degradationhsrcCs,tjj|tj|dddd}t||d}|S)a blur + bicubic downsampling Args: x: HxWxC image, [0, 1] k: hxw, double sf: down-scale factor Return: downsampled LR image Reference: @inproceedings{zhang2018learning, title={Learning a single convolutional super-resolution network for multiple degradations}, author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, pages={3262--3271}, year={2018} } r r wraprr)rfiltersconvolverrrrsrrrrrsrmd_degradationts rcCs,t||d}tjj|tj|dddd}|S)a bicubic downsampling + blur Args: x: HxWxC image, [0, 1] k: hxw, double sf: down-scale factor Return: downsampled LR image Reference: @inproceedings{zhang2019deep, title={Deep Plug-and-Play Super-Resolution for Arbitrary Blur Kernels}, author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, pages={1671--1681}, year={2019} } rr r rr)rrrrrrrrrrdpsr_degradations rcCs:tjj|tj|dddd}d}||d||d|dfS)z blur + downsampling Args: x: HxWxC image, [0, 1]/[0, 255] k: hxw, double sf: down-scale factor Return: downsampled LR image r r rrrN.)rrrrr)rsrrstrrrclassical_degradations rru2 c Cs|ddkr |d7}t|||fd}||}t|d|k}|d}t|||fd}|||}t|dd}||d||S)aUSM sharpening. borrowed from real-ESRGAN Input image: I; Blurry image: B. 1. K = I + weight * (I - B) 2. Mask = 1 if abs(I - B) > threshold, else: 0 3. Blur mask: 4. Out = Mask * K + (1 - Mask) * I Args: img (Numpy array): Input image, HWC, BGR; float32, [0, 1]. weight (float): Sharp weight. Default: 1. radius (float): Kernel size of Gaussian blur. Default: 50. threshold (int): r rr r)rZ GaussianBlurrrrr7r") rweightZradius thresholdrZresidualmaskZ soft_maskKrrradd_sharpenings   rcCsd|}dd|}|d}|d}tdkr8|t}|t}ttdddttj||d }ntd tddd|t}tjj|tj |dd d d }|S)N@r皙?rprur rgr rrrrrr mirrorr rrrandintrrrrrrrrrZwd2wdrrrrrr add_blur_1s(     rcCstj}|dkrtdd}n|dkrtd|d}nd}tj|t||jdt||jdftgdd }t |d d}|S) N皙?r r gffffff?rurrr r r  interpolationr) rrruniformrresizerrRchoicer")rrrnumsf1rrr add_resizes $ r"r c Cst||}tj}|dkr!|tjd|d|jtj}n^|dkr@|tjd|dg|jdddRtj}n?|d}ttjd}t tjdd}t t t |||}|tj gdt |d||jddtj}t|d d }|S) Nrrr皙?r r r rrrrr)rrrrnormalrRr7rrrr8rmrrrr" r noise_level1 noise_level2rr LrUconvrrradd_Gaussian_noises2    r-c Cs(t||}t|dd}t}|dkr)||tjd|d|jtj7}nb|dkrJ||tjd|dg|jdddRtj7}nA|d}ttj d }t tj d d }t t t |||}||tj gd t|d||jddtj7}t|dd}|S) Nrrrrrr$r r r r%)rrrr"r&rRr7rrrrr8rmrrrr'rrradd_speckle_noises4  "r.cCst|dddd}ddtd}tdkr,tj||tj|}n9t|ddd fgd }t|dddd}tj||tj||}||ddddtjf7}t|d d }|S) Nrrr r r rru.r )gA`"?gbX9?gv/?rr) rr"r#rZpoissonr7rr8Znewaxis)rvalsZimg_grayZ noise_grayrrradd_Poisson_noises  r0cCs\tdd}tt|tj}td|ttj|g\}}t |d}tt |tj }|S)NP_z.jpgr ) rrrrr$Z COLOR_RGB2BGRZimencoderZIMWRITE_JPEG_QUALITYZimdecoderr)rZquality_factorresultZencimgrrradd_JPEG_noise(s  r4@c Cs|jdd\}}td||}td||}|||||||ddf}t||t||}} |||||| | ||ddf}||fS)Nr r)rRrrr) ZlqZhqrZ lq_patchsizer]r^Zrnd_hZrnd_wZrnd_h_HZrnd_w_Hrrr random_crop2s"r6c Cst|}d\}}}|jdd\}}|d|||d|||df}|jdd\}} |dkrnt|krntjdkr_tj|td|jdtd|jdft gd d }nt |dd }t |d d }d}t t dd} | d| d} } | | kr| | | | | | <| | <| D]} | dkrt||d}q| dkr|jd|jd}}tdkrtdd|}tj|td||jdtd||jdft gd d }n2tddtdd|}t||}||}tjj|tj|dddd}|dd|dd|df}t |d d }q| dkrCtj|td||td||ft gd d }t |d d }q| dkrPt|ddd}q| dkr`t|kr`t|}qt|}t|}|S) This is the variant of the degradation model of BSRGAN from the paper "Designing a Practical Degradation Model for Deep Blind Image Super-Resolution" sf: scale factor isp_model: camera ISP model Returns img: low-quality patch, size: lq_patchsizeXlq_patchsizeXC, range: [0, 1] hq: corresponding high-quality patch, size: (lq_patchsizexsf)X(lq_patchsizexsf)XC, range: [0, 1] ?g?r9Nr .rprur rrrTrrr rrrr#皙?rr rrr(r)rj)rrRrrrrrrrrrr"sampleraindexrrrrrzrrrrr-r4r$imagerZ isp_model_Z jpeg_probZ scale2_probrZw1r]r^Z shuffle_orderZidx1Zidx2reabr!rZ k_shiftedrrrr>sv *"            cCsd|}dd|}tdkr2|t}|t}tdtdddttj||d}ntd dtddd|t}tjj|tj |dd d d }|S) Nrrrrur rgr rrr rrrrrrr add_blur_2s$      rDc Cst|}d\}}}|jdd\}}|d|||d|||df}|jdd\}} |dkrnt|krntjdkr_tj|td|jdtd|jdft gd d }nt |dd }t |d d }d}t t dd} | d| d} } | | kr| | | | | | <| | <| D]} | dkrt||d}q| dkrt||d}q| dkr&|jd|jd}}tdkrtdd|}tj|td||jdtd||jdft gd d }n2tddtdd|}t||}||}tjj|tj|dddd}|dd|dd|df}t |d d }q| dkrNtj|td||td||ft gd d }t |d d }q| dkr[t|ddd}q| dkrkt|krkt|}qt|}t|}|S)r7r8Nr .rprur rrrTrrr:r rg?rr#r;rr rrr<rj)rrRrrrrrrrrrr"r=rar>rDrrrrzrrrrr-r4r$r?rrrr sz *"            )r )T)r)r)rurr )rp)r r#)rpr5)rpN)>rWosrrrrrZ scipy.statsrrqrZscipy.interpolaterZ scipy.linalgrZtorchvision.utilsrenviron__all__rrr$r'r(r>rArBrNr_rfr`rtrrrrrrrrrrrcrrrrrrrrrrr"r-r.r0r4r6rrDr rrrrsx             9 JL      /         K