o *jXz@sddlZddlmZmZmZmZmZmZddlZddl Z ddl Z ddl m Z ddlZddlmZddlmZmZmZmZmZmZmZmZmZmZddlmZddlmZddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'ddl(m)Z)m*Z*m+Z+ddl,m-Z-dd l.m/Z/m0Z0dd l1m2Z2dd l3Tdd l4Tdd l5Tdd l6m7Z7m8Z8dd l9m:Z:e:Z;dZe-j>dGddde0Z? dddZ@GdddeZAdS)N)AnyCallableDictListOptionalUnion) AutoencoderKLControlNetModelDiffusionPipelineEulerAncestralDiscreteSchedulerEulerDiscreteScheduler(StableDiffusionControlNetImg2ImgPipeline(StableDiffusionControlNetInpaintPipelineStableDiffusionInpaintPipelineStableDiffusionPipelineUNet2DConditionModel)MultiControlNetModel)StableDiffusionPipelineOutput) deprecateis_accelerate_availableis_accelerate_versionis_compiled_modulelogging randn_tensorreplace_example_docstring)load_objload_objs_as_meshessave_obj)Models)Tensor TorchModel)MODELS)*) ModelFileTasks) get_loggera Examples: ```py >>> from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, DDIMScheduler >>> from diffusers.utils import load_image >>> import numpy as np >>> import torch >>> init_image = load_image(image_path) >>> init_image = init_image.resize((512, 512)) >>> generator = torch.Generator(device="cpu").manual_seed(1) >>> mask_image = load_image(mask_path) >>> mask_image = mask_image.resize((512, 512)) >>> def make_inpaint_condition(image, image_mask): ... image = np.array(image.convert("RGB")).astype(np.float32) / 255.0 ... image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0 ... assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size" ... image[image_mask > 0.5] = -1.0 # set as masked pixel ... image = np.expand_dims(image, 0).transpose(0, 3, 1, 2) ... image = torch.from_numpy(image) ... return image >>> control_image = make_inpaint_condition(init_image, mask_image) >>> controlnet = ControlNetModel.from_pretrained( ... "lllyasviel/control_v11p_sd15_inpaint", torch_dtype=torch.float16 ... ) >>> pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( ... "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16 ... ) >>> pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) >>> pipe.enable_model_cpu_offload() >>> image = pipe( ... "a handsome man with ray-ban sunglasses", ... num_inference_steps=20, ... generator=generator, ... eta=1.0, ... image=init_image, ... mask_image=mask_image, ... control_image=control_image, ... ).images[0] ``` ) module_namecs@eZdZfddZddZddZd dd Zd d d ZZS) Tex2Texturecstj|d|i|tjr td|_td|jnt dt |d}|d}|d}| dtj }t j||d |j|_tj||d |j|_tj||j|d |j|_td d S) aThe Tex2Texture is modified based on TEXTure and Text2Tex, publicly available at https://github.com/TEXTurePaper/TEXTurePaper & https://github.com/daveredrum/Text2Tex Args: model_dir: the root directory of the model files model_dircudaz Use GPU: {}zno gpu avaiablez /base_model/z/control_model/z/inpaint_model/ torch_dtype)r*) controlnetr*zmodel load overN)super__init__torchr)Z is_availabledeviceloggerinfoformatprintexitgetZfloat16r Zfrom_pretrainedtor+rZ inpaintmodel%StableDiffusionControlinpaintPipelinepipe)selfr(argskwargsZ model_pathZcontrolmodel_pathZinpaintmodel_pathr* __class__y/var/www/html/Deteccion_Ine/venv/lib/python3.10/site-packages/modelscope/models/cv/text_texture_generation/Tex2Texture.pyr-Vs8  zTex2Texture.__init__cCs0t||jd\}}}t|g|jd}||||fS)Nr/)rr/r)r9 mesh_pathvertsfacesauxmeshr>r>r? init_meshts zTex2Texture.init_meshc Cs|}|jd}|jdd|d}|| }|ddddf|ddddf}|}d|}|d|}||}||||fS)Nrdimg?) Zget_bounding_boxesZ verts_packedshapemeanrepeatZ offset_vertsmax unsqueezeZ scale_verts) r9rEZbboxZ num_verts mesh_centerZlensmax_lenscaleZnew_meshr>r>r?normalize_meshys $ zTex2Texture.normalize_meshnormalized.objc Cs@td|}t|||jd|j|j|jt|jdddS)Nz!=> saving normalized mesh file...r)rBrCZdecimal_places verts_uvsZ faces_uvsZ texture_map)r3rZ verts_idxrVZ textures_idxZtexture_imageslistkeys)r9rBrCrDpathZobj_pathr>r>r?save_normalized_objs zTex2Texture.save_normalized_objc CsD||\}}}}||\}}}}|||||||||||fSN)rFrSrZ) r9rAZ save_pathrErBrCrDrPrRr>r>r?mesh_normalizedszTex2Texture.mesh_normalized)rT) __name__ __module__ __qualname__r-rFrSrZr\ __classcell__r>r>r<r?r'Rs    r'Fcsf|durtd|durtdt|tjrt|tjs&tdt|d|jdkr;|jddks6Jd|d}|jdkrH|dd}|jdkr_|jdd krZ|d}n|d }|jd kri|jd ksmJd |jd d|jd dksJd |jd|jdksJd| dks| d krtd| dks| d krtdd||dk<d ||dk<|j tj d}nt|tjrtdt|dt|t jjtjfr|g}t|trt|dt jjrfdd|D}dd|D}tj|dd}nt|tr"t|dtjr"tjdd|Ddd}|ddd d}t|j tj ddd}t|t jjtjfrF|g}t|trut|dt jjrufdd|D}tjdd|Ddd}|tj d}nt|trt|dtjrtjdd|Ddd}d||dk<d ||dk<t|}||dk}|r|||fS||fS) Nz"`image` input cannot be undefined.z'`mask_image` input cannot be undefined.z,`image` is a torch.Tensor but `mask` (type: z is notrz2Image outside a batch should be of shape (3, H, W)rGrJz%Image and Mask must have 4 dimensionsz4Image and Mask must have the same spatial dimensionsz,Image and Mask must have the same batch sizez Image should be in [-1, 1] rangezMask should be in [0, 1] range?)dtypez,`mask` is a torch.Tensor but `image` (type: c"g|] }|jftjjdqS)ZresampleresizePILImageZLANCZOS.0iheightwidthr>r? z1prepare_mask_and_masked_image..cSs(g|]}t|ddddfqS)RGBNnparrayconvertrmr>r>r?rss()ZaxiscSsg|] }|dddfqSr[r>rmr>r>r?rssg_@?crgrhrirmrpr>r?rsrtcSs*g|]}t|dddddfqS)LNrvrnmr>r>r?rss*go@cSsg|] }|ddddfqSr[r>r|r>r>r?rss) ValueError isinstancer.r TypeErrortypendimrKrOminrNr6Zfloat32rkrlrwndarrayrWZ concatenateZ transposeZ from_numpyZastype)imagemaskrqrr return_image masked_imager>rpr?prepare_mask_and_masked_images                      rc2@sxeZdZeee                      d$d eee efd eej e j j fd eej e j j fdeej e j j eje ej e e j j e ejfdeedeededededeeee efdeededeeeje ejfdeej deej deej deededeeeeej gdfdedeeeefd eee efd!ef.d"d#ZdS)%r7Nrz2@rJpilTreFpromptr mask_image control_imagerqrrstrengthnum_inference_stepsguidance_scalenegative_promptnum_images_per_prompteta generatorlatents prompt_embedsnegative_prompt_embeds output_type return_dictcallbackcallback_stepscross_attention_kwargscontrolnet_conditioning_scale guess_modec=Cs||||\}}||||||| ||| |dur"t|tr"d}n|dur0t|tr0t|}n|jd}|j}| dk}t|j rE|j j n|j }t|t rZt|t rZ|gt|j }t|trc|jjn|j djj}|pm|}|durx|ddnd}|j||| || |||d}t|tr|j||||| | ||j||d }n't|t rg}|D]}|j||||| | ||j||d }||q|}nJt||||d d \} }!}"|jj||d |j|||d \}#}|#dd|| }$|dk}%|jjj}&|jjj}'|'d k}(|j|| |&|||j|| ||"|$|%d |(d })|(r|)\}}*}+n|)\}}*| | |!|| |||j|| | \} },|!| | }-|j"|d}.t#|#D]\}/}0|rUt$%|gdn|}1|j&|1|0}1|ru|ru|}2|j&|2|0}2|'dd}3n|1}2|}3|j |2|0|3|||dd\}4}5|r|rdd|4D}4t$%t$(|5|5g}5|'dkrt$j%|1| |,gdd}1|j|1|0|||4|5ddd}6|r|6'd\}7}8|7| |8|7}6|jj)|6|0|fi|-ddid}|'d kr|+dd}9| dd}:|/t|#dkr|j*|9|*t$+|0g}9d|:|9|:|}|/t|#dks#|/d|jj,dkr9|.-|dur9|/|dkr9||/|0|qFWdn 1sFwYt.|drh|j/durh|j0d|j 0dt$j12|dks|jj3||jjj4ddd}|5|||j\}};n|}d};|;durd g|jd} 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.FloatTensor`, *optional*): Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor will ge generated by sampling using the supplied random `generator`. prompt_embeds (`torch.FloatTensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.FloatTensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generate image. Choose between [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a plain tuple. callback (`Callable`, *optional*): A function that will be called every `callback_steps` steps during inference. The function will be called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. callback_steps (`int`, *optional*, defaults to 1): The frequency at which the `callback` function will be called. If not specified, the callback will be called at every step. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py). controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 0.5): The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original unet. If multiple ControlNets are specified in init, you can set the corresponding scale as a list. Note that by default, we use a smaller conditioning scale for inpainting than for [`~StableDiffusionControlNetPipeline.__call__`]. guess_mode (`bool`, *optional*, defaults to `False`): In this mode, the ControlNet encoder will try best to recognize the content of the input image even if you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended. Examples: Returns: [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is a list with the generated images, and the second element is a list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) content, according to the `safety_checker`. NrJrrzrR)rrZ lora_scale) rrrrq batch_sizerr/rfdo_classifier_free_guidancerFT)rr@)rrr/rb)rZtimestepis_strength_maxZ return_noisereturn_image_latents)totalrG)encoder_hidden_statesZcontrolnet_condZconditioning_scalerrcSs g|] }tt||gqSr>)r.cat zeros_like)rndr>r>r?rs;szBStableDiffusionControlinpaintPipeline.__call__.. rH)rrZdown_block_additional_residualsZmid_block_additional_residualrrfinal_offload_hookcpuZlatent)rcSsg|]}| qSr>r>)rnZhas_nsfwr>r>r?rss)rdo_denormalize)ZimagesZnsfw_content_detected):Z_default_height_widthZ check_inputsrstrrWlenrKZ_execution_devicerr+Z _orig_modrfloatnetsr configglobal_pool_conditionsr5Z_encode_promptZprepare_control_imagerfappendrZ schedulerZ set_timestepsZ get_timestepsrMZvaeZlatent_channelsZunetZ in_channelsZprepare_latentsZprepare_mask_latentsZprepare_extra_step_kwargs progress_bar enumerater.rZscale_model_inputchunkrstepZ add_noiseZtensororderupdatehasattrrr6r)Z empty_cachedecodeZscaling_factorZrun_safety_checkerZimage_processor postprocessZoffloadr)=r9rrrrrqrrrrrrrrrrrrrrrrrrrrr/rr+rZtext_encoder_lora_scaleZcontrol_imagesZcontrol_image_rrZ init_imageZ timestepsZlatent_timesteprZnum_channels_latentsZnum_channels_unetrZlatents_outputsnoiseZ image_latentsZmasked_image_latentsZextra_step_kwargsrrotZlatent_model_inputZcontrol_model_inputZcontrolnet_prompt_embedsZdown_block_res_samplesZmid_block_res_sampleZ noise_predZnoise_pred_uncondZnoise_pred_textZinit_latents_properZ init_maskZhas_nsfw_conceptrr>r>r?__call__s}                          [         z.StableDiffusionControlinpaintPipeline.__call__)NNNNNNrzrrNrJrNNNNrTNrJNreF)r]r^r_r.Zno_gradrEXAMPLE_DOC_STRINGrrrrrkrlZ FloatTensorrwrrintr Generatorboolrrrrr>r>r>r?r7s     r7)F)BostypingrrrrrrZcv2numpyrwrkZ PIL.Imagerlr.Ztorchvision.transformsZ transformsZ diffusersrr r r r r rrrrZ.diffusers.pipelines.controlnet.multicontrolnetrZ$diffusers.pipelines.stable_diffusionrZdiffusers.utilsrrrrrrrZ pytorch3d.iorrrZmodelscope.metainforZmodelscope.models.baserr Zmodelscope.models.builderr!Z8modelscope.models.cv.text_texture_generation.lib2.cameraZ;modelscope.models.cv.text_texture_generation.lib2.init_viewZ2modelscope.models.cv.text_texture_generation.utilsZmodelscope.utils.constantr#r$Zmodelscope.utils.loggerr%r0rZregister_moduleZtext_texture_generationr'rr7r>r>r>r?s@   0  $   +I l