how to replace an object in image with another object without disturbing an image

Question

Problem Statement -: I want to replace an object(Sofa or table) from an input image with an another object from my database.

I have create an Image segmentation model using yolov8 which is working fine and etecting objects and creating masks. Once I received the mask from the results i am using lama to remove the detected image. Here's the code :-

import cv2
import matplotlib.pyplot as plt
import os
from lama_inpaint import inpaint_img_with_lama
import numpy as np
import torch
%matplotlib inline
from ultralytics import YOLO

from utils import load_img_to_array, save_array_to_img, dilate_mask, \
    show_mask, show_points, get_clicked_point  
#these are the files I use from impaint-anything [github](https://github.com/geekyutao/Inpaint-Anything?tab=readme-ov-file#replace-anything) and only used remove anything part. 

model = YOLO('Soa_Table_29_2.pt') Link for the model -                  (https://drive.google.com/file/d/1Mtn5V6dO9ge226AxreQt4OvCFvBgNGLk/view?usp=drive_link)

imageName="C:\\Users\\spx016\\Downloads\\Testing Image\\test8.jpg"
orig_image = cv2.imread(imageName)
orig_image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB)
image = cv2.resize(orig_image, (700,540))

confidence_threshold = 0.20
results = model.predict(image, conf = confidence_threshold)

cordinates = result[0].boxes.xywh.cpu().numpy()
cordinates

object_index = 2 #The object i wish to remove is at this index. 

mask_cordinates = result.masks.xy[object_index]
mask_cordinates = mask_cordinates.astype(int)
mask = np.zeros_like(image)
masks = cv2.drawContours(mask, [np.array(mask_cordinates)], -1, (255, 255, 255), -1)

kernel = np.ones((30, 30), np.uint8)
mask = cv2.dilate(masks, kernel)

mask = (mask/255).astype(int)

lama_config = "C:\\Users\\spx016\\virtual_Furniture\\configs\\prediction\\default.yaml"
lama_ckpt = "C:\\Users\\spx016\\virtual_Furniture\\big-lama"

img_inpainted = inpaint_img_with_lama(
      image, mask, lama_config, lama_ckpt, device="cuda")

save_array_to_img(img_inpainted, "img_inpainted.png")

This is the code for what I have done till now an this is removing the object from the image. Now, I am wishing to replace or add a new object(sofa) in place of the removed object.

I have tried this using open cv -:

replaceable_obj = cv2.imread("C:\\Users\\spx016\\Downloads\\pngfind.com-steel-sofa-set-png-5395375.jpg")
plt.imshow(replaceable_obj)

original_roi = inpainted_image[y_min:y_max, x_min:x_max]
replacement_roi_resized = cv2.resize(replaceable_obj, (original_roi.shape[1], original_roi.shape[0]))
inpainted_image[y_min:y_max, x_min:x_max] = replacement_roi_resized

plt.imshow(cv2.cvtColor(inpainted_image, cv2.COLOR_BGR2RGB))
plt.axis('off')
plt.title("Replace object with another object in image.")
plt.show()

I am not getting correct result as this is just pasting the object on the image. I want to removal an replace without hampering the original image. I true stable diffusion but that is using a prompt which I don't have so that's too not working.

I also tried seamless clone in open cv:

import cv2
import numpy as np
import matplotlib.pyplot as plt
im = cv2.imread("img_inpainted_resized.png")
obj= cv2.imread("C:\\Users\\spx016\\Downloads\\transparentSofa.png")
masks = cv2.imread("maskOfSofa.png")
normal_clone = cv2.seamlessClone(obj, im, masks, center, cv2.NORMAL_CLONE)
mixed_clone = cv2.seamlessClone(obj, im, masks, center, cv2.MIXED_CLONE)

Can anyone help me with this. I have attached the drive link for yolo model, it's just trained for testing purpose for now.