render views from an object with pyrender

Question

I want to create 12 images of an object (.obj file) from different angles. The idea is to use pyrender to render a view from upfront of the object, then rotate the camera in 12 steps around the object and render another view each time.

The problem with most of the rendered images is, that parts of the objects are cut off in the image.

I tried to change the parameters of the Orthographic camera (xmag, ymag, znear, zfar) which did not work. I also tried using a perspective camera, which reolved in the same problem. Changing light/camera positions didnt work either.

image_example:

edges_cut_off

def render_views(in_path, out_path):
    
    def create_rotation_matrix(cam_pose, center, axis, angle):
        # Step 1: Translate camera pose to the origin
        translation_matrix = np.eye(4)
        translation_matrix[:3, 3] = -center
        
        translated_cam_pose = np.dot(translation_matrix, cam_pose)

        # Step 2: Create rotation matrix
        rotation_matrix = rotation_matrix_from_axis_angle(axis, angle)

        # Step 3: Combine translated camera pose and rotation matrix
        final_cam_pose = np.dot(rotation_matrix, translated_cam_pose)

        return final_cam_pose

    def rotation_matrix_from_axis_angle(axis, angle):
        # Normalize the axis
        axis = axis / np.linalg.norm(axis)
        
        # Calculate components of the rotation matrix
        c = np.cos(angle)
        s = np.sin(angle)
        t = 1 - c
        x, y, z = axis

        # Build the rotation matrix
        rotation_matrix = np.array([
            [t*x*x + c,   t*x*y - z*s, t*x*z + y*s, 0],
            [t*x*y + z*s, t*y*y + c,   t*y*z - x*s, 0],
            [t*x*z - y*s, t*y*z + x*s, t*z*z + c,   0],
            [0,           0,           0,           1]
        ])

        return rotation_matrix

    ### number of views to create
    increment = 12

    ### factor to set light distance to object -> multiplied with largest dimension later
    light_distance_factor = 1
    ### dimension factor, camera distance to object -> multiplied with largest dimension later
    dim_factor = 1 

    ### load obj_file and create mesh
    mesh = trimesh.load(in_path)
    mesh = pyrender.Mesh.from_trimesh(mesh)

    ### create light
    # light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.0)

    ### create scene and add mesh of object
    scene = pyrender.Scene()
    scene.add(mesh)

    ### Find the largest dimension of the object to set camera and light distance accordingly
    ### Supposed to represent the distance of the two points furthest apart from each other
    larg_dim = np.max(mesh.bounds[1]-mesh.bounds[0])
    print('###\n ', larg_dim, '\n###')

    ### set camera distance 
    cam_dist = dim_factor * larg_dim
    # print('###\n ', cam_dist, '\n###')

    ### set lights distance
    light_distance = light_distance_factor * larg_dim
    # if lights distance is under 5 it resolves in overlightening
    if light_distance <= 5:
        light_distance = 5

    ### create lights around the object and add to scene
    for direction in ['front', 'back', 'left', 'right', 'top', 'bottom']:
        light_pose = np.eye(4)
        print(light_pose)
        if direction == 'front':
            light_pose[2, 3] = light_distance
        elif direction == 'back':
            light_pose[2, 3] = -light_distance
        elif direction == 'left':
            light_pose[0, 3] = -light_distance
        elif direction == 'right':
            light_pose[0, 3] = light_distance
        elif direction == 'top':
            light_pose[1, 3] = light_distance
        elif direction == 'bottom':
            light_pose[1, 3] = -light_distance
        
        light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=50.0)
        scene.add(light, pose=light_pose)

    ### set orthographic camera and add to scene
    def_pose = np.eye(4)
    camera = pyrender.OrthographicCamera(xmag=cam_dist, ymag=cam_dist, znear=0.05, zfar=3*larg_dim)
    cam_node = scene.add(camera, pose=def_pose)

    ### create renderer
    ren = pyrender.OffscreenRenderer(800, 800)
    
    ### create views
    center_point = np.array([0, 0, 0])
    axis_base = [0,1,0]
    for inc in range(1,increment+1):
        ### get current camera-pose, rotate around axis, set new pose to scene           
        cam_pose =  scene.get_pose(cam_node)
        rot_axis = np.array(axis_base)
        rot_angle = np.pi / increment
        cam_pose = create_rotation_matrix(cam_pose, center_point, rot_axis, rot_angle)
        scene.set_pose(cam_node, cam_pose)
        ### render scene
        color, _ = ren.render(scene)
        ### save image
        im = Image.fromarray(color)
        im.save(os.path.join(out_path, 'renview' + '['+ str(inc) +'].png')) 

render_views(src_path, dst_path)

Answer 1

Use the below function instead to generate new poses. You can change it from random if you like to.

def random_camera_pose(radius=2.0):
theta = np.random.uniform(0, 2* np.pi)  # theta is the angle with the z-axis

rot_z = np.array([
    [np.cos(theta), -np.sin(theta), 0, 0],
    [np.sin(theta),  np.cos(theta), 0, 0],
    [0,              0,             1, 0],
    [0,              0,             0, 1]
])

pose = np.array([
    [0.0,  -np.sqrt(2)/2, np.sqrt(2)/2, 0.5],
    [1.0, 0.0,           0.0,           0.0],
    [0.0,  np.sqrt(2)/2,  np.sqrt(2)/2, 0.5],
    [0.0,  0.0,           0.0,          1.0]
])

cam_pose_rotated = np.dot(rot_z, pose)
print(cam_pose_rotated)

return cam_pose_rotated

Use the camera pose obtained from the above in all the scene.add() for the camera node

ren = pyrender.OffscreenRenderer(800, 800)

# Render the scene from each random camera pose - 12 views
for i in range(12):
    # Create a camera
    camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)

    # Generate a random camera pose
    camera_pose = random_camera_pose()

    # Add the mesh obj to the scene
    mesh_node = scene.add(mesh, pose=drill_pose)

    # Add the camera to the scene with the specified pose
    camera_node = scene.add(camera, pose=camera_pose)

    # Render the scene
    color, depth = renderer.render(scene)


    im = Image.fromarray(color)
    im.save(os.path.join(out_path, 'renview' + '['+ str(inc) +'].png'))

    # Remove the camera from the scene
    scene.remove_node(camera_node)