How to visualize multiple point-cloud files as video using Open3d?

Question

I use open3d to visualize the pointclouds (*.bin) and the predicted bounding boxes (code below, https://github.com/shangjie-li/pointpillars/blob/master/utils/open3d_vis_utils.py). The code shows the point cloud and the bounding boxes for 1.5s, and then close the current window and show the next files in a new window. I am wondering how can I modify my code such that I would be able to visualize the sequence of point clouds as a video (real-time) without opening/closing the window.

import open3d
import torch
import matplotlib
import numpy as np
import time

from . import common_utils, box_utils


box_colormap = {
    'Car': (0, 1, 0),
} # RGB


def draw_scenes(points, gt_boxes=None, ref_boxes=None, ref_labels=None,
    point_colors=None, point_size=1.0, window_name='Open3D'):
    if isinstance(points, torch.Tensor):
        points = points.cpu().numpy()
    if isinstance(gt_boxes, torch.Tensor):
        gt_boxes = gt_boxes.cpu().numpy()
    if isinstance(ref_boxes, torch.Tensor):
        ref_boxes = ref_boxes.cpu().numpy()
    if isinstance(point_colors, torch.Tensor):
        point_colors = point_colors.cpu().numpy()

    vis = open3d.visualization.Visualizer()
    vis.create_window(window_name=window_name, width=1280, height=720)

    vis.get_render_option().point_size = point_size
    vis.get_render_option().background_color = np.asarray([0.4, 0.4, 0.4])

    pts = open3d.geometry.PointCloud()
    pts.points = open3d.utility.Vector3dVector(points[:, :3])

    vis.add_geometry(pts)
    if point_colors is not None:
        pts.colors = open3d.utility.Vector3dVector(point_colors)
    else:
        pts.colors = open3d.utility.Vector3dVector(np.ones((points.shape[0], 3)) * 0.9)

    if gt_boxes is not None:
        vis = draw_box(vis, gt_boxes, color=(1, 0, 0))
    if ref_boxes is not None:
        if ref_labels is not None:
            vis = draw_box(vis, ref_boxes, ref_labels)
        else:
            vis = draw_box(vis, ref_boxes, color=(0, 1, 0))

    #press esc or q to close the pointcloud window
    # vis.run()
    # vis.destroy_window()
    
    #close the pointcloud window after 1.5s
    vis.poll_events()
    vis.update_renderer() 
    time.sleep(1.5)
    vis.destroy_window()
    



    


def draw_box(vis, boxes, labels=None, color=(1, 0, 0)):
    """
        7 -------- 4
       /|         /|
      6 -------- 5 .
      | |        | |
      . 3 -------- 0
      |/         |/
      2 -------- 1
    Args:
        boxes: [x, y, z, dx, dy, dz, heading], (x, y, z) is the box center
        labels: [name]

    Returns:
    """
    for i in range(boxes.shape[0]):
        corners3d = box_utils.boxes_to_corners_3d(np.array([boxes[i]]))[0]
        edges = np.array([
            [0, 1], [1, 2], [2, 3], [3, 0],
            [4, 5], [5, 6], [6, 7], [7, 4],
            [0, 4], [1, 5], [2, 6], [3, 7],
            [0, 5], [1, 4], # heading
        ])
        line_set = open3d.geometry.LineSet()
        line_set.points = open3d.utility.Vector3dVector(corners3d)
        line_set.lines = open3d.utility.Vector2iVector(edges)
        if labels is not None:
            line_set.paint_uniform_color(box_colormap[labels[i]])
        else:
            line_set.paint_uniform_color(color)
        vis.add_geometry(line_set)
    return vis

I appreciate any help in advance.

Thanks,

Abbas

Answer 1

Read the official tutorial here for information on non-blocking visualization. The below code will show how to generate new pointcloud data and update pointcloud in Visualizer. This will generate a dynamic visualization.

import time

import numpy as np
import open3d as o3d

# Create a visualization object and window
vis = o3d.visualization.Visualizer()
vis.create_window()

# create pcd object
point_cloud_o3d = o3d.geometry.PointCloud()  # Important to use same object for repeated call to update_geometry
dataset = o3d.data.PLYPointCloud()

start_time = time.time()
prev_sec = -1
translation_index = 0
translation_speed = 0.01
first_call = True
while True:
    curr_sec = int(time.time() - start_time)
    if curr_sec - prev_sec >= 1.0:  # Check if 1 second has passed
        prev_sec = curr_sec

        # Create a sample pointcloud
        office = o3d.io.read_point_cloud(dataset.path)
        points = np.asarray(office.points) + translation_index * translation_speed

        # Modify original pointcloud a little to show a video
        # Note - Always edit in-place.
        point_cloud_o3d.points = o3d.utility.Vector3dVector(points)

        # add or update geometry objects
        if first_call:
            vis.add_geometry(point_cloud_o3d)  # add point cloud
            first_call = False  # change flag
        else:
            vis.update_geometry(point_cloud_o3d)  # update point cloud

        translation_index += 1

    # update o3d window
    if not vis.poll_events():
        break
    vis.update_renderer()

vis.close()
vis.destroy_window()

One thing that I realized late during my Open3D learning was vis.add_geometry(point_cloud_o3d) binds the memory of point_cloud_o3d to graphics pipeline (opengl). The problem is if you call vis.add_geometry(point_cloud_o3d) on an empty geometry, aka point_cloud_o3d = o3d.geometry.PointCloud(), it will have no memory allocated for point_cloud_o3d.points which is of type std::vector in C++. So, even if you add points later to the pointcloud object, the OpenGL pipeline will be bound to an incorrect address.

Also, see related SO answer for C++ with more remarks. Also, see this SO answer which focuses on remove_geometry and reset_bounding_box arguments.