Opencv Mat to opengl with OSVR

Question

I have a problem with opencv and opengl. I need to show with opengl the images retrieved by a webcam with opencv, and put this in Razer OSVR. But with my current code the framerate is around 1 fps or 2 fps, and I don't know what I am doing wrong. Here is my code, I think the wrong thing is in draw_cube() function.

Main.cpp

// Internal Includes

#include <osvr/ClientKit/ClientKit.h>
#include <osvr/ClientKit/Display.h>
#include "SDL2Helpers.h"

#include "OpenGLCube.h"

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/opencv.hpp>


// Library/third-party includes
#include <SDL.h>
#include <SDL_opengl.h>

// Standard includes
#include <iostream>

static auto const WIDTH = 1920;
static auto const HEIGHT = 1080;

// Forward declarations of rendering functions defined below.
void render(osvr::clientkit::DisplayConfig &disp);

int main(int argc, char *argv[]) {
    namespace SDL = osvr::SDL2;

    // Open SDL
    SDL::Lib lib;

    // Use OpenGL 2.1
    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 1);

    // Create a window
    auto window = SDL::createWindow("OSVR", SDL_WINDOWPOS_UNDEFINED,
                                SDL_WINDOWPOS_UNDEFINED, WIDTH, HEIGHT,
                                SDL_WINDOW_OPENGL | SDL_WINDOW_SHOWN);
    if (!window) {
        std::cerr << "Could not create window: " << SDL_GetError() << std::endl;
        return -1;
    }

    // Create an OpenGL context and make it current.
    SDL::GLContext glctx(window.get());

    // Turn on V-SYNC
    SDL_GL_SetSwapInterval(1);

    // Start OSVR and get OSVR display config
    osvr::clientkit::ClientContext ctx("com.osvr.example.SDLOpenGL");
    osvr::clientkit::DisplayConfig display(ctx);
    if (!display.valid()) {
        std::cerr << "\nCould not get display config (server probably not "
                 "running or not behaving), exiting."
              << std::endl;
        return -1;
    }

    std::cout << "Waiting for the display to fully start up, including "
             "receiving initial pose update..."
          << std::endl;
    while (!display.checkStartup()) {
        ctx.update();
    }
    std::cout << "OK, display startup status is good!" << std::endl;

    // Event handler
    SDL_Event e;
#ifndef __ANDROID__ // Don't want to pop up the on-screen keyboard
    SDL::TextInput textinput;
#endif
    bool quit = false;
    while (!quit) {
        // Handle all queued events
        while (SDL_PollEvent(&e)) {
            switch (e.type) {
            case SDL_QUIT:
                // Handle some system-wide quit event
                quit = true;
                break;
            case SDL_KEYDOWN:
                if (SDL_SCANCODE_ESCAPE == e.key.keysym.scancode) {
                    // Handle pressing ESC
                    quit = true;
                }
                break;
            }
            if (e.type == SDL_QUIT) {
                quit = true;
            }
        }

        // Update OSVR
        ctx.update();

        // Render
        render(display);

        // Swap buffers
        SDL_GL_SwapWindow(window.get());
    }

    return 0;
}


void render(osvr::clientkit::DisplayConfig &disp) {

    /// For each viewer, eye combination...
    disp.forEachEye([](osvr::clientkit::Eye eye) {

      /// For each display surface seen by the given eye of the given
    /// viewer...
        eye.forEachSurface([](osvr::clientkit::Surface surface) {
            auto viewport = surface.getRelativeViewport();
            glViewport(static_cast<GLint>(viewport.left),
                   static_cast<GLint>(viewport.bottom),
                   static_cast<GLsizei>(viewport.width),
                   static_cast<GLsizei>(viewport.height));

        glLoadIdentity();

   cv::VideoCapture cap(0); // open the default camera

   cv::Mat img;
   cap >> img; // get a new frame from camera

     cv::flip(img,img,0);

//resize(img, img, Size(160, 140), 0, 0, INTER_CUBIC);

draw_cube(img);
        });
    });

}

OpenGLCube.h

#ifndef INCLUDED_OpenGLVIDEO_h
#define INCLUDED_OpenGLVIDEO_h_

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/opencv.hpp>

using namespace cv;


GLuint texture;

void draw_cube(cv::Mat img) 
{

glGenTextures(1, &texture);

glBindTexture(GL_TEXTURE_2D, texture);

glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);


glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB,img.size().width,img.size().height, 0, GL_BGR,GL_UNSIGNED_BYTE,img.data);


glEnable(GL_TEXTURE_2D);

glBegin(GL_QUADS);
glTexCoord2d(0.0, 1.0);
glVertex2d(-1,1);
glTexCoord2d(0.0, 0.0);
glVertex2d(-1,-1);
glTexCoord2d(1.0, 0.0);
glVertex2d(1,-1);
glTexCoord2d(1.0, 1.0);
glVertex2d(1,1);
glEnd();

glDisable(GL_TEXTURE_2D);

glDeleteTextures(1, &texture);
}

#endif 

Thanks.

Answer 1

This line: cv::VideoCapture cap(0); // open the default camera

It mustn't be in a mainloop.

Thanks for your replies.

Answer 2

glGenTextures should be called once, before your main loop since you don't need to create a brand new texture every frame. You can simply overwrite the previous data using the same texture object using glTexImage2D just as you are now. Similarly, as a good convention, glDeleteTextures should be called once before exiting the program.

For example, the following should be done outside of your main loop:

glGenTextures(1, &texture);

glBindTexture(GL_TEXTURE_2D, texture);

glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);

You will, of course, need to declare texture outside of your main loop and draw function and pass that in.