Using the C++ Actor Framewok with OpenCV

Question

I am trying to make a distributed vision system by using the C++ Actor Framework and OpenCV. I started with a proof of concept code that compiles but while running the "edges" window is created but do not display anything (code shown hereafter).

I really do not understand why it do not work, any help will be appreciated.

Thanks in advance.

PS Code

main.cpp

#include <vector>
#include "opencv2/opencv.hpp"

#include "caf/all.hpp"

using namespace cv;
using namespace caf;
using namespace std;

struct Image {
    Image(Mat mat = Mat()) {
        data.assign(mat.datastart,mat.dataend);
        type = mat.type();
        rows = mat.rows;
        cols = mat.cols;
    }

    Mat toMat() const {
        return Mat(rows,cols,type,(void *)data.data());
    }

    vector<uchar> data;
    int type;
    int rows;
    int cols;
};

bool operator==(const Image& lhs, const Image& rhs) {
    return  lhs.data == rhs.data
            && lhs.type == rhs.type
               && lhs.rows == rhs.rows
                  && lhs.cols == rhs.cols;
}

class VideoCaptureActor : public event_based_actor{
    VideoCapture cap;
    actor buddy;
protected:
    behavior make_behavior() override {
        send(this,get_atom::value);
        return {
                [=](get_atom){
                    while(true){
                        Mat frame;
                        cap >> frame;
                        this->send(buddy,put_atom::value,Image(frame));
                        if(waitKey(60) >= 0){
                            send(this,ok_atom::value);
                            break;
                        }
                    }
                },
                [=](ok_atom){
                    cout << "Hello "<< buddy.id() <<" !"<<endl;
                },
                others >> [=](){
                    cerr << "unexpected: " << to_string(this->current_message()) << buddy.id() << endl;
                }

        };
    }

public:
    VideoCaptureActor(const actor &buddy){
        this->buddy = buddy;
        cap.open(0);
        if(!cap.isOpened())
            throw -1;
    }
};

class CannyActor : public event_based_actor {
    Mat edges;
protected:
    behavior make_behavior() override {
        return {
                [=](put_atom,Image image){
                    cvtColor(image.toMat(), edges, CV_BGR2GRAY);
                    GaussianBlur(edges, edges, Size(7,7), 1.5, 1.5);
                    Canny(edges, edges, 0, 30, 3);
                    imshow("edges", edges);
                },
                others >> [=] {
                    cerr << "unexpected: " << to_string(this->current_message()) << endl;
                }

        };
    }
public:
    CannyActor(){
        namedWindow("edges",1);
    }
};

int main(int, char**) {
    announce<Image>("Image",&Image::data,&Image::type,&Image::rows,&Image::cols);
    try {
        spawn<VideoCaptureActor>(spawn<CannyActor>());
    }catch(int x){
        cerr<<x<<endl;
    }
    await_all_actors_done();
    shutdown();
    return 0;
}

CMakeLists.txt

cmake_minimum_required(VERSION 3.2)
project(SmartVision CXX)

set (CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR})

find_package(OpenCV REQUIRED)
find_package(Libcaf COMPONENTS core io REQUIRED)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11  -Wextra -Wall -pedantic")
set(CMAKE_CXX_FLAGS_DEBUG          "-O0 -g")
set(CMAKE_CXX_FLAGS_MINSIZEREL     "-Os")
set(CMAKE_CXX_FLAGS_RELEASE        "-O4")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")

include_directories(${OpenCV_INCLUDE_DIRS} ${LIBCAF_INCLUDE_DIRS})
set(SOURCE_FILES main.cpp)
add_executable(SmartVision ${SOURCE_FILES})
target_link_libraries(SmartVision ${OpenCV_LIBS} ${LIBCAF_LIBRARIES})

I placed also the FindLibcaf.cmake file in my project root folder in order to enable cmake to find the CAF libraries

I am using MacOS X Yosemite with CLion 1.0.2 as IDE When running the program use a lot of CPU ressources: I have heard the fan of my macbook for the first time when it runs ! CAF is supposed to be a lightweight framework but perhaps I am using it in the wrong way.

Answer 1

Disclaimer: I have no experience with OpenCV. So I can only help figuring out what's happening on the CAF side of things.

Actors are supposed to be asynchronous, nonblocking and cooperative. The while (true) loop in VideoCaptureActor blocks a worker thread in the scheduler of CAF. Is it safe to call imshow in a different thread each time? Because this is what can end up happening to CannyActor. So it might just boil down to threading issues in OpenCV.

The first thing I would suggest doing is:

spawn<VideoCaptureActor, detached>(spawn<CannyActor, detached>());

This will assign a dedicated thread to each of your two actors.

As long as you're not sending the image over network, it is safe to just send the Mat directly (and not announce it) for now to see if the back-and-forth conversion with Image broke something. You can also write a custom serializer at some point in the future that allows you to serialize/deserialize Mat directly.

[=](get_atom) {
  while (true) {
    Mat frame;
    cap >> frame;
    send(buddy, put_atom::value, std::move(frame));
    if(waitKey(60) >= 0){
      send(this, ok_atom::value);
      break;
    }
  }
},

In case Mat has no move constructor, you could store the frame in a std::shared_ptr<Mat> or something like that to get rid of the extra copy completely (at the extend of an additional heap allocation + indirection).

I hope that helps.