converting xtensor xarray to opencv mat back and forth in cpp

Question

I am looking for a fast and reliable way to convert xt::xarray <-> cv::mat / cv::mat1f. In my specific case, I what it for 2 dimensions and datatype float, but more general answers are welcome, of course!

At the moment, I have an elementwise solution. This is quite robust, but I would appreciate a faster one. I have some trouble to implement this one.

Elementwise solutions

#include <iostream>

#include <opencv2/opencv.hpp>

#include "xtensor/xarray.hpp"
#include "xtensor/xio.hpp"
#include "xtensor/xadapt.hpp"


cv::Mat xarray_to_mat_elementwise(xt::xarray<float> xarr)
{
    int ndims = xarr.dimension();
    assert(ndims == 2  && "can only convert 2d xarrays");
    int nrows = xarr.shape()[0];
    int ncols = xarr.shape()[1];
    cv::Mat mat(nrows, ncols, CV_32FC1);
    for (int rr=0; rr<nrows; rr++)
    {
        for (int cc=0; cc<ncols; cc++)
        {
            mat.at<float>(rr, cc) = xarr(rr, cc);
        }
    }
    return mat;
}

xt::xarray<float> mat_to_xarray_elementwise(cv::Mat mat)
{
    int ndims = mat.dims;
    assert(ndims == 2  && "can only convert 2d xarrays");
    int nrows = mat.rows;
    int ncols = mat.cols;
    xt::xarray<float> xarr = xt::empty<float>({nrows, ncols});
    for (int rr=0; rr<nrows; rr++)
    {
        for (int cc=0; cc<ncols; cc++)
        {
            xarr(rr, cc) = mat.at<float>(rr, cc);
        }
    }
    return xarr;
}



int main()
{
    int nrows = 2, ncols = 3;
    float data[150];
    for (int i=0; i<nrows * ncols; i++)
    {
        data[i] = .1 * i;
    }

    cv::Mat mat (nrows, ncols, CV_32FC1, data, 0);
    std::cout << "mat:\n" << mat << std::endl;

    xt::xarray<float> xarr = xt::adapt(
        (float*) data, nrows * ncols, xt::no_ownership(), std::vector<std::size_t> {nrows, ncols});
    std::cout << "xarr:\n" << xarr << std::endl;

    cv::Mat mat2_ew = xarray_to_mat_elementwise(xarr);
    std::cout << "mat2_ew (from xt::xarray):\n" << mat2_ew << std::endl;

    xt::xarray<float> xarr2_ew = mat_to_xarray_elementwise(mat);
    std::cout << "xarr2_ew (from cv::mat):\n" << xarr2_ew << std::endl;

    return 0;
}

output

mat:
[0, 0.1, 0.2;
 0.30000001, 0.40000001, 0.5]
xarr:
{{ 0.      ,  0.1     ,  0.2     },
 { 0.3     ,  0.4     ,  0.5     }}
mat2_ew (from xt::xarray):
[0, 0.1, 0.2;
 0.30000001, 0.40000001, 0.5]
xarr2_ew (from cv::mat):
{{ 0.      ,  0.1     ,  0.2     },
 { 0.3     ,  0.4     ,  0.5     }}

Answer 1

Solution based on pointers

This converts xt::xarray <-> cv::mat(1f) based on the pointers to the underlying data of the source object. No copying of data is necessary, but deallocation of the memory may be necessary.

Pointer to the user data. Matrix constructors that take data and step parameters do not allocate matrix data. Instead, they just initialize the matrix header that points to the specified data, which means that no data is copied. This operation is very efficient and can be used to process external data using OpenCV functions. The external data is not automatically deallocated, so you should take care of it.

Compare the documentation.

Do not use call by value, when converting: If wrapping the converter in a function (compare xarray_to_mat), be carefull to use a call by reference. Otherwise the pointer xarr.data() is pointing to a non-allocated memory position.

xt::layout_type::column_major: Of course the column_major layout in xtensor makes some trouble. Maybe it makes sense to fall back to the elementwise assignment in this case.

#include <iostream>
#include <opencv2/opencv.hpp>
#include "xtensor/xarray.hpp"
#include "xtensor/xio.hpp"
#include "xtensor/xadapt.hpp"

/**
 * Converts xt::xarray to cv::mat.
 *
 * First few elements are wrong (not reproducible).
 * This can be repaired by using call by reference: xt::xarray<float> xarr -> xt::xarray<float> & xarr
 */
cv::Mat xarray_to_mat(xt::xarray<float> xarr)
{
    cv::Mat mat (xarr.shape()[0], xarr.shape()[1], CV_32FC1, xarr.data(), 0);
    return mat;
}

xt::xarray<float> mat_to_xarray(cv::Mat mat)
{
    xt::xarray<float> res = xt::adapt(
        (float*) mat.data, mat.cols * mat.rows, xt::no_ownership(), std::vector<std::size_t> {mat.rows, mat.cols});
    return res;
}

cv::Mat xarray_to_mat_elementwise(xt::xarray<float> xarr)
{
    int ndims = xarr.dimension();
    assert(ndims == 2  && "can only convert 2d xarrays");
    int nrows = xarr.shape()[0];
    int ncols = xarr.shape()[1];
    cv::Mat mat(nrows, ncols, CV_32FC1);
    for (int rr=0; rr<nrows; rr++)
    {
        for (int cc=0; cc<ncols; cc++)
        {
            mat.at<float>(rr, cc) = xarr(rr, cc);
        }
    }
    return mat;
}

xt::xarray<float> mat_to_xarray_elementwise(cv::Mat mat)
{
    int ndims = mat.dims;
    assert(ndims == 2  && "can only convert 2d xarrays");
    int nrows = mat.rows;
    int ncols = mat.cols;
    xt::xarray<float> xarr = xt::empty<float>({nrows, ncols});
    for (int rr=0; rr<nrows; rr++)
    {
        for (int cc=0; cc<ncols; cc++)
        {
            xarr(rr, cc) = mat.at<float>(rr, cc);
        }
    }
    return xarr;
}

int main()
{
    int nrows = 2, ncols = 3;
    float data[150];
    for (int i=0; i<nrows * ncols; i++)
    {
        data[i] = .1 * i;
    }

    cv::Mat mat (nrows, ncols, CV_32FC1, data, 0);
    std::cout << "mat:\n" << mat << std::endl;

    xt::xarray<float> xarr = xt::adapt(
        (float*) data, nrows * ncols, xt::no_ownership(), std::vector<std::size_t> {nrows, ncols});
    std::cout << "xarr:\n" << xarr << std::endl;

    cv::Mat mat2 (nrows, ncols, CV_32FC1, xarr.data(), 0);
    std::cout << "mat2 (from xt::xarray, works):\n" << mat2 << std::endl;

    cv::Mat mat3 = xarray_to_mat(xarr);
    std::cout << "mat3 (from xt::xarray, call by value -> fails):\n" << mat3 << std::endl;

    xt::xarray<float> xarr2 = mat_to_xarray(mat);
    std::cout << "xarr2 (from cv::mat):\n" << xarr2 << std::endl;

    std::cout << "\n=========== works for cv::mat1f analoguous ===========\n" << std::endl;

    cv::Mat1f mat_1f (nrows, ncols, data, 0);
    std::cout << "mat_1f:\n" << mat_1f << std::endl;

    cv::Mat1f mat_1f_2 (nrows, ncols, (float*) xarr.data(), 0);
    std::cout << "mat_1f_2 (from xt::xarray, works):\n" << mat_1f_2 << std::endl;

    std::cout << "\n=========== column_major layout in xtensor ===========\n" << std::endl;

    xt::xarray<float, xt::layout_type::column_major> xarr_cm = xt::adapt(
        (float*) data, nrows * ncols, xt::no_ownership(), std::vector<std::size_t> {nrows, ncols});
    std::cout << "xarr_cm:\n" << xarr_cm << std::endl;

    cv::Mat mat_cm (nrows, ncols, CV_32FC1, xarr_cm.data(), 0);
    std::cout << "mat_cm (from xt::xarray, pointer based -> fails):\n" << mat_cm << std::endl;

    cv::Mat mat_cm_ew = xarray_to_mat_elementwise(xarr_cm);
    std::cout << "mat_cm_ew (from xt::xarray, elementwise -> works):\n" << mat_cm_ew << std::endl;

    return 0;
}

** output **

mat:
[0, 0.1, 0.2;
 0.30000001, 0.40000001, 0.5]
xarr:
{{ 0.      ,  0.1     ,  0.2     },
 { 0.3     ,  0.4     ,  0.5     }}
mat2 (from xt::xarray, works):
[0, 0.1, 0.2;
 0.30000001, 0.40000001, 0.5]
mat3 (from xt::xarray, call by value -> fails):
[4.3654774e-38, 0, 0.2;
 0.30000001, 0.40000001, 0.5]
xarr2 (from cv::mat):
{{ 0.      ,  0.1     ,  0.2     },
 { 0.3     ,  0.4     ,  0.5     }}

=========== works for cv::mat1f analoguous ===========

mat_1f:
[0, 0.1, 0.2;
 0.30000001, 0.40000001, 0.5]
mat_1f_2 (from xt::xarray, works):
[0, 0.1, 0.2;
 0.30000001, 0.40000001, 0.5]

=========== column_major layout in xtensor ===========

xarr_cm:
{{ 0.      ,  0.1     ,  0.2     },
 { 0.3     ,  0.4     ,  0.5     }}
mat_cm (from xt::xarray, pointer based -> fails):
[0, 0.30000001, 0.1;
 0.40000001, 0.2, 0.5]
mat_cm_ew (from xt::xarray, elementwise -> works):
[0, 0.1, 0.2;
 0.30000001, 0.40000001, 0.5]

Answer 2

If the Mat passed to mat_to_xarray is not a float then the cast will be bad.. Converting the Mat to the same data type as your cast might make things a little safer. mat.convertTo(mat, CV_32F);