A continuous capture of a window sceen as QPixmap? for OpenCV

Question

I'm trying to get a live view of a specific window using Qt6.6. So far I have succeed to do that for the whole screen. However, when I implemented the code for a specific window I get stuck at capturing the window screen as QPixmap. The object is QWindowCapture but it doesn't have a method for capturing or something similar. Once I get that, I'll convert it into OpenCV format for further processing which is what I have done with screen capturing. That's my goal why I need capturing.

Here is the code if you want to try it:

#include <QApplication>
#include <QScreen>
#include <QPixmap>
#include <QImage>
#include <opencv2/opencv.hpp>
#include <QAbstractListModel>
#include <QCapturableWindow>
#include <QWindowCapture>

QT_USE_NAMESPACE

class WindowListModel : public QAbstractListModel
{
    Q_OBJECT

public:
    explicit WindowListModel(QObject *parent = nullptr);

    int rowCount(const QModelIndex &parent = QModelIndex()) const override;
    QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override;

    QCapturableWindow window(const QModelIndex &index) const;

public Q_SLOTS:
    void populate();

private:
    QList<QCapturableWindow> windowList;
};

class Screenshot {
public:
    void displayLiveView();

private:
    QPixmap grabScreen();
};

QPixmap Screenshot::grabScreen() {
    QScreen *screen = QGuiApplication::primaryScreen();
    if (!screen)
        return QPixmap();

    return screen->grabWindow(0);
}

void Screenshot::displayLiveView() {
    while (true) {
        QPixmap pixmap = grabScreen();
        if (pixmap.isNull()) {
            qDebug() << "Failed to capture the screen.";
            continue;
        }
        QImage image = pixmap.toImage();
        cv::Mat mat(image.height(), image.width(), CV_8UC4, (void*)image.constBits(), image.bytesPerLine());
        cv::imshow("Live View", mat);
        
        // Break if 'ESC' is pressed
        if (cv::waitKey(30) == 27) {
            break;
        }
    }
}

void captureAndShowWindow(const QString& windowName) {
    // Obtain a list of available windows
    WindowListModel windowListModel;
    windowListModel.populate();

    QCapturableWindow targetWindow; // Window we are looking for

    bool windowFound = false;
    for (int i = 0; i < windowListModel.rowCount(); i++) {
        QModelIndex index = windowListModel.index(i);
        if (index.isValid()) {
            QString currentWindowDesc = windowListModel.data(index, Qt::DisplayRole).toString();
            if (currentWindowDesc.contains(windowName)) {
                targetWindow = windowListModel.window(index);
                windowFound = true;
                break;
            }
        }
    }

    if (!windowFound) {
        qDebug() << "Window not found!";
        return;
    }

    // Capture the specific window
    QWindowCapture windowCapture;
    windowCapture.setWindow(targetWindow);
    //STUCK
}

int main(int argc, char *argv[]) {
    QApplication app(argc, argv);

    // Print available windows
    WindowListModel model;
    for (int i = 0; i < model.rowCount(); i++) {
        QModelIndex index = model.index(i);
        if (index.isValid()) {
            QString windowDesc = model.data(index, Qt::DisplayRole).toString();
            std::cout << i + 1 << ". " << windowDesc.toStdString() << std::endl;
        }
    }

    Screenshot screenshot;
    screenshot.displayLiveView();
    captureAndShowWindow("Settings");

    return 0;
}

The CMakeLists:

cmake_minimum_required(VERSION 3.10)

project(ScreenshotApp)

# Set up the required version of Qt 
set(CMAKE_AUTOMOC ON)
set(CMAKE_AUTORCC ON)
set(CMAKE_AUTOUIC ON)

include_directories(include)

list(APPEND CMAKE_PREFIX_PATH "/home/user/Qt/6.6.0/gcc_64/lib/cmake")

find_package(Qt6 COMPONENTS Widgets Multimedia REQUIRED)
find_package(OpenCV 4.7.0 REQUIRED)

set_property(SOURCE main.cpp PROPERTY SKIP_AUTOMOC ON)

add_executable(main main.cpp ${SOURCES})
target_link_libraries(main Qt6::Widgets ${OpenCV_LIBS} Qt6::Multimedia)

Now I'm working on Ubuntu but later the code should run on Windows.

Answer 1

The trick is to get targetWindowId and feed that to QScreen grabWindow. This will give you QPixmap format which is easily converted into OpenCV cv::Mat.

For that I picked a class which is called private in Qt source code because it is not part of their API.

class QCapturableWindowPrivate : public QSharedData {
public:
    using Id = size_t;

    QString description;
    Id id = 0;

    static const QCapturableWindowPrivate *handle(const QCapturableWindow &window)
    {
        return window.d.get();
    }

    QCapturableWindow create() { return QCapturableWindow(this); }
};

class Screenshot {
public:
    Screenshot(const QString& windowName);
    void displayLiveView();

private:
    QPixmap grabWindow();
    WId targetWindowId = 0;
    QWindowCapture windowCapture;
};

Screenshot::Screenshot(const QString& windowName) {
    WindowListModel windowListModel;
    windowListModel.populate();

    QCapturableWindow targetWindow; // Window we are looking for

    bool windowFound = false;
    for (int i = 0; i < windowListModel.rowCount(); i++) {
        QModelIndex index = windowListModel.index(i);
        if (index.isValid()) {
            QString currentWindowDesc = windowListModel.data(index, Qt::DisplayRole).toString();
            if (currentWindowDesc.contains(windowName)) {
                targetWindow = windowListModel.window(index);
                windowFound = true;
                break;
            }
        }
    }

    if (!windowFound) {
        qDebug() << "Window not found!";
        return;
    }

    windowCapture.setWindow(targetWindow);

    auto handle = QCapturableWindowPrivate::handle(targetWindow);
    targetWindowId = handle ? handle->id : 0;
}

QPixmap Screenshot::grabWindow() {
    if (targetWindowId == 0) return QPixmap();

    QScreen *screen = QGuiApplication::primaryScreen();
    if (!screen) return QPixmap();

    return screen->grabWindow(targetWindowId);
}

void Screenshot::displayLiveView() {
    while (true) {
        QPixmap pixmap = grabWindow();
        if (pixmap.isNull()) {
            qDebug() << "Failed to capture the window.";
            continue;
        }

        QImage image = pixmap.toImage();
        cv::Mat mat(image.height(), image.width(), CV_8UC4, (void*)image.constBits(), image.bytesPerLine());
        cv::imshow("Live View", mat);

        if (cv::waitKey(30) == 27) {
            break;
        }
    }
}

int main(int argc, char *argv[]) {
    QApplication app(argc, argv);

    // Print available windows
    WindowListModel model;
    for (int i = 0; i < model.rowCount(); i++) {
        QModelIndex index = model.index(i);
        if (index.isValid()) {
            QString windowDesc = model.data(index, Qt::DisplayRole).toString();
            std::cout << i + 1 << ". " << windowDesc.toStdString() << std::endl;
        }
    }

    Screenshot screenshot("Settings");
    screenshot.displayLiveView();

    return 0;
}