ISampleGrabberCB sampleCB() not getting called when using DirectShow

Question

I'm trying to capture Video frames from USB Video Device(UVC) using DirectShow. Actually comparing Media Foundation with DirectShow because I'm seeing around 11 milliseconds buffering when using Media Foundation hence not getting actual frame rate. Want to isolate the 11 ms buffering to Media Foundation or low level usbVideo.sys driver, hence using Directshow to see if I get the same 11 ms buffering issue.

To measure frame rate I have registered CB using ISampleGrabberCB. But for some reason teh CB is not getting called. I have checked for error.

Is this the right way of getting hold of frame buffer?

#include <iostream>
#include <windows.h>
#include <dshow.h>
#include "qedit.h"

#pragma comment(lib, "strmiids.lib")

void EnumerateVideoDevices() {
    // Initialize COM
    CoInitialize(NULL);

    // Create System Device Enumerator
    ICreateDevEnum* pCreateDevEnum = nullptr;
    CoCreateInstance(CLSID_SystemDeviceEnum, NULL, CLSCTX_INPROC_SERVER, IID_ICreateDevEnum, (void**)&pCreateDevEnum);

    // Create EnumMoniker
    IEnumMoniker* pEnumMoniker = nullptr;
    pCreateDevEnum->CreateClassEnumerator(CLSID_VideoInputDeviceCategory, &pEnumMoniker, 0);

    // Check if there are devices
    if (pEnumMoniker != nullptr) {
        IMoniker* pMoniker = nullptr;

        // Display available devices
        int index = 1;
        while (pEnumMoniker->Next(1, &pMoniker, NULL) == S_OK) {
            IPropertyBag* pPropBag;
            pMoniker->BindToStorage(0, 0, IID_IPropertyBag, (void**)&pPropBag);

            // Retrieve device name
            VARIANT var;
            VariantInit(&var);
            pPropBag->Read(L"FriendlyName", &var, 0);
            std::wcout << index << L". " << var.bstrVal << std::endl;

            // Release resources
            VariantClear(&var);
            pPropBag->Release();
            pMoniker->Release();

            index++;
        }

        // Release enumerator
        pEnumMoniker->Release();
    }

    // Release COM objects
    pCreateDevEnum->Release();
    CoUninitialize();
}

IBaseFilter* ChooseVideoDevice() {
    // Prompt the user to choose a video device
    std::cout << "Choose a video device by entering its index: ";
    int selectedIndex;
    std::cin >> selectedIndex;

    // Initialize COM
    CoInitialize(NULL);

    // Create System Device Enumerator
    ICreateDevEnum* pCreateDevEnum = nullptr;
    CoCreateInstance(CLSID_SystemDeviceEnum, NULL, CLSCTX_INPROC_SERVER, IID_ICreateDevEnum, (void**)&pCreateDevEnum);

    // Create EnumMoniker
    IEnumMoniker* pEnumMoniker = nullptr;
    pCreateDevEnum->CreateClassEnumerator(CLSID_VideoInputDeviceCategory, &pEnumMoniker, 0);

    // Check if there are devices
    if (pEnumMoniker != nullptr) {
        IMoniker* pMoniker = nullptr;

        // Find the selected device
        int index = 1;
        while (pEnumMoniker->Next(1, &pMoniker, NULL) == S_OK) {
            if (index == selectedIndex) {
                // Bind to the selected device
                IBaseFilter* pVideoCaptureFilter = nullptr;
                pMoniker->BindToObject(NULL, NULL, IID_IBaseFilter, (void**)&pVideoCaptureFilter);

                // Release enumerator
                pEnumMoniker->Release();
                pCreateDevEnum->Release();
                CoUninitialize();

                return pVideoCaptureFilter;
            }

            pMoniker->Release();
            index++;
        }

        // Release enumerator
        pEnumMoniker->Release();
    }

    // Release COM objects
    pCreateDevEnum->Release();
    CoUninitialize();

    return nullptr;
}

IPin* GetOutPin(IBaseFilter* pFilter, PIN_DIRECTION PinDir) {
    IEnumPins* pEnum;
    IPin* pPin = NULL;

    if (SUCCEEDED(pFilter->EnumPins(&pEnum))) {
        ULONG fetched;
        while (pEnum->Next(1, &pPin, &fetched) == S_OK) {
            PIN_DIRECTION ThisPinDir;
            if (SUCCEEDED(pPin->QueryDirection(&ThisPinDir)) && (ThisPinDir == PinDir)) {
                break;
            }
            pPin->Release();
        }
        pEnum->Release();
    }

    return pPin;
}

IPin* GetInPin(IBaseFilter* pFilter, PIN_DIRECTION PinDir) {
    IEnumPins* pEnum;
    IPin* pPin = NULL;

    if (SUCCEEDED(pFilter->EnumPins(&pEnum))) {
        ULONG fetched;
        while (pEnum->Next(1, &pPin, &fetched) == S_OK) {
            PIN_DIRECTION ThisPinDir;
            if (SUCCEEDED(pPin->QueryDirection(&ThisPinDir)) && (ThisPinDir == PinDir)) {
                break;
            }
            pPin->Release();
        }
        pEnum->Release();
    }

    return pPin;
}


class CallbackObject : public ISampleGrabberCB {
public:

    CallbackObject() {};

    STDMETHODIMP QueryInterface(REFIID riid, void** ppv)
    {
        if (NULL == ppv) return E_POINTER;
        if (riid == __uuidof(IUnknown)) {
            *ppv = static_cast<IUnknown*>(this);
            return S_OK;
        }
        if (riid == IID_ISampleGrabberCB) {
            *ppv = static_cast<ISampleGrabberCB*>(this);
            return S_OK;
        }
        return E_NOINTERFACE;
    }
    STDMETHODIMP_(ULONG) AddRef() { return S_OK; }
    STDMETHODIMP_(ULONG) Release() { return S_OK; }

    //ISampleGrabberCB
    STDMETHODIMP SampleCB(double SampleTime, IMediaSample* pSample);
    STDMETHODIMP BufferCB(double SampleTime, BYTE* pBuffer, long BufferLen) { return S_OK; }
};

STDMETHODIMP CallbackObject::SampleCB(double SampleTime, IMediaSample* pSample)
{
    if (!pSample)
        return E_POINTER;
    long sz = pSample->GetActualDataLength();
    BYTE* pBuf = NULL;
    pSample->GetPointer(&pBuf);
    if (sz <= 0 || pBuf == NULL) return E_UNEXPECTED;
    for (int i = 0;i < sz;i += 2)
        pBuf[i] = 255 - pBuf[i];
    pSample->Release();
    return S_OK;
}

int main() {
    // Enumerate video devices
    std::cout << "Available Video Devices:" << std::endl;
    EnumerateVideoDevices();

    // Choose a video device
    IBaseFilter* pVideoCaptureFilter = ChooseVideoDevice();
    if (pVideoCaptureFilter == nullptr) {
        std::cerr << "Failed to choose a video device." << std::endl;
        return 1;
    }
    
    HRESULT hr;

    // Initialize COM
    CoInitialize(NULL);

    // Create the Filter Graph Manager
    IGraphBuilder* pGraph = nullptr;
    CoCreateInstance(CLSID_FilterGraph, NULL, CLSCTX_INPROC_SERVER, IID_IGraphBuilder, (void**)&pGraph);

    // Add the video capture filter to the filter graph
    hr = pGraph->AddFilter(pVideoCaptureFilter, L"Video Capture");
    if (FAILED(hr)) {
        std::cerr << "Failed to pVideoCaptureFilter. Error: " << hr << std::endl;
    }

    // Create the Capture Graph Builder
    ICaptureGraphBuilder2* pCaptureGraphBuilder = nullptr;
    CoCreateInstance(CLSID_CaptureGraphBuilder2, NULL, CLSCTX_INPROC_SERVER, IID_ICaptureGraphBuilder2, (void**)&pCaptureGraphBuilder);

    // Set the filter graph for the capture graph builder
    pCaptureGraphBuilder->SetFiltergraph(pGraph);

    // Render the video stream
    pCaptureGraphBuilder->RenderStream(&PIN_CATEGORY_PREVIEW, &MEDIATYPE_Video, pVideoCaptureFilter, NULL, NULL);

    // Create the SampleGrabber filter
    IBaseFilter* pSampleGrabberFilter = nullptr;
    CoCreateInstance(CLSID_SampleGrabber, NULL, CLSCTX_INPROC_SERVER, IID_IBaseFilter, (void**)&pSampleGrabberFilter);

    // Add the SampleGrabber filter to the graph
    hr = pGraph->AddFilter(pSampleGrabberFilter, L"Sample Grabber");
    if (FAILED(hr)) {
        std::cerr << "Failed to pSampleGrabberFilter. Error: " << hr << std::endl;
    }

    // Connect the filters
    IPin* pCapturePin = GetOutPin(pVideoCaptureFilter, PINDIR_OUTPUT);
    IPin* pSamplePin = GetInPin(pSampleGrabberFilter, PINDIR_INPUT);
    pGraph->Connect(pCapturePin, pSamplePin);

    // Configure SampleGrabber to receive callbacks
    ISampleGrabber* pSampleGrabber = nullptr;
    pSampleGrabberFilter->QueryInterface(IID_ISampleGrabber, (void**)&pSampleGrabber);


    AM_MEDIA_TYPE mt;
    ZeroMemory(&mt, sizeof(AM_MEDIA_TYPE));
    mt.majortype = MEDIATYPE_Video;
    mt.subtype = MEDIASUBTYPE_YUY2;
    hr = pSampleGrabber->SetMediaType(&mt);
    if (FAILED(hr)) {
        std::cerr << "Failed to SetMediaType. Error: " << hr << std::endl;
    }
    //hr = pSampleGrabber->SetOneShot(FALSE);
    //hr = pSampleGrabber->SetBufferSamples(TRUE);
    CallbackObject* pCallback = new CallbackObject();
    pSampleGrabber->SetCallback(pCallback, 0);

    // Run the graph to start capturing
    IMediaControl* pMediaControl = nullptr;
    pGraph->QueryInterface(IID_IMediaControl, (void**)&pMediaControl);
    pMediaControl->Run();


    // Wait for user input to stop capturing
    std::cout << "Press Enter to stop capturing." << std::endl;
    std::cin.ignore();
    std::cin.get();

    // Stop capturing
    pMediaControl->Stop();

    // Release COM objects
    pMediaControl->Release();
    pCaptureGraphBuilder->Release();
    pGraph->Release();
    pVideoCaptureFilter->Release();
    pSampleGrabber->Release();
    pSampleGrabberFilter->Release();
    delete pCallback; // Release the callback object

    // Uninitialize COM
    CoUninitialize();

    return 0;
}

The video from Laptop's Integrated Webcam seem to be getting captured and rendered but the SampleCB() function doesn't seem to be getting called. Any pointers what I'm doing wrong?

Thanks, Vinay

Answer 1

There are a few problems with your code including that it does not build on its own and use of COM is inaccurate.

Still, the easiest way to - sort of - fix is to replace your cin with a MessageBox call (because in this thread you are responsible for window message delivery as video capture goes):

    //std::cout << "Press Enter to stop capturing." << std::endl;
    //std::cin.ignore();
    //std::cin.get();
    MessageBoxW(GetActiveWindow(), L"Close to stop capturing", nullptr, MB_OK);

At this point the setup will become more or less usable and you will be able to find out that you are creating a filter graph like this, where your Sample Grabber is outside of your pipeline:

To solve this your code needs a bit of patching. I commented a few calls you don't need and added one line. From there you have your callback.

    // Render the video stream
    //pCaptureGraphBuilder->RenderStream(&PIN_CATEGORY_PREVIEW, &MEDIATYPE_Video, pVideoCaptureFilter, NULL, NULL);

    // Create the SampleGrabber filter
    IBaseFilter* pSampleGrabberFilter = nullptr;
    CoCreateInstance(__uuidof(SampleGrabber), NULL, CLSCTX_INPROC_SERVER, IID_IBaseFilter, (void**)&pSampleGrabberFilter);

    // Add the SampleGrabber filter to the graph
    hr = pGraph->AddFilter(pSampleGrabberFilter, L"Sample Grabber");
    if (FAILED(hr)) {
        std::cerr << "Failed to pSampleGrabberFilter. Error: " << hr << std::endl;
    }

    // Connect the filters
    //IPin* pCapturePin = GetOutPin(pVideoCaptureFilter, PINDIR_OUTPUT);
    //IPin* pSamplePin = GetInPin(pSampleGrabberFilter, PINDIR_INPUT);
    //pGraph->Connect(pCapturePin, pSamplePin);

    // Configure SampleGrabber to receive callbacks
    ISampleGrabber* pSampleGrabber = nullptr;
    pSampleGrabberFilter->QueryInterface(__uuidof(ISampleGrabber), (void**)&pSampleGrabber);


    AM_MEDIA_TYPE mt;
    ZeroMemory(&mt, sizeof(AM_MEDIA_TYPE));
    mt.majortype = MEDIATYPE_Video;
    mt.subtype = MEDIASUBTYPE_YUY2;
    hr = pSampleGrabber->SetMediaType(&mt);
    if (FAILED(hr)) {
        std::cerr << "Failed to SetMediaType. Error: " << hr << std::endl;
    }
    //hr = pSampleGrabber->SetOneShot(FALSE);
    //hr = pSampleGrabber->SetBufferSamples(TRUE);
    CallbackObject* pCallback = new CallbackObject();
    pSampleGrabber->SetCallback(pCallback, 0);

    pCaptureGraphBuilder->RenderStream(&PIN_CATEGORY_PREVIEW, &MEDIATYPE_Video, pVideoCaptureFilter, pSampleGrabberFilter, NULL); // <<--- Added

It makes it something like this: