Is there min size of IMFSample when ProcessInput?

Question

• now I am using media foundation to process h264 data and it is live data.
• receive data by socket
• MFCreateMemoryBuffer to create IMFMediaBuffer and copy the data to the buffer
• MFCreateSample
• ProcessInput to pass the sample created.
• get outputData.

now my question is when I first receive data and ProcessInput it is all right. but then I will receive some small data (may be only 3kb?) and the image is not right and will lack something(only part of the whole right image).

so I think it does not receive the whole data needed to to IMFTransform so I will keep the data until it is 30kb(or other larger) and then go to MFCreateMemoryBuffer.

yes now the image frame is right until the last frame.

so I guess I do not get the whole right data of one Frame right? how to get the right size of one completely IMFSample(Frame)?

I do as @Simon Mourier 's code but there is error image.

Answer 1

Here is a sample code that decodes an H264 stream of bytes using Media Foundation H.264 Video Decoder transform.

It simulates the stream of various-sized chunks of bytes using a file but the principles are the same.

Key points:

• You can feed chunks with any number of bytes as input to the transform
• The transform needs fixed-sized output samples
• The transform cannot allocate samples by itself
• Initially nobody knows the samples required size, so we have to feed the transform with bytes until it tells us what the samples size must be.

#include <windows.h>
#include <atlbase.h>
#include <mfapi.h>
#include <mferror.h>
#include <mfidl.h>
#include <mftransform.h>
#include <cstdlib>

#pragma comment(lib, "mfplat.lib")
#pragma comment(lib, "mfuuid.lib")

#define HRCHECK(__expr) {auto __hr=(__expr);if(FAILED(__hr)){wprintf(L"FAILURE 0x%08X (%i)\n\tline: %u file: '%s'\n\texpr: '" _CRT_WIDE(#__expr) L"'\n",__hr,__hr,__LINE__,_CRT_WIDE(__FILE__));_CrtDbgBreak();}}
#define WIN32CHECK(__expr) {if(!(__expr)){auto __hr=HRESULT_FROM_WIN32(GetLastError());{wprintf(L"FAILURE 0x%08X (%i)\n\tline: %u file: '%s'\n\texpr: '" _CRT_WIDE(#__expr) L"'\n",__hr,__hr,__LINE__,_CRT_WIDE(__FILE__));_CrtDbgBreak();}}}

static HRESULT SetOutputType(IMFTransform* transform, GUID format)
{
    DWORD index = 0;
    do
    {
        CComPtr<IMFMediaType> outputType;
        auto hr = transform->GetOutputAvailableType(0, index++, &outputType);
        if (FAILED(hr))
            return hr;

        GUID guid;
        if (SUCCEEDED(outputType->GetGUID(MF_MT_SUBTYPE, &guid)) && guid == format)
        {
            HRCHECK(transform->SetOutputType(0, outputType, 0));
            return S_OK;
        }
    } while (true);
}

int main()
{
    HRCHECK(CoInitialize(nullptr));
    {
        HRCHECK(MFStartup(MF_VERSION));
        // open file
        auto file = CreateFile(L"h264.h264", GENERIC_READ, 0, nullptr, OPEN_EXISTING, 0, nullptr);
        WIN32CHECK(file != INVALID_HANDLE_VALUE);

        // create H264 transform https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-decoder
        CComPtr<IMFTransform> decoder;
        HRCHECK(decoder.CoCreateInstance(CLSID_MSH264DecoderMFT));

        // You can check in decoder attributes that MF_MT_FIXED_SIZE_SAMPLES is set to TRUE.
        // Calling GetOutputStreamInfo this will tell you the MFT cannot provide samples
        // as MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES and MFT_OUTPUT_STREAM_PROVIDES_SAMPLES are not set

        // So we don't know enough information yet, we'll feed input samples until we get MF_E_TRANSFORM_STREAM_CHANGE and then we'll provide a sample as per doc:
        //   "If the input type contains only these two attributes, the decoder will offer a default output type, which acts as a placeholder."
        //   "When the decoder receives enough input samples to produce an output frame, it signals a format change by returning MF_E_TRANSFORM_STREAM_CHANGE"
        UINT32 sampleSize = 0;

        // input type is H264
        CComPtr<IMFMediaType> inputType;
        HRCHECK(MFCreateMediaType(&inputType));
        HRCHECK(inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
        HRCHECK(inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264));
        HRCHECK(decoder->SetInputType(0, inputType, 0)); // video is id 0

        // get (and set) NV12 output type (could be I420, IYUV, YUY2, YV12)
        HRCHECK(SetOutputType(decoder, MFVideoFormat_NV12));

        do
        {
            // get a random chunk size between 500 and 1500
            DWORD chunkSize = 500 + (1000 * (RAND_MAX - std::rand())) / RAND_MAX;

            // create an MF input buffer & read into it
            CComPtr<IMFMediaBuffer> inputBuffer;
            HRCHECK(MFCreateMemoryBuffer(chunkSize, &inputBuffer));
            BYTE* chunk;
            HRCHECK(inputBuffer->Lock(&chunk, nullptr, nullptr));
            DWORD read;
            WIN32CHECK(ReadFile(file, chunk, chunkSize, &read, nullptr));
            HRCHECK(inputBuffer->SetCurrentLength(read));
            HRCHECK(inputBuffer->Unlock());
            if (read)
            {
                CComPtr<IMFSample> inputSample;
                HRCHECK(MFCreateSample(&inputSample));
                HRCHECK(inputSample->AddBuffer(inputBuffer));

                auto hr = decoder->ProcessInput(0, inputSample, 0);
                if (hr != MF_E_NOTACCEPTING) // just go on
                {
                    HRCHECK(hr);
                }
            }
            else
            {
                // end of file, ask decoder to process all data from previous calls
                HRCHECK(decoder->ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0));
            }

            CComPtr<IMFSample> outputSample;
            HRCHECK(MFCreateSample(&outputSample));
            MFT_OUTPUT_DATA_BUFFER outputBuffer{};
            outputBuffer.pSample = outputSample;

            if (sampleSize)
            {
                // now we know the size so we can (and must) allocate the MF output buffer
                CComPtr<IMFMediaBuffer> outputBuffer;
                HRCHECK(MFCreateMemoryBuffer(sampleSize, &outputBuffer));
                HRCHECK(outputSample->AddBuffer(outputBuffer));
            } // else just continue to process

            DWORD status = 0;
            auto hr = decoder->ProcessOutput(0, 1, &outputBuffer, &status);
            if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) // just go on
            {
                if (!read) // file is all read
                    break;

                continue;
            }

            // https://learn.microsoft.com/en-us/windows/win32/medfound/handling-stream-changes
            if (hr == MF_E_TRANSFORM_STREAM_CHANGE)
            {
                // get (and set) NV12 output type (could be I420, IYUV, YUY2, YV12)
                HRCHECK(SetOutputType(decoder, MFVideoFormat_NV12));

                // now get the sample size
                CComPtr<IMFMediaType> type;
                HRCHECK(decoder->GetOutputCurrentType(0, &type));
                HRCHECK(type->GetUINT32(MF_MT_SAMPLE_SIZE, &sampleSize));
                continue;
            }
            HRCHECK(hr);

            LONGLONG time, duration;
            HRCHECK(outputSample->GetSampleTime(&time));
            HRCHECK(outputSample->GetSampleDuration(&duration));
            wprintf(L"Sample time: %I64u ms duration: %I64u ms\n", time / 10000, duration / 10000);
        } while (true);

        // close file
        CloseHandle(file);
        HRCHECK(MFShutdown());
    }
    CoUninitialize();
    return 0;
}

Full project is available here https://github.com/smourier/MFDecodeH264