White border when using IDCompositionGaussianBlurEffect

Question

My main goal is to create Acrylic effect like in Windows UWP Settings UI in C++ for windows. For creating back visual I'm using private API DwmpCreateSharedVirtualDesktopVisual, this works. But now when I apply the gaussian effect using IDCompositionGaussianBlurEffect to the rootvisual that produces white borders like in the following image, I tried several things like scaling the root visual and translating to hide the borders but it's not working.

#include <Unknwn.h>
#include <Windows.h>
#include <comutil.h>
#include <d2d1_2.h>
#include <d2d1_2helper.h>
#include <d3d11_2.h>
#include <dcomp.h>
#include <dwmapi.h>
#include <dxgi1_3.h>
#include <ntstatus.h>
#include <winternl.h>
#include <wrl\implements.h>

void WINAPIV debug(const char* fmt, ...)
{
    char s[2048];
    va_list args;
    va_start(args, fmt);
    vsprintf(s, fmt, args);
    va_end(args);
    OutputDebugStringA(s);
}

//Either use this or add in your linker configuration
#pragma comment(lib, "dxgi")
#pragma comment(lib, "d3d11")
#pragma comment(lib, "d2d1")
#pragma comment(lib, "dcomp")
#pragma comment(lib, "dwmapi")
#pragma comment(lib, "ole32")

//-------------------- Definitions
enum THUMBNAIL_TYPE {
    TT_DEFAULT = 0x0,
    TT_SNAPSHOT = 0x1,
    TT_ICONIC = 0x2,
    TT_BITMAPPENDING = 0x3,
    TT_BITMAP = 0x4
};

typedef HRESULT(WINAPI* DwmpCreateSharedThumbnailVisual)(
    IN HWND hwndDestination,
    IN HWND hwndSource,
    IN DWORD dwThumbnailFlags,
    IN DWM_THUMBNAIL_PROPERTIES* pThumbnailProperties,
    IN VOID* pDCompDevice,
    OUT VOID** ppVisual,
    OUT PHTHUMBNAIL phThumbnailId);

typedef HRESULT(WINAPI* DwmpQueryWindowThumbnailSourceSize)(
    IN HWND hwndSource,
    IN BOOL fSourceClientAreaOnly,
    OUT SIZE* pSize);

typedef HRESULT(WINAPI* DwmpQueryThumbnailType)(
    IN HTHUMBNAIL hThumbnailId,
    OUT THUMBNAIL_TYPE* thumbType);

//pre-cobalt/pre-iron
typedef HRESULT(WINAPI* DwmpCreateSharedVirtualDesktopVisual)(
    IN HWND hwndDestination,
    IN VOID* pDCompDevice,
    OUT VOID** ppVisual,
    OUT PHTHUMBNAIL phThumbnailId);

//cobalt/iron (20xxx+)
//No changes except for the function name.
typedef HRESULT(WINAPI* DwmpCreateSharedMultiWindowVisual)(
    IN HWND hwndDestination,
    IN VOID* pDCompDevice,
    OUT VOID** ppVisual,
    OUT PHTHUMBNAIL phThumbnailId);

//pre-cobalt/pre-iron
typedef HRESULT(WINAPI* DwmpUpdateSharedVirtualDesktopVisual)(
    IN HTHUMBNAIL hThumbnailId,
    IN HWND* phwndsInclude,
    IN DWORD chwndsInclude,
    IN HWND* phwndsExclude,
    IN DWORD chwndsExclude,
    OUT RECT* prcSource,
    OUT SIZE* pDestinationSize);

//cobalt/iron (20xxx+)
//Change: function name + new DWORD parameter.
//Pass "1" in dwFlags. Feel free to explore other flags.
typedef HRESULT(WINAPI* DwmpUpdateSharedMultiWindowVisual)(
    IN HTHUMBNAIL hThumbnailId,
    IN HWND* phwndsInclude,
    IN DWORD chwndsInclude,
    IN HWND* phwndsExclude,
    IN DWORD chwndsExclude,
    OUT RECT* prcSource,
    OUT SIZE* pDestinationSize,
    IN DWORD dwFlags);

#define DWM_TNP_FREEZE 0x100000
#define DWM_TNP_ENABLE3D 0x4000000
#define DWM_TNP_DISABLE3D 0x8000000
#define DWM_TNP_FORCECVI 0x40000000
#define DWM_TNP_DISABLEFORCECVI 0x80000000

//We also need these if you want to use MultiWindow/VirtualDesktop.

enum WINDOWCOMPOSITIONATTRIB {
    WCA_UNDEFINED = 0x0,
    WCA_NCRENDERING_ENABLED = 0x1,
    WCA_NCRENDERING_POLICY = 0x2,
    WCA_TRANSITIONS_FORCEDISABLED = 0x3,
    WCA_ALLOW_NCPAINT = 0x4,
    WCA_CAPTION_BUTTON_BOUNDS = 0x5,
    WCA_NONCLIENT_RTL_LAYOUT = 0x6,
    WCA_FORCE_ICONIC_REPRESENTATION = 0x7,
    WCA_EXTENDED_FRAME_BOUNDS = 0x8,
    WCA_HAS_ICONIC_BITMAP = 0x9,
    WCA_THEME_ATTRIBUTES = 0xA,
    WCA_NCRENDERING_EXILED = 0xB,
    WCA_NCADORNMENTINFO = 0xC,
    WCA_EXCLUDED_FROM_LIVEPREVIEW = 0xD,
    WCA_VIDEO_OVERLAY_ACTIVE = 0xE,
    WCA_FORCE_ACTIVEWINDOW_APPEARANCE = 0xF,
    WCA_DISALLOW_PEEK = 0x10,
    WCA_CLOAK = 0x11,
    WCA_CLOAKED = 0x12,
    WCA_ACCENT_POLICY = 0x13,
    WCA_FREEZE_REPRESENTATION = 0x14,
    WCA_EVER_UNCLOAKED = 0x15,
    WCA_VISUAL_OWNER = 0x16,
    WCA_HOLOGRAPHIC = 0x17,
    WCA_EXCLUDED_FROM_DDA = 0x18,
    WCA_PASSIVEUPDATEMODE = 0x19,
    WCA_LAST = 0x1A,
};

typedef struct WINDOWCOMPOSITIONATTRIBDATA {
    WINDOWCOMPOSITIONATTRIB Attrib;
    void* pvData;
    DWORD cbData;
};

typedef BOOL(WINAPI* SetWindowCompositionAttribute)(
    IN HWND hwnd,
    IN WINDOWCOMPOSITIONATTRIBDATA* pwcad);

typedef BOOL(WINAPI* GetWindowCompositionAttribute)(
    IN HWND hwnd,
    OUT WINDOWCOMPOSITIONATTRIBDATA* pAttrData
    );

//------------------------- Getting functions
DwmpQueryThumbnailType lDwmpQueryThumbnailType;
DwmpCreateSharedThumbnailVisual lDwmpCreateSharedThumbnailVisual;
DwmpQueryWindowThumbnailSourceSize lDwmpQueryWindowThumbnailSourceSize;

//PRE-IRON
DwmpCreateSharedVirtualDesktopVisual lDwmpCreateSharedVirtualDesktopVisual;
DwmpUpdateSharedVirtualDesktopVisual lDwmpUpdateSharedVirtualDesktopVisual;

//20xxx+
DwmpCreateSharedMultiWindowVisual lDwmpCreateSharedMultiWindowVisual;
DwmpUpdateSharedMultiWindowVisual lDwmpUpdateSharedMultiWindowVisual;

SetWindowCompositionAttribute lSetWindowCompositionAttribute;
GetWindowCompositionAttribute lGetWindowCompositionAttribute;

bool InitPrivateDwmAPIs()
{
    auto dwmapiLib = LoadLibrary(L"dwmapi.dll");

    if (!dwmapiLib)
        return false;

    lDwmpQueryThumbnailType = (DwmpQueryThumbnailType)GetProcAddress(dwmapiLib, MAKEINTRESOURCEA(114));
    lDwmpCreateSharedThumbnailVisual = (DwmpCreateSharedThumbnailVisual)GetProcAddress(dwmapiLib, MAKEINTRESOURCEA(147));
    lDwmpQueryWindowThumbnailSourceSize = (DwmpQueryWindowThumbnailSourceSize)GetProcAddress(dwmapiLib, MAKEINTRESOURCEA(162));

    //PRE-IRON
    lDwmpCreateSharedVirtualDesktopVisual = (DwmpCreateSharedVirtualDesktopVisual)GetProcAddress(dwmapiLib, MAKEINTRESOURCEA(163));
    lDwmpUpdateSharedVirtualDesktopVisual = (DwmpUpdateSharedVirtualDesktopVisual)GetProcAddress(dwmapiLib, MAKEINTRESOURCEA(164));

    //20xxx+
    lDwmpCreateSharedMultiWindowVisual = (DwmpCreateSharedMultiWindowVisual)GetProcAddress(dwmapiLib, MAKEINTRESOURCEA(163));
    lDwmpUpdateSharedMultiWindowVisual = (DwmpUpdateSharedMultiWindowVisual)GetProcAddress(dwmapiLib, MAKEINTRESOURCEA(164));

    if (false) //Just a placeholder, don't.
        return false;

    return true;
}

bool InitPrivateUser32APIs()
{
    auto user32Lib = LoadLibrary(L"user32.dll");

    if (!user32Lib)
        return false;

    lSetWindowCompositionAttribute = (SetWindowCompositionAttribute)GetProcAddress(user32Lib, "SetWindowCompositionAttribute");

    if (!lSetWindowCompositionAttribute)
        return false;

    lGetWindowCompositionAttribute = (GetWindowCompositionAttribute)GetProcAddress(user32Lib, "GetWindowCompositionAttribute");
    assert(lGetWindowCompositionAttribute);

    return true;
}

//--------------------- Create Device func
using namespace Microsoft::WRL;

ComPtr<ID3D11Device> direct3dDevice;
ComPtr<IDXGIDevice> dxgiDevice;
ComPtr<ID2D1Factory> d2dFactory2;
ComPtr<IDCompositionDesktopDevice> dcompDevice;
ComPtr<IDCompositionDevice3> dcompDevice3;

#include <vector>
std::vector<HWND> windowList;
ComPtr<IDCompositionVisual2> rootVisual;
ComPtr<IDCompositionRectangleClip> rootClip;
std::vector<ComPtr<IDCompositionVisual>> visuals;
ComPtr<IDCompositionTranslateTransform> translateTransform;
ComPtr<IDCompositionGaussianBlurEffect> gaussianEffect;
HWND gHWND{};
RECT gHWNDRect{};

bool CreateDevice()
{
    if (D3D11CreateDevice(0, //Adapter
        D3D_DRIVER_TYPE_HARDWARE,
        NULL,
        D3D11_CREATE_DEVICE_BGRA_SUPPORT,
        NULL,
        0,
        D3D11_SDK_VERSION,
        direct3dDevice.GetAddressOf(),
        nullptr,
        nullptr)
        != S_OK) {
        //Maybe try creating with D3D_DRIVER_TYPE_WARP before returning false.
        //Always listen to device changes.
        return false;
    }

    if (direct3dDevice->QueryInterface(dxgiDevice.GetAddressOf()) != S_OK) {
        return false;
    }

    if (D2D1CreateFactory(
            D2D1_FACTORY_TYPE::D2D1_FACTORY_TYPE_SINGLE_THREADED,
            __uuidof(ID2D1Factory2),
            (void**)d2dFactory2.GetAddressOf())
        != S_OK) {
        return false;
    }

    if (DCompositionCreateDevice3(
        dxgiDevice.Get(),
        __uuidof(dcompDevice),
        (void**)dcompDevice.GetAddressOf())
        != S_OK) {
        return false;
    }

    if (dcompDevice->QueryInterface(dcompDevice3.GetAddressOf()) != S_OK)
        return false;

    return true;
}

LRESULT CALLBACK MyWndProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam);
HWND CreateWindowInternal(LPCWSTR lptitleName, LPCWSTR lpclassName, DWORD dwExStyle, DWORD dwStyle)
{
    HINSTANCE hInstance = GetModuleHandle(NULL);

    WNDCLASS wc = {};
    wc.lpfnWndProc = MyWndProc;
    wc.hInstance = hInstance;
    wc.cbWndExtra = 0;
    wc.lpszClassName = lpclassName;

    if (!RegisterClass(&wc))
        return nullptr;

    const auto hwndParent = CreateWindowEx(dwExStyle, lpclassName, lptitleName,
        dwStyle, CW_USEDEFAULT, CW_USEDEFAULT, 1000, 640,
        nullptr, nullptr, hInstance, NULL);


    if (!hwndParent)
        return nullptr;

    return hwndParent;
}

void DemoCreateWindowThumbnail(HWND targetWindow, HWND myWnd, IDCompositionVisual** ppWindowVisual)
{
    assert(targetWindow != myWnd);

    //Query the exact thumbnail source size
    SIZE windowSize{};
    lDwmpQueryWindowThumbnailSourceSize(targetWindow, FALSE, &windowSize);
    debug("windosize: %d %d\n", windowSize.cx, windowSize.cy);

    //Pretty explanatory and also documented by Microsoft
    DWM_THUMBNAIL_PROPERTIES thumb{};
    thumb.dwFlags = DWM_TNP_SOURCECLIENTAREAONLY | DWM_TNP_VISIBLE | DWM_TNP_RECTDESTINATION | DWM_TNP_RECTSOURCE | DWM_TNP_OPACITY | DWM_TNP_ENABLE3D;
    thumb.opacity = 255;
    thumb.fVisible = TRUE;
    thumb.fSourceClientAreaOnly = FALSE;
    thumb.rcDestination = RECT{ 0, 0, windowSize.cx, windowSize.cy };
    thumb.rcSource = RECT{ 0, 0, windowSize.cx, windowSize.cy };

    HTHUMBNAIL hThumbWindow;
    //Create the shared visual!
    auto res = lDwmpCreateSharedThumbnailVisual(myWnd, targetWindow, 2, &thumb,
        dcompDevice.Get(), (void**)ppWindowVisual, &hThumbWindow);
    if (!SUCCEEDED(res)) {
        debug("failed create shared thumnail %d", GetLastError());
        return;
    }
    RECT targetRect;
    GetWindowRect(targetWindow, &targetRect);
    (*ppWindowVisual)->SetOffsetX(targetRect.left);
    (*ppWindowVisual)->SetOffsetY(targetRect.top);

}

void DemoCreateMultiWindowThumbnail(HWND myWnd, IDCompositionVisual2** ppVirtualDesktopVisual)
{
    HTHUMBNAIL hThumbVirtualDesktop;
    auto virtualDeskRes = lDwmpCreateSharedMultiWindowVisual(myWnd, dcompDevice.Get(),
        (void**)ppVirtualDesktopVisual, &hThumbVirtualDesktop);

    auto monitorSize = RECT{ 0, 0, 1920, 1080 };
    auto targetSize = SIZE{ 960, 540 };

    HWND hwndTest = (HWND)0x1; //Exclude from the list what you want to exclude
    HWND* excludeArray = new HWND[1];
    excludeArray[0] = hwndTest;

    //The include list is useless as it will include every window in every case.
    //You can only play with the exclusion list.
    auto virtualDesktopUpdate = lDwmpUpdateSharedMultiWindowVisual(hThumbVirtualDesktop, NULL, 0,
        excludeArray, 1, &monitorSize, &targetSize, 1); //Last parameter has to be "1"

    //use lDwmpUpdateSharedVirtualDesktopVisual for 19043 or older
}


BOOL iswindowintersect(HWND hwnd)
{
    RECT hwndRect;
    if (!GetWindowRect(hwnd, &hwndRect))
        return FALSE;

    RECT intersectRect;
    return IntersectRect(&intersectRect, &gHWNDRect, &hwndRect);
}

BOOL checkWindowVisible(HWND hwnd)
{
    BOOL attr = FALSE;
    WINDOWCOMPOSITIONATTRIBDATA data{ WCA_CLOAKED, &attr, sizeof attr };
    if (!SUCCEEDED(lGetWindowCompositionAttribute(hwnd, &data)))
        debug("getwinattrr failed");
    return attr == 0 && IsWindowVisible(hwnd);
}

BOOL collectHWNDBehind(
    HWND hwnd,
    LPARAM lParam)
{
    if (hwnd != gHWND && checkWindowVisible(hwnd) /*&& iswindowintersect(hwnd)*/)
        windowList.push_back(hwnd);

    return TRUE;
}

void buildVisual(BOOL recollectHWND)
{
    debug("building visual");
    if (!rootVisual)
    {
        debug("rootVisual is null");
        return;
    }

    GetWindowRect(gHWND, &gHWNDRect);

    if (recollectHWND)
    {
        windowList.clear();
        EnumWindows(collectHWNDBehind, NULL);
        debug("windowlist %zu\n", windowList.size());
    }


    rootVisual->RemoveAllVisuals();
    visuals.clear();
    for (auto window : windowList) {
        visuals.push_back({});
        auto& newVisual = visuals.back();
        DemoCreateWindowThumbnail(window, gHWND, newVisual.GetAddressOf());
        auto r = rootVisual->AddVisual(newVisual.Get(), TRUE, NULL);
        if (!SUCCEEDED(r))
        {
            debug("failed to add visual");
        }
    }

    dcompDevice->Commit();
    DwmFlush();
}

//------------------------ Create window func
LRESULT CALLBACK MyWndProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    switch (message) {
    case WM_ACTIVATE:
        buildVisual(TRUE);
        break;
    case WM_WINDOWPOSCHANGED: {

        RECT rect;
        GetWindowRect(hwnd, &rect);
        rootClip->SetLeft((float)rect.left);
        rootClip->SetRight((float)rect.right);
        rootClip->SetTop((float)rect.top);
        rootClip->SetBottom((float)rect.bottom);
        rootVisual->SetClip(rootClip.Get());

        translateTransform->SetOffsetX(-1 * (float)rect.left - (GetSystemMetrics(SM_CXFRAME) + GetSystemMetrics(SM_CXPADDEDBORDER)));
        translateTransform->SetOffsetY(-1 * (float)rect.top - (GetSystemMetrics(SM_CYFRAME) + GetSystemMetrics(SM_CYCAPTION) + GetSystemMetrics(SM_CXPADDEDBORDER)));
        rootVisual->SetTransform(translateTransform.Get());

        gaussianEffect->SetBorderMode(D2D1_BORDER_MODE_HARD);
        gaussianEffect->SetStandardDeviation(20);
        rootVisual->SetEffect(gaussianEffect.Get());

        dcompDevice->Commit();
        DwmFlush();
        //buildVisual(FALSE);
        break;
    }
    case WM_CLOSE:
        PostQuitMessage(0);
        break;
    default:
        break;
    }
    return DefWindowProc(hwnd, message, wParam, lParam);
}

int APIENTRY wWinMain(HINSTANCE hInstance,
    HINSTANCE hPrevInstance,
    LPWSTR lpCmdLine,
    int nCmdShow)
{
    if (!InitPrivateDwmAPIs() || !InitPrivateUser32APIs() || !CreateDevice())
        return 0;

    ComPtr<IDCompositionTarget> dcompTarget;

    dcompDevice->CreateVisual(rootVisual.GetAddressOf());
    if (!rootVisual) {
        debug("failed to initialize root visual");
        return 0;
    }

    dcompDevice->CreateRectangleClip(rootClip.GetAddressOf());
    if (!rootClip) {
        debug("failed to initialize root clip");
        return 0;
    }

    dcompDevice->CreateTranslateTransform(translateTransform.GetAddressOf());
    if (!translateTransform) {
        debug("failed to initialize translate transform");
        return 0;
    }
    dcompDevice3->CreateGaussianBlurEffect(gaussianEffect.GetAddressOf());
    if (!gaussianEffect)
    {
        debug("failed to initialise gaussian effect");
        return 0;
    }

    auto myWnd = CreateWindowInternal(L"Noice window", L"DwmThumbExampleClass", WS_EX_OVERLAPPEDWINDOW /*| WS_EX_NOREDIRECTIONBITMAP*/, WS_OVERLAPPEDWINDOW);
    ShowWindow(myWnd, SW_SHOWMAXIMIZED);
    gHWND = myWnd;

    //Create the target for the window.
    if (dcompDevice->CreateTargetForHwnd(myWnd, FALSE, dcompTarget.GetAddressOf()) != S_OK)
        return 0;

    //You can either set the virtual desktop visual as root or create a new visual and set the content.
    if (dcompTarget->SetRoot(rootVisual.Get()) != S_OK)
        return 0;

    //Remember to commit.
    if (dcompDevice->Commit() != S_OK)
        return 0;
    buildVisual(TRUE);
    MSG msg{};
    while (GetMessage(&msg, nullptr, 0, 0)) {
        TranslateMessage(&msg);
        DispatchMessage(&msg);
    }

    //ComPtr automatically releases for you.
}