Custom resolution on GCP VM (with T4 GPU on Windows Server 2019)

Question

I am currently searching for a way to set a fully custom resolution on a Windows Server 2019 VM with GPU (T4, with grid licence, and virtual workstation grid drivers) with C++.

I have tried different way to achieve this, I can make this work on my laptop, but seems to have some limitations on GCP VMs (or Windows Server limitation).

I have tried to do this with ChangeDisplaySettings/ChangeDisplaySettingsEx (winuser.h), I can change to a known resolution, but can't make it works with a custom one (not even with CDS_ENABLE_UNSAFE_MODE).

DWORD deviceIndex = 0;
DISPLAY_DEVICE displayDevice = { 0 };
displayDevice.cb = sizeof(DISPLAY_DEVICE);

while (EnumDisplayDevices(NULL, deviceIndex, &displayDevice, 0)) {
    deviceIndex++;

    DEVMODE dm = { 0 };
    dm.dmSize = sizeof(DEVMODE);

    DEVMODE finalDm = { 0 };
    finalDm.dmSize = sizeof(DEVMODE);

    //Check if able to retrieve current settings
    if (!EnumDisplaySettings(displayDevice.DeviceName, ENUM_CURRENT_SETTINGS, &dm)) {
        continue;
    }

    //Check if there is a difference in resolution list if UNSAFE_MODE is enabled or not (it seems to not change anything)
    int result = ChangeDisplaySettingsEx(displayDevice.DeviceName, &dm, 0, CDS_DISABLE_UNSAFE_MODES, NULL);
    std::cout << "CDS_DISABLE_UNSAFE_MODE" << std::endl;
    if (result == DISP_CHANGE_SUCCESSFUL) {
        for (int i = 0; EnumDisplaySettings(displayDevice.DeviceName, i, &dm) != 0; i++) {
            if (dm.dmBitsPerPel == 32) {
                std::cout << i << ". Found available resolution : " << dm.dmPelsWidth << " x " << dm.dmPelsHeight << " x " << dm.dmBitsPerPel << " @ " << dm.dmDisplayFrequency << std::endl;
            }
        }
    }

    result = ChangeDisplaySettingsEx(displayDevice.DeviceName, &dm, 0, CDS_ENABLE_UNSAFE_MODES, NULL);
    std::cout << "CDS_ENABLE_UNSAFE_MODE" << std::endl;
    if (result == DISP_CHANGE_SUCCESSFUL) {
        for (int i = 0; EnumDisplaySettings(displayDevice.DeviceName, i, &dm) != 0; i++) {
            if (dm.dmBitsPerPel == 32) {
                std::cout << i << ". Found available resolution : " << dm.dmPelsWidth << " x " << dm.dmPelsHeight << " x " << dm.dmBitsPerPel << " @ " << dm.dmDisplayFrequency << std::endl;
            }
        }
    }


    std::cout << "Please enter width : ";
    int width, height;
    std::cin >> width;
    std::cout << "Please enter height : ";
    std::cin >> height;

    dm.dmPelsWidth = width;
    dm.dmPelsHeight = height;
    if (width > height) {
        dm.dmDisplayOrientation = DMDO_DEFAULT;
    }
    else {
        dm.dmDisplayOrientation = DMDO_90;
    }
    dm.dmFields = DM_PELSWIDTH | DM_PELSHEIGHT | DM_DISPLAYORIENTATION;

    //result = ChangeDisplaySettingsEx(displayDevice.DeviceName, &dm, NULL, CDS_TEST, NULL);
    result = ChangeDisplaySettingsEx(displayDevice.DeviceName, &dm, NULL, 0, NULL);

    if (result != DISP_CHANGE_SUCCESSFUL) {
        std::cout << "Impossible to ChangeDisplaySettings" << endl;
    }
    else {
        std::cout << "OK" << endl;
    }

    break;
}

I then take a look at NVAPI, and same here, I can make it works on my PC but still nothing on the GCP VMs... I have found a way to make NVAPI create and use custom resolution on my local PC, but can't make it works on GCP VM once again... (Code example found here)

NvAPI_Status result = NVAPI_ERROR;
NvU32 primaryDisplayId = 0;


//Testing resolution
int horizontal = 1920, vertical = 1090;

result = NvAPI_Initialize();

if (result != NVAPI_OK) {
    printf("Could not initialize NvAPI");
    return false;
}

MONITORINFOEX monInfo;
HMONITOR hMon;
const POINT ptZero = { 0, 0 };

// determine the location of the primary monitor
hMon = MonitorFromPoint(ptZero, MONITOR_DEFAULTTOPRIMARY);
ZeroMemory(&monInfo, sizeof(monInfo));
monInfo.cbSize = sizeof(monInfo);
GetMonitorInfo(hMon, &monInfo);

result = NvAPI_DISP_GetGDIPrimaryDisplayId(&primaryDisplayId);

if (result != NVAPI_OK) {
    printf("Could not get display ID from device");
    NvAPI_Unload();
    return false;
}

NvU32 deviceCount = 0;
NV_CUSTOM_DISPLAY cd[NVAPI_MAX_DISPLAYS] = { 0 };

float refreshRate = 60;
// timing computation (to get timing that suits the changes made)
NV_TIMING_FLAG flag = { 0 };
NV_TIMING_INPUT timing = { 0 };
timing.version = NV_TIMING_INPUT_VER;
timing.height = vertical;
timing.width = horizontal;
timing.rr = refreshRate;
timing.flag = flag;
timing.type = NV_TIMING_OVERRIDE_CVT_RB;

result = NvAPI_DISP_GetTiming(primaryDisplayId, &timing, &cd[0].timing);

if (result != NVAPI_OK) {
    printf("Failed to get timing for display"); // failed to get custom display timing
    NvAPI_Unload();
    return false;
}



cd[0].width = horizontal;
cd[0].height = vertical;
cd[0].xRatio = 1;
cd[0].yRatio = 1;
cd[0].srcPartition = { 0, 0, 1.0, 1.0 };
cd[0].depth = 32;
cd[0].version = NV_CUSTOM_DISPLAY_VER;
cd[0].colorFormat = NV_FORMAT_A8R8G8B8;

//Returns NVAPI_ERROR on GCP but NVAPI_OK on my laptop
result = NvAPI_DISP_TryCustomDisplay(&primaryDisplayId, 1, cd); 

if (result != NVAPI_OK) {
    printf("Could not set custom resolution");
    NvAPI_DISP_RevertCustomDisplayTrial(&primaryDisplayId, 1);
    NvAPI_Unload();
    return false;
}
else {
    NvAPI_DISP_SaveCustomDisplay(&primaryDisplayId, 1, true, true);
}

This part works perfectly well on my laptop, I can use a new dynamic resolution (It works with 1920x400, 1920x500, 1920x600), but not on my GCP VM, this parts :

NvAPI_DISP_TryCustomDisplay(&primaryDisplayId, 1, cd);

always returns NVAPI_ERROR

I have found another trick, I can edit this registry entry : HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Video{RANDOM_ID}\0001\NV_Modes (Here is an old pdf, after some testing, it seems it is still working this way) If I add some resolution using NVAPI, I can then set through ChangeDisplaySettingsEx function this resolution (it needs GPU driver restart, or Windows restart be able to change to a fresh new added resolution). But I need to be able to rotate screen, playing with "dmDisplayOrientation", and it does not seem to work on GCP VM once again, if I authorize for example 1920x1090 I can set resolution to this, but cannot set 1090x1920 with a "dmDisplayOrientation = DMDO_90" (even if I authorize 1090x1920 too...)

So if anyone found a way, or have any idea on how to do this, it would be great, I am running out of idea right now...