DetourDetach() throws ERROR_INVALID_BLOCK error

Question

I want to hook the Bitblt function with the Detours library.

https://github.com/microsoft/Detours/blob/master/samples/simple/simple.cpp

By referring to the example source above, I succeeded in creating a dll that hooks the Bitblt function, but the unhooking does not work properly.

I want the original function to be restored when the dll is detached from the target process, but the DetourDetach function throws an ERROR_INVALID_BLOCK error, and access violation of the target process occurs.

How can I fix this error?

Below is the source code I wrote.

<testwinapi / main.cpp>

#include <stdio.h>
#include <Windows.h>

void capture(HBITMAP* canvas);

int WINAPI WinMain(_In_ HINSTANCE hInstance, _In_opt_ HINSTANCE hPrevInstance, _In_ LPSTR lpCmdLine, _In_ int nShowCmd) {
    HMODULE hModule = LoadLibrary(TEXT("testdll.dll"));
    if (!hModule) return 1;

    int i = 0;
    while(1) {
        HBITMAP canvas;
        capture(&canvas);
        Sleep(2000);
        if (++i >= 1) 
            FreeLibrary(hModule);
    }

    //FreeLibrary(hModule);
    return 0;
}

void capture(HBITMAP* canvas) {
    RECT srcRect;
    HWND hSrcWnd;
    HDC hSrcDC, hDestDC;

    hSrcWnd = GetDesktopWindow();
    hSrcDC = GetDC(hSrcWnd);

    GetWindowRect(hSrcWnd, &srcRect);
    int SrceenWidth = srcRect.right - srcRect.left;
    int SrceenHeight = srcRect.bottom - srcRect.top;

    hDestDC = CreateCompatibleDC(hSrcDC);
    *canvas = CreateCompatibleBitmap(hSrcDC, SrceenWidth, SrceenHeight);
    SelectObject(hDestDC, *canvas);

    for (int y = 0; y < SrceenHeight; y += 50) {
        BitBlt(hDestDC, 0, y, SrceenWidth, 50, hSrcDC, 0, y, SRCCOPY);
        Sleep(2);
    }

    ReleaseDC(hSrcWnd, hSrcDC);
    DeleteDC(hDestDC);
}

<testdll / dllmain.cpp>

#include "pch.h"

BOOL(WINAPI* originFunc) (HDC, int, int, int, int, HDC, int, int, DWORD);
BOOL(WINAPI* detourFunc) (HDC, int, int, int, int, HDC, int, int, DWORD);

DWORD WriteLog(LPCTSTR format, ...) {
    TCHAR szLog[500];
    DWORD dwBytesWriten;

    va_list args;
    va_start(args, format);
    _vstprintf_s(szLog, 500, format, args);
    va_end(args);

    WriteConsole(GetStdHandle(STD_OUTPUT_HANDLE), szLog, _tcslen(szLog), &dwBytesWriten, 0);

    return dwBytesWriten;
}

BOOL WINAPI MyBitBlt(HDC hdc, int x, int y, int cx, int cy, HDC hdcSrc, int x1, int y1, DWORD rop) {
    WriteLog(TEXT("Function called : Bitblt(0x%X, %d, %d, %d, %d, 0x%X, %d, %d, 0x%X)\n"), 
        (DWORD)hdc, x, y, cx, cy, (DWORD)hdcSrc, x1, y1, rop);

    return originFunc(hdc, x, y, cx, cy, hdcSrc, x1, y1, rop);
}

BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) {
    if (DetourIsHelperProcess())
        return TRUE;

    originFunc = BitBlt;
    detourFunc = MyBitBlt;

    LONG error;
    switch (ul_reason_for_call) {
    case DLL_PROCESS_ATTACH:
        AllocConsole();

        DetourRestoreAfterWith();

        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourAttach(&(PVOID&)originFunc, detourFunc);
        error = DetourTransactionCommit();

        WriteLog(TEXT("Detour result(Attach) : 0x%08X\n"), error);

        break;

    case DLL_PROCESS_DETACH:
        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourDetach(&(PVOID&)originFunc, detourFunc);
        error = DetourTransactionCommit();

        WriteLog(TEXT("Detour result(Detach) : 0x%08X\n"), error);
        Sleep(500);

        FreeConsole();

        break;
    }

    return TRUE;
}

<testdll / pch.h>

#include "framework.h"
#include <stdio.h>
#include <stdarg.h>
#include <tchar.h>
#include <detours.h>

(This article has been translated by Google Translate.)

Answer 1

I figured out what was the problem!

#include "pch.h"

BOOL(WINAPI* originFunc) (HDC, int, int, int, int, HDC, int, int, DWORD) = Bitblt; //HERE
BOOL(WINAPI* detourFunc) (HDC, int, int, int, int, HDC, int, int, DWORD);

DWORD WriteLog(LPCTSTR format, ...) {
    TCHAR szLog[500];
    DWORD dwBytesWriten;

    va_list args;
    va_start(args, format);
    _vstprintf_s(szLog, 500, format, args);
    va_end(args);

    WriteConsole(GetStdHandle(STD_OUTPUT_HANDLE), szLog, _tcslen(szLog), &dwBytesWriten, 0);

    return dwBytesWriten;
}

BOOL WINAPI MyBitBlt(HDC hdc, int x, int y, int cx, int cy, HDC hdcSrc, int x1, int y1, DWORD rop) {
    WriteLog(TEXT("Function called : Bitblt(0x%X, %d, %d, %d, %d, 0x%X, %d, %d, 0x%X)\n"), 
        (DWORD)hdc, x, y, cx, cy, (DWORD)hdcSrc, x1, y1, rop);

    return originFunc(hdc, x, y, cx, cy, hdcSrc, x1, y1, rop);
}

BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) {
    if (DetourIsHelperProcess())
        return TRUE;

    //NOT HERE
    //originFunc = BitBlt;
    detourFunc = MyBitBlt;

    LONG error;
    switch (ul_reason_for_call) {
    case DLL_PROCESS_ATTACH:
        AllocConsole();

        DetourRestoreAfterWith();

        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourAttach(&(PVOID&)originFunc, detourFunc);
        error = DetourTransactionCommit();

        WriteLog(TEXT("Detour result(Attach) : 0x%08X\n"), error);

        break;

    case DLL_PROCESS_DETACH:
        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourDetach(&(PVOID&)originFunc, detourFunc);
        error = DetourTransactionCommit();

        WriteLog(TEXT("Detour result(Detach) : 0x%08X\n"), error);
        Sleep(500);

        FreeConsole();

        break;
    }

    return TRUE;
}

Make sure that originFunc is initialized only once before hook. (because CopyOnWrite)

(This article has been translated by Google Translate.)