Trying to hook DirectX9 using C++ - Keeps crashing

Question

I posted a thread on StackOverflow a few days ago, and it's got little attention, so i though it's possible this might be a better place for it.

As the title says, i'm trying to hook/detour DirectX9 functions so i can render some information on screen.

I was able to detour regular functions in my own application by finding the offset address in IDA, i believe i understand what hooking/detouring is and how it works, at least to a reasonable extent.

The problem is, i'm not sure how to even find the EndScene function in IDA, and any attempt I've made at creating a dummy device and getting a V-Table pointer for the function have just not worked.

Here's my code, someone please point out what i'm doing wrong here. I'd like to fix this and learn what I've done wrong so i can avoid making the same mistake in the future.

// dllmain.cpp : Defines the entry point for the DLL application.
#include "stdafx.h"
#include <iostream>
#include <Windows.h>
#include <intrin.h>  
#include <tchar.h>
#include <tlhelp32.h>
#include <Psapi.h>
#include <winsock2.h>
#include <vector>
#include <ws2tcpip.h>
#pragma comment( lib, "Ws2_32.lib" )
#include <d3d9.h>
#pragma comment(lib, "d3d9.lib")
#include <d3dx9.h>
#pragma comment(lib, "d3dx9.lib")
#include <detours.h>
#pragma comment(lib, "detours.lib")

using namespace std;

D3DCOLOR RED = D3DCOLOR_ARGB(255, 255, 0, 0);

typedef HRESULT(__stdcall* EndScene)(IDirect3DDevice9*);
static EndScene EndScene_orig = NULL;

HRESULT __stdcall EndScene_hook(IDirect3DDevice9* pDevice)
{
//  D3DRECT rec = { 100,100,200,200 };
//  pDevice->Clear(1, &rec, D3DCLEAR_TARGET, RED, 0, 0);
//  MessageBoxA(0, "In EndScene", "", 0); //    <<<<----- This function is called over and over when not commented.
    return EndScene_orig(pDevice);
}

void WINAPI InitHook()
{

    HWND game_window = FindWindow(NULL, _T("Skinned Mesh"));

    auto d3dpp = D3DPRESENT_PARAMETERS{};
    auto d3d = Direct3DCreate9(D3D_SDK_VERSION);
    if (d3d)
    {
        d3dpp.BackBufferCount = 1;
        d3dpp.MultiSampleType = D3DMULTISAMPLE_NONE;
        d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
        d3dpp.hDeviceWindow = game_window;
        d3dpp.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;
        d3dpp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;
        d3dpp.BackBufferFormat = D3DFMT_R5G6B5;
        d3dpp.Windowed = TRUE;
        IDirect3DDevice9* Device{};
        if (SUCCEEDED(d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, game_window, D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &Device)))
        {
//          MessageBoxA(0, "We made it here...", "", 0);

            DWORD** pVTable = *reinterpret_cast<DWORD***>(Device);

            DetourTransactionBegin();
            DetourUpdateThread(GetCurrentThread());

            EndScene_orig = (EndScene)pVTable[42];
            DetourAttach(&(LPVOID&)pVTable[42], EndScene_hook);

            if (DetourTransactionCommit() == NO_ERROR)
                cout << "Detoured successfully" << endl;
        }
    }

}


void SetupConsole()
{
    AllocConsole();
    freopen("CONOUT$", "wb", stdout);
    freopen("CONOUT$", "wb", stderr);
    freopen("CONIN$", "rb", stdin);
    SetConsoleTitle("CSGOHAX");
}


BOOL APIENTRY DllMain(HMODULE hModule,
    DWORD  ul_reason_for_call,
    LPVOID lpReserved
)
{
    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        SetupConsole();
        DisableThreadLibraryCalls(hModule);
        CreateThread(0, 0, (LPTHREAD_START_ROUTINE)InitHook, 0, 0, NULL);
        break;
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

Answer 1

I figured it out, reversed the binary in IDA, found the module address and the EndScene function aswell as its address, calculated the offset. Then used ollydbg and found the function again, made a signature from it, now i can find it dynamically using a signature scanning function.

So i can get the function address with this signature.

DWORD dwEndScene = FindPattern("d3d9.dll",
    "\x6A\x18\xB8\x00\x00\x00\x00\xE8\x00\x00\x00\x00\x8B\x7D\x08\x8B\xDF\x8D\x47\x04\xF7\xDB\x1B\xDB\x23\xD8\x89\x5D\xE0\x33\xF6\x89\x75\xE4\x39\x73\x18\x75\x73",
    "xxx????x????xxxxxxxxxxxxxxxxxxxxxxxxxxx");

Then i just detour the function

DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
EndScene_orig = (oEndScene)(dwEndScene);
DetourAttach(&(LPVOID&)EndScene_orig, EndScene_hook);
if (DetourTransactionCommit() == NO_ERROR)
    cout << "Detoured successfully" << endl;

This is much easier than trying to find the function in the V-Table using a dummy device as i was before.