Identify Cause of NULL_CLASS_PTR_READ_c0000005 on combase!CStdMarshal::UnmarshalIPID in PowerBuilder 10 App

Question

What are some alternate steps that could be used to identify the root cause of this access violation read crash. I don't have access to source code or the customer env so need to specify steps to collect required information. The issue only occurs when 3rd party firewall software is running, of which it's driver can be seen in stack traces of the process when using ProcMon stack summary feature. However I need to work out more explicitly how the issue is caused to enable a fix from 3rd party firewall vendor.

I have tried to walk customer through capturing API Monitor traces and Time Travel Debugging Trace however they are triggering the application to crash before issue can be reproduced.

The application seems to be built with PowerBuilder 10.2.1.0. Crash dump output shows the following:

0:000> !analyze -v
ADDITIONAL_XML: 1

OS_BUILD_LAYERS: 1

NTGLOBALFLAG:  0

PROCESS_BAM_CURRENT_THROTTLED: 0

PROCESS_BAM_PREVIOUS_THROTTLED: 0

APPLICATION_VERIFIER_FLAGS:  0

CONTEXT:  (.ecxr)
eax=00000010 ebx=009c653c ecx=06da3ce4 edx=0019ccf8 esi=00000000 edi=00a63f88
eip=756afcbf esp=0019cb7c ebp=0019cbc0 iopl=0         nv up ei pl nz na pe nc
cs=0023  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00210206
combase!CStdMarshal::UnmarshalIPID+0xb8:
756afcbf 8b5804          mov     ebx,dword ptr [eax+4] ds:002b:00000014=????????
Resetting default scope

EXCEPTION_RECORD:  (.exr -1)
ExceptionAddress: 756afcbf (combase!CStdMarshal::UnmarshalIPID+0x000000b8)
   ExceptionCode: c0000005 (Access violation)
  ExceptionFlags: 00000000
NumberParameters: 2
   Parameter[0]: 00000000
   Parameter[1]: 00000014
Attempt to read from address 00000014

PROCESS_NAME:  appname.exe

READ_ADDRESS:  00000014 

ERROR_CODE: (NTSTATUS) 0xc0000005 - The instruction at 0x%p referenced memory at 0x%p. The memory could not be %s.

EXCEPTION_CODE_STR:  c0000005

EXCEPTION_PARAMETER1:  00000000

EXCEPTION_PARAMETER2:  00000014

STACK_TEXT:  
0019cbc0 756c62a5     0019ccf8 0019cd08 009c5aa0 combase!CStdMarshal::UnmarshalIPID+0xb8
0019cc48 756d61b0     0019ccf0 0019cf1c 00000000 combase!CStdMarshal::UnmarshalObjRef+0x155
0019ce54 5b309fe8     009ef828 5b2f5fa4 0019cf1c combase!CoUnmarshalInterface+0xdc0
0019ceec 5b312666     5b2f5fa4 0019cf1c 0000003d oleacc!SharedBuffer_Free+0x2b8
0019cf7c 5b3097fb     00002728 0000003d 00000001 oleacc!SharedBuffer_Allocate+0x8e36
0019cfa0 1123c095     11437d18 ffffffff 03d977cc oleacc!EXTERNAL_LresultFromObject+0x4b
WARNING: Stack unwind information not available. Following frames may be wrong.
0019cfd4 112363b7     0007079e ffffffff fffffffc pbvm100!FN_EvtTimerWnd+0xa75
0019d068 75db44bb     0007079e 0000003d ffffffff pbvm100!FN_DataWindowWnd+0x4a7
0019d094 75d94ffc     11235f10 0007079e 0000003d user32!_InternalCallWinProc+0x2b
0019d178 75d94b9a     11235f10 00000000 0000003d user32!UserCallWinProcCheckWow+0x3ac
0019d1dc 75d9e1df     01098290 00000000 0000003d user32!DispatchClientMessage+0xea
0019d218 7758428d     0019d234 00000020 0019d408 user32!__fnDWORD+0x3f
0019d250 759a2c0c     75db2d92 0007079e 0000003d ntdll!KiUserCallbackDispatcher+0x4d
0019d254 75db2d92     0007079e 0000003d ffffffff win32u!NtUserMessageCall+0xc
0019d298 75db2cf2     ffffffff fffffffc 00000002 user32!SendMessageTimeoutWorker+0x9b
0019d2bc 5b309695     0007079e 0000003d ffffffff user32!SendMessageTimeoutW+0x22
0019d2f8 5b30954b     0007079e fffffffc 5b2f5cc4 oleacc!NativeIAccessibleFromWindow+0x70
0019d328 5b32f49e     5b2f5cc4 0019d348 fffffffc oleacc!AccessibleObjectFromWindow+0x27
0019d360 5b32fa57     00000000 0019d3f8 0019d3c0 oleacc!AccessibleObjectFromEvent+0x5e
0019d374 76080c8b     0007079e fffffffc 00000000 oleacc!EXTERNAL_AccessibleObjectFromEvent+0x27
0019d39c 76080bae     00000000 0019d3f8 0019d3c0 msctf!AccessibleObjectFromEvent+0x38
0019d418 7606a0c5     0007079e fffffffc 00000000 msctf!CThreadInputMgr::OnAccFocusEvent+0x9f
0019d468 76069320     00008005 0007079e fffffffc msctf!CThreadInputMgr::OnCiceroEvent+0xa5
0019d4c4 75d9f019     001f02b7 00008005 0007079e msctf!WinEventProc+0xf0
0019d508 7758428d     0019d524 00000020 0019d928 user32!__ClientCallWinEventProc+0x39
0019d540 11297886     0007079e 0007079e 0704d628 ntdll!KiUserCallbackDispatcher+0x4d
0019d558 11234dd9     000b0736 00000000 00000000 pbvm100!FN_WndProc+0x766
0019d828 75db44bb     000b0736 00000007 00360360 pbvm100!FN_WindowWnd+0x14e9
0019d854 75d94ffc     112338f0 000b0736 00000007 user32!_InternalCallWinProc+0x2b
0019d938 75d9454f     112338f0 00000000 00000007 user32!UserCallWinProcCheckWow+0x3ac
0019d970 10b8f24d     112338f0 000b0736 00000007 user32!CallWindowProcW+0x7f
0019d998 75db44bb     000b0736 00000007 00360360 pbshr100!PBC_MainProc3D+0x16d
0019d9c4 75d94ffc     10b8f0e0 000b0736 00000007 user32!_InternalCallWinProc+0x2b
0019daa8 75d94b9a     10b8f0e0 00000000 00000007 user32!UserCallWinProcCheckWow+0x3ac
0019db0c 75d9e1df     010832e0 00000000 00000007 user32!DispatchClientMessage+0xea
0019db48 7758428d     0019db64 00000020 0019dce4 user32!__fnDWORD+0x3f
0019db80 759a30ac     75da6b6e 000b0736 00000007 ntdll!KiUserCallbackDispatcher+0x4d
0019db84 75da6b6e     000b0736 00000007 00000000 win32u!NtUserSetFocus+0xc
0019dbc4 75dee1b9     0102ab40 00000000 00000007 user32!MDIClientWndProcWorker+0x14e
0019dbe4 75db44bb     00360360 00000007 00360360 user32!MDIClientWndProcW+0x29
0019dc10 75d94ffc     75dee190 00360360 00000007 user32!_InternalCallWinProc+0x2b
0019dcf4 75d9454f     75dee190 00000000 00000007 user32!UserCallWinProcCheckWow+0x3ac
0019dd2c 10b8cdd2     75dee190 00360360 00000007 user32!CallWindowProcW+0x7f
0019dd54 75d952a1     00000000 77554470 112338f0 pbshr100!PBC_NormalProc3D+0x72
0019dd90 75d94ffc     10b8cd60 00360360 00000007 user32!UserCallWinProcCheckWow+0x651
0019de74 75d9454f     10b8cd60 00000000 00000007 user32!UserCallWinProcCheckWow+0x3ac
0019deac 112380fd     10b8cd60 00360360 00000007 user32!CallWindowProcW+0x7f
0019deec 75db44bb     00360360 00000007 00360360 pbvm100!FN_MDIClientWnd+0x2dd
0019df18 75d94ffc     11237e20 00360360 00000007 user32!_InternalCallWinProc+0x2b
0019dffc 75d947ad     11237e20 00000000 00000007 user32!UserCallWinProcCheckWow+0x3ac
0019e060 75d94473     0102ab40 00000000 00000000 user32!SendMessageWorker+0x1fd
0019e094 75da669a     00360360 00000007 00360360 user32!SendMessageW+0x123
0019e0d0 75da62a7     0102ab40 00000000 010832e0 user32!xxxMDIActivate+0x201
0019e114 75deea18     00000000 00000000 00000000 user32!DefMDIChildProcWorker+0x1f7
0019e128 112971f8     000b0736 00000022 00000000 user32!DefMDIChildProcW+0x18
0019e150 112dcf2a     000b0736 00000022 00000000 pbvm100!FN_WndProc+0xd8
0019e168 112dcba9     11297120 000b0736 00000022 pbvm100!fn_txnservice_create_instance+0x912a
0019e18c 11235260     11297120 000b0736 00000022 pbvm100!fn_txnservice_create_instance+0x8da9
0019e470 75db44bb     000b0736 00000022 00000000 pbvm100!FN_WindowWnd+0x1970
0019e49c 75d94ffc     112338f0 000b0736 00000022 user32!_InternalCallWinProc+0x2b
0019e580 75d9454f     112338f0 00000000 00000022 user32!UserCallWinProcCheckWow+0x3ac
0019e5b8 10b8f24d     112338f0 000b0736 00000022 user32!CallWindowProcW+0x7f
0019e5e0 75db44bb     000b0736 00000022 00000000 pbshr100!PBC_MainProc3D+0x16d
0019e60c 75d94ffc     10b8f0e0 000b0736 00000022 user32!_InternalCallWinProc+0x2b
0019e6f0 75d94b9a     10b8f0e0 00000000 00000022 user32!UserCallWinProcCheckWow+0x3ac
0019e754 75d9e1df     010832e0 00000000 00000022 user32!DispatchClientMessage+0xea
0019e790 7758428d     0019e7ac 00000020 0019ea6c user32!__fnDWORD+0x3f
0019e7c8 759a2ddc     75dcaa02 000b0736 00000000 ntdll!KiUserCallbackDispatcher+0x4d
0019e7cc 75dcaa02     000b0736 00000000 00000000 win32u!NtUserSetWindowPos+0xc
0019e820 75d8fa58     00000001 00000000 0019ee94 user32!MDICompleteChildCreation+0x1b895
0019e8d8 75da6be3     03d994c4 16cf0000 80000000 user32!CreateWindowInternal+0x2ec
0019e94c 75dee1b9     0102ab40 00000000 00000220 user32!MDIClientWndProcWorker+0x1c3
0019e96c 75db44bb     00360360 00000220 00000000 user32!MDIClientWndProcW+0x29
0019e998 75d94ffc     75dee190 00360360 00000220 user32!_InternalCallWinProc+0x2b
0019ea7c 75d9454f     75dee190 00000000 00000220 user32!UserCallWinProcCheckWow+0x3ac
0019eab4 10b8cdd2     75dee190 00360360 00000220 user32!CallWindowProcW+0x7f
0019eaec 75db44bb     00360360 00000220 00000000 pbshr100!PBC_NormalProc3D+0x72
00000000 00000000     00000000 00000000 00000000 user32!_InternalCallWinProc+0x2b


SYMBOL_NAME:  oleacc!SharedBuffer_Free+2b8

MODULE_NAME: oleacc

IMAGE_NAME:  oleacc.dll

STACK_COMMAND:  ~0s ; .ecxr ; kb

FAILURE_BUCKET_ID:  NULL_CLASS_PTR_READ_c0000005_oleacc.dll!SharedBuffer_Free

OS_VERSION:  10.0.18362.239

BUILDLAB_STR:  19h1_release_svc_prod1

OSPLATFORM_TYPE:  x86

OSNAME:  Windows 10

IMAGE_VERSION:  7.2.18362.1

FAILURE_ID_HASH:  {2506b308-0659-fc20-d871-5cc908d8929d}

Followup:     MachineOwner

The parameters passed to the COM APIs are:

0c 0019cbc0 756c62a5     combase!CStdMarshal::UnmarshalIPID(struct _GUID * riid = 0x0019ccf8 {618736E0-3C3D-11CF-810C-00AA00389B71}, struct tagSTDOBJREF * pStd = 0x0019cd08, class OXIDEntry * pOXIDEntry = 0x009c5aa0, void ** ppv = 0x0019cf1c)+0xb8 [onecore\com\combase\dcomrem\marshal.cxx @ 2406] 
0d 0019cc48 756d61b0     combase!CStdMarshal::UnmarshalObjRef(struct tagOBJREF * objref = 0x0019ccf0, void ** ppv = 0x0019cf1c)+0x155 [onecore\com\combase\dcomrem\marshal.cxx @ 2194] 
0e (Inline) --------     combase!UnmarshalSwitch(void)+0xe7 [onecore\com\combase\dcomrem\marshal.cxx @ 1825] 
0f (Inline) --------     combase!UnmarshalObjRef(void)+0x1f8 [onecore\com\combase\dcomrem\marshal.cxx @ 1963] 
10 0019ce54 5b309fe8     combase!CoUnmarshalInterface(struct IStream * pStm = 0x009ef828, struct _GUID * riid = 0x5b2f5fa4 {00000000-0000-0000-C000-000000000046}, void ** ppv = 0x0019cf1c)+0xdc0 [onecore\com\combase\dcomrem\coapi.cxx @ 1993] 

This seems to reference IAccessible interface which has the following registry info:

Windows Registry Editor Version 5.00

[HKEY_CLASSES_ROOT\interface\{618736E0-3C3D-11CF-810C-00AA00389B71}]
@="IAccessible"

[HKEY_CLASSES_ROOT\interface\{618736E0-3C3D-11CF-810C-00AA00389B71}\ProxyStubClsid32]
@="{00020424-0000-0000-C000-000000000046}"

[HKEY_CLASSES_ROOT\interface\{618736E0-3C3D-11CF-810C-00AA00389B71}\TypeLib]
@="{1EA4DBF0-3C3B-11CF-810C-00AA00389B71}"
"Version"="1.1"

This then seems to refer to PSOAInterface:

Windows Registry Editor Version 5.00

[HKEY_CLASSES_ROOT\clsid\{00020424-0000-0000-C000-000000000046}]
@="PSOAInterface"

[HKEY_CLASSES_ROOT\clsid\{00020424-0000-0000-C000-000000000046}\InprocServer32]
@="C:\\Windows\\System32\\oleaut32.dll"
"ThreadingModel"="Both"

Reversing the crashing function to psuedocode this code is within the PowerBuilder runtime i.e. pbvm100.dll:

0:000> lmvm pbvm100
Browse full module list
start    end        module name
111e0000 11528000   pbvm100    (export symbols)       pbvm100.dll
    Loaded symbol image file: pbvm100.dll
    Image path: C:\Program Files\AppName\pbvm100.dll
    Image name: pbvm100.dll
    Browse all global symbols  functions  data
    Timestamp:        Wed Aug 15 13:28:53 2007 (46C272F5)
    CheckSum:         004132DA
    ImageSize:        00348000
    File version:     10.2.1.9948
    Product version:  10.2.1.0
    File flags:       2 (Mask 3) Pre-release
    File OS:          10001 DOS Win16
    File type:        1.65 App
    File date:        00000000.00000000
    Translations:     0409.04e4
    Information from resource tables:
        CompanyName:      Sybase Inc.
        ProductName:      PowerBuilder/InfoMaker
        InternalName:     PB 10.0
        FileVersion:      10.2.1.9948
        FileDescription:  Sybase Inc. Product File
        LegalCopyright:   Copyright Sybase Inc. 2004

LRESULT __stdcall FN_DataWindowWnd(HWND hWnd, UINT Msg, WPARAM wParam, LPARAM lParam)
{
 etc..

 if ( Msg <= 0x106 )
  {
    if ( Msg == 262 )
      return CallWindowProcW(*((WNDPROC *)v12 + 3), hWnd, 0x106u, wParam, lParam);
     switch ( Msg )
    {
      case 0x3Du:
        if ( lParam != -4 )
          goto LABEL_72;
        result = CrashingFunction(hWnd, wParam, -4, 0);
        break;
       etc...
     }
   }
}

int __cdecl CrashingFunction(HWND hWnd, int a2, int a3, int a4)
{
  int result; // eax
  int v5; // [esp+Ch] [ebp-14h]
  int v6; // [esp+14h] [ebp-Ch]
  int v7; // [esp+18h] [ebp-8h] BYREF
  int v8; // [esp+1Ch] [ebp-4h]

  v8 = 0;
  v7 = 0;
  v6 = GetWindowProperties(hWnd);
  if ( !v6 )
    return 0;
  if ( *(_DWORD *)(v6 + 628) || (v8 = PB_CreateAccessibleService(v6 + 628)) == 0 && *(_DWORD *)(v6 + 628) )
  {
    v5 = GetWinProp(hWnd);
    if ( v5 )
    {
      if ( *(_DWORD *)v5 == 16509 )
      {
        v8 = (*(int (__stdcall **)(_DWORD, HWND, int))(**(_DWORD **)(v6 + 628) + 16))(
               *(_DWORD *)(v6 + 628),
               hWnd,
               v5 + 256);
        if ( !v8 && *(_DWORD *)(v5 + 256) )
        {
          if ( a4 )
          {
            (*(void (__stdcall **)(_DWORD, int, int *))(**(_DWORD **)(v5 + 256) + 116))(*(_DWORD *)(v5 + 256), a4, &v7);
            if ( v7 )
              result = PB_LresultFromObject(&unk_11437D18, a2, v7);
            else
              result = 0;
          }
          else
          {
            result = PB_LresultFromObject(&unk_11437D18, a2, *(_DWORD *)(v5 + 256));
          }
        }
        else
        {
          switch ( v8 )
          {
            case -2147467261:
              DisplayErrorMsg(0x4B0u, 0x4B4u);
              break;
            case -2147024890:
              DisplayErrorMsg(0x4B0u, 0x4B2u);
              break;
            case -2147024882:
              DisplayErrorMsg(0x4B0u, 0x4B6u);
              break;
          }
          result = 0;
        }
      }
      else
      {
        result = 0;
      }
    }
    else
    {
      result = 0;
    }
  }
  else
  {
    if ( v8 == -2147024882 )
      DisplayErrorMsg(0x4B0u, 0x4B6u);
    else
      DisplayErrorMsg(0x4B0u, 0x4B1u);
    result = 0;
  }
  return result;
}