Calling into a member function of a foreign C++ DLL with mangled names from Python

For completeness, here is the shim that resulted from @Quuxplusone's answers. Caveat that the final function call into FooLib is throwing an exception, but it seems to be a separate issue.

Key elements:

• extern "C" blocks cannot handle C++ types, because it's C code, so they just have to go in a separate function that returns a C type.

• Dummy namespace, class, and function definitions matching the mangled functions being looked up.

• using statements to declare the pointer type locally.

• Pass the class instance as the first argument to the member function.

#include <string>
#include <windows.h>
#include <cstdio>

// Forward declarations for the Foo library classes
namespace FooLib {
    class ApplicationData {
        public:
            std::wstring getDataRootPath() const;
    };

    class Foo {
        public:
            ApplicationData& getApplicationData();
    };

    Foo& getFoo();
}

// Static variable to store the library path
static HMODULE g_hFooLib = NULL;

wchar_t* CopyWideString(const std::wstring& wstr) {
    if (wstr.empty()) {
        wchar_t* result = new wchar_t[1];
        result[0] = L'\0';
        return result;
    }

    size_t len = wstr.length() + 1; // +1 for null terminator
    wchar_t* result = new wchar_t[len];
    wcscpy_s(result, len, wstr.c_str());
    return result;
}

// C++ types need to be kept away from C interface
wchar_t* GetDataRootPathCPP() {
    if (!g_hFooLib) {
        fwprintf(stdout, L"GetDataRootPathCPP: library not loaded, call LoadFooLib(path) first.\n");
        return nullptr;
    }
    using FooLib_GetFoo = FooLib::Foo&();
    auto *getFoo = (FooLib_GetFoo*)GetProcAddress(g_hFooLib, "?getFoo@FooLib@@YAAEAVFoo@1@XZ");
    if (!getFoo) {
        fwprintf(stdout, L"GetDataRootPathCPP: GetProcAddress for getFoo failed\n");
        return nullptr;
    }

    using FooLib_GetApplicationData = FooLib::ApplicationData&(FooLib::Foo&);
    auto *getApplicationData = (FooLib_GetApplicationData*)GetProcAddress(g_hFooLib, "?getApplicationData@Foo@FooLib@@QEAAAEAVApplicationData@2@XZ");
    if (!getApplicationData) {
        fwprintf(stdout, L"GetDataRootPathCPP: GetProcAddress for getApplicationData failed\n");
        return nullptr;
    }
    
    using FooLib_GetDataRootPath = std::wstring(FooLib::ApplicationData&);
    auto *getDataRootPath = (FooLib_GetDataRootPath*)GetProcAddress(g_hFooLib, "?getDataRootPath@ApplicationData@FooLib@@QEBA?AV?$basic_string@_WU?$char_traits@_W@std@@V?$allocator@_W@2@@std@@XZ");
    if (!getDataRootPath) {
        fwprintf(stdout, L"GetDataRootPathCPP: GetProcAddress for getDataRootPath failed\n");
        return nullptr;
    }

    try {
        FooLib::Foo& Foo = getFoo();
        fwprintf(stdout, L"getFoo succeeded.\n");
        FooLib::ApplicationData& appData = getApplicationData(Foo);
        fwprintf(stdout, L"getApplicationData succeeded.\n");
        std::wstring ws = getDataRootPath(appData);
        fwprintf(stdout, L"getDataRootPath succeeded.\n");
        auto path = CopyWideString(ws);
        fwprintf(stdout, L"CopyWideString succeeded.\n");
        return path;
    }
    catch (...) {
        fwprintf(stdout, L"GetDataRootPathCPP: exception caught while obtaining data root path\n");
        return nullptr;
    }
}

//// External interface

// Function to set the library path
extern "C" __declspec(dllexport) bool LoadFooLib(const wchar_t* path) {
    // If we already have a library loaded, free it
    if (g_hFooLib) {
        fwprintf(stdout, L"LoadFooLib: freeing previously loaded library\n");
        FreeLibrary(g_hFooLib);
        g_hFooLib = NULL;
    }

    // Try to load the library
    g_hFooLib = LoadLibraryW(path);

    if (g_hFooLib == NULL) {
        DWORD error = GetLastError();
        LPWSTR msgBuf = NULL;
        DWORD fmt_flags = FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS;
        DWORD len = FormatMessageW(fmt_flags, NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPWSTR)&msgBuf, 0, NULL);

        // Trim trailing newlines/spaces commonly appended by FormatMessage
        if (msgBuf && len > 0) {
            // remove CR/LF and trailing spaces
            while (len > 0 && (msgBuf[len-1] == L'\n' || msgBuf[len-1] == L'\r' || msgBuf[len-1] == L' ' || msgBuf[len-1] == L'\t')) {
                msgBuf[len-1] = L'\0';
                --len;
            }
        }

        if (msgBuf && len > 0) {
            fwprintf(stdout, L"LoadFooLib: Failed to load library '%s' (Win32 error %lu): %s\n", path ? path : L"(null)", error, msgBuf);
            LocalFree(msgBuf);
        } else {
            fwprintf(stdout, L"LoadFooLib: Failed to load library '%s' (Win32 error %lu). No additional message available.\n", path ? path : L"(null)", error);
        }
        return false;
    }
    fwprintf(stdout, L"LoadFooLib: successfully loaded '%s'\n", path ? path : L"(null)");
    return true;
}

extern "C" __declspec(dllexport) wchar_t* GetDataRootPath() {
    return GetDataRootPathCPP();
}

extern "C" __declspec(dllexport) void FreeString(wchar_t* str) {
    delete[] str;
}

Part of the issue here (which you might not have realized, coming from Python) is that in C++ there's a difference between std::string and std::wstring. The latter is a string of wchar_t "wide characters," which are popular on Windows but not really anywhere else — they're 16-bit UTF-16. So your function isn't even giving you a string of char suitable for Python!

Writing a shim DLL is definitely the path of least resistance. It could be something like this [completely untested, of course]:

#include <cstdlib>
#include <string>

namespace FooLib {
  class ApplicationData {
  public:
    std::wstring getDataRootPath() const;
  };

  class Foo {
  public:
    ApplicationData& getApplicationData();
    const ApplicationData& getApplicationData() const;
  };
  Foo& getFoo();
}
char *createMyThing() {
  std::wstring ws = FooLib::getFoo().getApplicationData().getDataRootPath();
  char *p = (char*)std::malloc(ws.size() + 1);
  for (size_t i = 0; i <= ws.size(); ++i) {
    p[i] = ws[i];  // assuming there's no actually non-ASCII characters in there
  }
  return p;
}
void freeMyThing(char *p) {
  std::free(p);
}

We see that this code compiled with MSVC calls:

call ?getFoo@FooLib@@YAAEAVFoo@1@XZ
call ?getApplicationData@Foo@FooLib@@QEAAAEAVApplicationData@2@XZ
call ?getDataRootPath@ApplicationData@FooLib@@QEBA?AV?$basic_string@_WU?$char_traits@_W@std@@V?$allocator@_W@2@@std@@XZ

which at a glance looks like the right symbol names for what you said.

Be aware that this assumes your shim DLL will be built with the same version of std::wstring as the original DLL was. Microsoft's ABI is not officially stable (AFAIK), although I wouldn't expect them to have changed anything so fundamental as basic_string either.

Btw, Copilot got the first one wrong: it's not Foo FooLib::getFoo() but rather Foo& FooLib::getFoo(). Which is lucky, because it makes the shim a wee bit easier to write, compared to if we had to deal with a Foo returned by value. :)