win32: use HINST_THISCOMPONENT

This is a common idiom used in MSDN docs and Raymond Chen's example
programs to get a HINSTANCE for the current module, regardless of
whether it's an .exe or a .dll. Using GetModuleHandle(NULL) for this is
technically incorrect, since it always gets a handle to the .exe, even
when the executing code (in libmpv) is running in a .dll. In this case,
using the wrong HINSTANCE could cause namespace issues with window
classes, since CreateWindowEx uses the HINSTANCE to search for the
matching window class name.

See:
https://blogs.msdn.microsoft.com/oldnewthing/20050418-59/?p=35873
https://blogs.msdn.microsoft.com/oldnewthing/20041025-00/?p=37483

2 files changed, 13 insertions, 9 deletions

diff --git a/video/out/opengl/context_dxinterop.c b/video/out/opengl/context_dxinterop.c
index 4dfc3c2108..a4dc165132 100644
--- a/video/out/opengl/context_dxinterop.c
+++ b/video/out/opengl/context_dxinterop.c
@@ -27,6 +27,9 @@
 // For WGL_ACCESS_WRITE_DISCARD_NV, etc.
 #include <GL/wglext.h>
 
+EXTERN_C IMAGE_DOS_HEADER __ImageBase;
+#define HINST_THISCOMPONENT ((HINSTANCE)&__ImageBase)
+
 // mingw-w64 header typo?
 #ifndef IDirect3DSwapChain9Ex_GetBackBuffer
 #define IDirect3DSwapChain9Ex_GetBackBuffer IDirect3DSwapChain9EX_GetBackBuffer
@@ -99,7 +102,7 @@ static int os_ctx_create(struct MPGLContext *ctx)
         .cbSize = sizeof(WNDCLASSEXW),
         .style = CS_OWNDC,
         .lpfnWndProc = DefWindowProc,
-        .hInstance = GetModuleHandleW(NULL),
+        .hInstance = HINST_THISCOMPONENT,
         .lpszClassName = os_wnd_class,
     });
 
@@ -107,7 +110,7 @@ static int os_ctx_create(struct MPGLContext *ctx)
     // possible to use the VO window, but MSDN recommends against drawing to
     // the same window with flip mode present and other APIs, so play it safe.
     p->os_wnd = CreateWindowExW(0, os_wnd_class, os_wnd_class, 0, 0, 0, 200,
-        200, NULL, NULL, GetModuleHandleW(NULL), NULL);
+        200, NULL, NULL, HINST_THISCOMPONENT, NULL);
     p->os_dc = GetDC(p->os_wnd);
     if (!p->os_dc) {
         MP_FATAL(ctx->vo, "Couldn't create window for offscreen rendering\n");
diff --git a/video/out/w32_common.c b/video/out/w32_common.c
index ea8ec8772e..f3b59f183f 100644
--- a/video/out/w32_common.c
+++ b/video/out/w32_common.c
@@ -45,6 +45,9 @@
 #include "misc/rendezvous.h"
 #include "mpv_talloc.h"
 
+EXTERN_C IMAGE_DOS_HEADER __ImageBase;
+#define HINST_THISCOMPONENT ((HINSTANCE)&__ImageBase)
+
 static const wchar_t classname[] = L"mpv";
 
 static __thread struct vo_w32_state *w32_thread_context;
@@ -1178,7 +1181,7 @@ static void gui_thread_reconfig(void *ptr)
         vo->dheight = r.bottom;
     }
 
-    // Recenter window around old position on new video size 
+    // Recenter window around old position on new video size
     // excluding the case when initial positon handled by win_state.
     if (!pos_init) {
         w32->window_x += w32->dw / 2 - vo->dwidth / 2;
@@ -1221,14 +1224,12 @@ static void *gui_thread(void *ptr)
 
     thread_disable_ime();
 
-    HINSTANCE hInstance = GetModuleHandleW(NULL);
-
     WNDCLASSEXW wcex = {
         .cbSize = sizeof wcex,
         .style = CS_HREDRAW | CS_VREDRAW,
         .lpfnWndProc = WndProc,
-        .hInstance = hInstance,
-        .hIcon = LoadIconW(hInstance, L"IDI_ICON1"),
+        .hInstance = HINST_THISCOMPONENT,
+        .hIcon = LoadIconW(HINST_THISCOMPONENT, L"IDI_ICON1"),
         .hCursor = LoadCursor(NULL, IDC_ARROW),
         .lpszClassName = classname,
     };
@@ -1246,13 +1247,13 @@ static void *gui_thread(void *ptr)
                                       classname,
                                       WS_CHILD | WS_VISIBLE,
                                       0, 0, r.right, r.bottom,
-                                      w32->parent, 0, hInstance, NULL);
+                                      w32->parent, 0, HINST_THISCOMPONENT, NULL);
     } else {
         w32->window = CreateWindowExW(0, classname,
                                       classname,
                                       update_style(w32, 0),
                                       CW_USEDEFAULT, SW_HIDE, 100, 100,
-                                      0, 0, hInstance, NULL);
+                                      0, 0, HINST_THISCOMPONENT, NULL);
     }
 
     if (!w32->window) {