vo_opengl: only open one OpenGL/DX interop handle when using dxva2

Previously, gl->DXOpenDeviceNV was called twice using dxva2 with dxinterop. AMD
drivers refused to allow this. With this commit, context_dxinterop sets its own
implementation of MPGetNativeDisplay, which can return either a
IDirect3DDevice9Ex or a dxinterop_device_HANDLE depending on the "name" request
string. hwdec_dxva2gldx then requests both of these avoiding the need to call
gl->DXOpenDeviceNV a second time.

2 files changed, 23 insertions, 18 deletions

diff --git a/video/out/opengl/context_dxinterop.c b/video/out/opengl/context_dxinterop.c
index d3287f1f28..4dfc3c2108 100644
--- a/video/out/opengl/context_dxinterop.c
+++ b/video/out/opengl/context_dxinterop.c
@@ -543,6 +543,20 @@ static int GLAPIENTRY dxinterop_swap_interval(int interval)
     return 1;
 }
 
+static void * GLAPIENTRY dxinterop_get_native_display(const char *name)
+{
+    if (!current_ctx || !name)
+        return NULL;
+    struct priv *p = current_ctx->priv;
+
+    if (p->device && strcmp("IDirect3DDevice9Ex", name) == 0) {
+        return p->device;
+    } else if (p->device_h && strcmp("dxinterop_device_HANDLE", name) == 0) {
+        return p->device_h;
+    }
+    return NULL;
+}
+
 static int dxinterop_init(struct MPGLContext *ctx, int flags)
 {
     struct priv *p = ctx->priv;
@@ -566,6 +580,8 @@ static int dxinterop_init(struct MPGLContext *ctx, int flags)
 
     gl->SwapInterval = dxinterop_swap_interval;
 
+    gl->MPGetNativeDisplay = dxinterop_get_native_display;
+
     if (d3d_create(ctx) < 0)
         goto fail;
     if (d3d_size_dependent_create(ctx) < 0)
@@ -581,9 +597,6 @@ static int dxinterop_init(struct MPGLContext *ctx, int flags)
 
     DwmEnableMMCSS(TRUE);
 
-    ctx->native_display_type = "IDirect3DDevice9Ex";
-    ctx->native_display = p->device;
-
     return 0;
 fail:
     dxinterop_uninit(ctx);
diff --git a/video/out/opengl/hwdec_dxva2gldx.c b/video/out/opengl/hwdec_dxva2gldx.c
index 9e193fc3a5..69be0ccd18 100644
--- a/video/out/opengl/hwdec_dxva2gldx.c
+++ b/video/out/opengl/hwdec_dxva2gldx.c
@@ -72,13 +72,8 @@ static void destroy_objects(struct gl_hwdec *hw)
 static void destroy(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
-    GL *gl = hw->gl;
     destroy_objects(hw);
 
-    if (!gl->DXCloseDeviceNV(p->device_h))
-        MP_ERR(hw, "Failed to close Direct3D device in OpenGL %s\n",
-               mp_LastError_to_str());
-
     if (p->device)
         IDirect3DDevice9Ex_Release(p->device);
 }
@@ -94,28 +89,25 @@ static int create(struct gl_hwdec *hw)
     struct priv *p = talloc_zero(hw, struct priv);
     hw->priv = p;
 
+    // AMD drivers won't open multiple dxinterop HANDLES on the same D3D device,
+    // so we request the one already in use by context_dxinterop
+    p->device_h = gl->MPGetNativeDisplay("dxinterop_device_HANDLE");
+    if (!p->device_h)
+        return -1;
+
+    // But we also still need the actual D3D device
     p->device = gl->MPGetNativeDisplay("IDirect3DDevice9Ex");
     if (!p->device)
         return -1;
     IDirect3DDevice9Ex_AddRef(p->device);
     p->ctx.d3d9_device = (IDirect3DDevice9 *)p->device;
 
-    p->device_h = gl->DXOpenDeviceNV(p->device);
-    if (!p->device_h) {
-        MP_ERR(hw, "Failed to open Direct3D device in OpenGL: %s\n",
-               mp_LastError_to_str());
-        goto fail;
-    }
-
     p->ctx.hwctx.type = HWDEC_DXVA2;
     p->ctx.hwctx.d3d_ctx = &p->ctx;
 
     hw->hwctx = &p->ctx.hwctx;
     hw->converted_imgfmt = SHARED_SURFACE_MPFMT;
     return 0;
-fail:
-    destroy(hw);
-    return -1;
 }
 
 static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)