vo_opengl: fix d3d11 hardware decoding probing on Windows 7

Although D3D11 video decoding is unuspported on Windows 7, the
associated APIs almost work. Where they fail is texture creation, where
we try to create D3D11_BIND_DECODER surfaces. So specifically try to
detect this situation.

One issue is that once the hwdec interop is created, the damage is done,
and it can't use another backend (because currently only 1 hwdec backend
is supported). So that's where we prevent attempts to use it.

It still can fail when trying to use d3d11va-copy (since that doesn't
require an interop backend), but at that point we don't care anymore -
dxva2(-copy) is tried before that anyway.

4 files changed, 38 insertions, 0 deletions

diff --git a/video/out/opengl/angle_common.c b/video/out/opengl/angle_common.c
new file mode 100644
index 0000000000..21cc924714
--- /dev/null
+++ b/video/out/opengl/angle_common.c
@@ -0,0 +1,13 @@
+#include "angle_common.h"
+
+// Test if Direct3D11 can be used by us. Basically, this prevents trying to use
+// D3D11 on Win7, and then failing somewhere in the process.
+bool d3d11_check_decoding(ID3D11Device *dev)
+{
+    HRESULT hr;
+    // We assume that NV12 is always supported, if hw decoding is supported at
+    // all.
+    UINT supported = 0;
+    hr = ID3D11Device_CheckFormatSupport(dev, DXGI_FORMAT_NV12, &supported);
+    return !FAILED(hr) && (supported & D3D11_BIND_DECODER);
+}
diff --git a/video/out/opengl/angle_common.h b/video/out/opengl/angle_common.h
new file mode 100644
index 0000000000..14ecd6ab3c
--- /dev/null
+++ b/video/out/opengl/angle_common.h
@@ -0,0 +1,13 @@
+#ifndef MP_ANGLE_COMMON_H
+#define MP_ANGLE_COMMON_H
+
+#include <initguid.h>
+#include <assert.h>
+#include <windows.h>
+#include <d3d11.h>
+
+#include <stdbool.h>
+
+bool d3d11_check_decoding(ID3D11Device *dev);
+
+#endif
\ No newline at end of file
diff --git a/video/out/opengl/hwdec_d3d11egl.c b/video/out/opengl/hwdec_d3d11egl.c
index dbf52dc3d2..26d992dea3 100644
--- a/video/out/opengl/hwdec_d3d11egl.c
+++ b/video/out/opengl/hwdec_d3d11egl.c
@@ -23,6 +23,7 @@
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
 
+#include "angle_common.h"
 #include "angle_dynamic.h"
 
 #include "common/common.h"
@@ -173,6 +174,11 @@ static int create(struct gl_hwdec *hw)
         goto fail;
     ID3D11Device_AddRef(p->d3d11_device);
 
+    if (!d3d11_check_decoding(p->d3d11_device)) {
+        MP_VERBOSE(hw, "D3D11 video decoding not supported on this system.\n");
+        goto fail;
+    }
+
     ID3D10Multithread *multithread;
     hr = ID3D11Device_QueryInterface(p->d3d11_device, &IID_ID3D10Multithread,
                                      (void **)&multithread);
diff --git a/video/out/opengl/hwdec_d3d11eglrgb.c b/video/out/opengl/hwdec_d3d11eglrgb.c
index adb60ef4d2..9e83ac4b36 100644
--- a/video/out/opengl/hwdec_d3d11eglrgb.c
+++ b/video/out/opengl/hwdec_d3d11eglrgb.c
@@ -23,6 +23,7 @@
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
 
+#include "angle_common.h"
 #include "angle_dynamic.h"
 
 #include "common/common.h"
@@ -154,6 +155,11 @@ static int create(struct gl_hwdec *hw)
     ID3D10Multithread_SetMultithreadProtected(multithread, TRUE);
     ID3D10Multithread_Release(multithread);
 
+    if (!d3d11_check_decoding(p->d3d11_device)) {
+        MP_VERBOSE(hw, "D3D11 video decoding not supported on this system.\n");
+        goto fail;
+    }
+
     hr = ID3D11Device_QueryInterface(p->d3d11_device, &IID_ID3D11VideoDevice,
                                      (void **)&p->video_dev);
     if (FAILED(hr))