video: change hw_subfmt meaning

The hw_subfmt field roughly corresponds to the field
AVHWFramesContext.sw_format in ffmpeg. The ffmpeg one is of the type
AVPixelFormat (instead of the underlying hardware format), so it's a
good idea to switch to this too for preparation.

Now the hw_subfmt field is an mp_imgfmt instead of an opaque/API-
specific number. VDPAU and Direct3D11 already used mp_imgfmt, but
Videotoolbox and VAAPI had to be switched.

One somewhat user-visible change is that the verbose log will now always
show the hw_subfmt as image format, instead of as nonsensical number.

(In the end it would be good if we could switch to AVHWFramesContext
completely, but the upstream API is incomplete and doesn't cover
Direct3D11 and Videotoolbox.)

2 files changed, 4 insertions, 5 deletions

diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index 6ddfa66e0a..17399a80f4 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -186,7 +186,7 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 {
     assert(params->imgfmt == hw->driver->imgfmt);
 
-    struct vt_format *f = vt_get_gl_format(params->hw_subfmt);
+    struct vt_format *f = vt_get_gl_format_from_imgfmt(params->hw_subfmt);
     if (!f) {
         MP_ERR(hw, "Unsupported CVPixelBuffer format.\n");
         return -1;
diff --git a/video/out/opengl/hwdec_vaegl.c b/video/out/opengl/hwdec_vaegl.c
index 6c52cdde11..1de5969bc4 100644
--- a/video/out/opengl/hwdec_vaegl.c
+++ b/video/out/opengl/hwdec_vaegl.c
@@ -250,17 +250,16 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     }
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
-    p->current_mpfmt = va_fourcc_to_imgfmt(params->hw_subfmt);
+    p->current_mpfmt = params->hw_subfmt;
     if (p->current_mpfmt != IMGFMT_NV12 &&
         p->current_mpfmt != IMGFMT_420P)
     {
         MP_FATAL(p, "unsupported VA image format %s\n",
-                 mp_tag_str(params->hw_subfmt));
+                 mp_imgfmt_to_name(p->current_mpfmt));
         return -1;
     }
 
-    MP_VERBOSE(p, "format: %s %s\n", mp_tag_str(params->hw_subfmt),
-               mp_imgfmt_to_name(p->current_mpfmt));
+    MP_VERBOSE(p, "hw format: %s\n", mp_imgfmt_to_name(p->current_mpfmt));
 
     params->imgfmt = p->current_mpfmt;