video: fix VideoToolbox/VDA autodetection

This affects vo_opengl_cb in particular: it'll most likely auto-load
VDA, and then the VideoToolbox decoder won't work. And everything fails.

This is mainly caused by FFmpeg using separate pixfmts for the _same_
thing (CVPixelBuffers), simply because libavcodec's architecture demands
that hwaccel backends are selected by pixfmts. (Which makes no sense,
but now we have the mess.)

So instead of duplicating FFmpeg's misdesign, just change the format to
our own canonical one on the image output by the decoder. Now the GL
interop code is exactly the same for VDA and VT, and we use the VT name
only.

1 files changed, 12 insertions, 2 deletions

diff --git a/video/decode/vda.c b/video/decode/vda.c
index cc2c7b0795..372061f194 100644
--- a/video/decode/vda.c
+++ b/video/decode/vda.c
@@ -26,11 +26,10 @@
 #include "video/decode/lavc.h"
 #include "config.h"
 
-
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *decoder)
 {
-    hwdec_request_api(info, "vda");
+    hwdec_request_api(info, "videotoolbox");
     if (!info || !info->hwctx)
         return HWDEC_ERR_NO_CTX;
     if (mp_codec_to_av_codec_id(decoder) != AV_CODEC_ID_H264)
@@ -98,6 +97,16 @@ static void uninit(struct lavc_ctx *ctx)
         av_vda_default_free(ctx->avctx);
 }
 
+static struct mp_image *process_image(struct lavc_ctx *ctx, struct mp_image *img)
+{
+    // Same representation. IMGFMT_VDA is only needed to select the libavcodec
+    // hwaccel driver.
+    if (img->imgfmt == IMGFMT_VDA)
+        mp_image_setfmt(img, IMGFMT_VIDEOTOOLBOX);
+
+    return img;
+}
+
 const struct vd_lavc_hwdec mp_vd_lavc_vda = {
     .type = HWDEC_VDA,
     .image_format = IMGFMT_VDA,
@@ -105,4 +114,5 @@ const struct vd_lavc_hwdec mp_vd_lavc_vda = {
     .init = init,
     .uninit = uninit,
     .init_decoder = init_decoder,
+    .process_image = process_image,
 };