vdpau: get surface data as nv12 if possible

For now, this affects taking screenshots only.

If the video mixer is actually needed, we want to go through the video
mixer for screenshots too - which is why this function simply always
used the video mixer, and then copied the surface to RAM as RGBA.

Add reading the surface as nv12 if possible. If the format is correct,
and no special video processing (like deinterlacing) is used, then it's
read directly without going through the video mixer.

There's no particular reason for doing this, other than avoiding the
video mixer (like vo_opengl can do it now). Also, the next commit will
make use of this in vf_vdpaurb.c.

2 files changed, 54 insertions, 0 deletions

diff --git a/video/vdpau.c b/video/vdpau.c
index 73aa844abd..dffb02e22f 100644
--- a/video/vdpau.c
+++ b/video/vdpau.c
@@ -28,6 +28,53 @@
 #include "mp_image_pool.h"
 #include "vdpau_mixer.h"
 
+static struct mp_image *download_image_yuv(struct mp_hwdec_ctx *hwctx,
+                                           struct mp_image *mpi,
+                                           struct mp_image_pool *swpool)
+{
+    struct mp_vdpau_ctx *ctx = hwctx->ctx;
+    struct vdp_functions *vdp = &ctx->vdp;
+    VdpStatus vdp_st;
+
+    if (mpi->imgfmt != IMGFMT_VDPAU || mp_vdpau_mixed_frame_get(mpi))
+        return NULL;
+
+    VdpVideoSurface surface = (uintptr_t)mpi->planes[3];
+
+    VdpChromaType s_chroma_type;
+    uint32_t s_w, s_h;
+    vdp_st = vdp->video_surface_get_parameters(surface, &s_chroma_type, &s_w, &s_h);
+    CHECK_VDP_ERROR_NORETURN(ctx,
+                    "Error when calling vdp_video_surface_get_parameters");
+    if (vdp_st != VDP_STATUS_OK)
+        return NULL;
+
+    // Don't bother supporting other types for now.
+    if (s_chroma_type != VDP_CHROMA_TYPE_420)
+        return NULL;
+
+    // The allocation needs to be uncropped, because get_bits writes to it.
+    struct mp_image *out = mp_image_pool_get(swpool, IMGFMT_NV12, s_w, s_h);
+    if (!out)
+        return NULL;
+
+    mp_image_set_size(out, mpi->w, mpi->h);
+    mp_image_copy_attributes(out, mpi);
+
+    vdp_st = vdp->video_surface_get_bits_y_cb_cr(surface,
+                                                 VDP_YCBCR_FORMAT_NV12,
+                                                 (void * const *)out->planes,
+                                                 out->stride);
+    CHECK_VDP_ERROR_NORETURN(ctx,
+                "Error when calling vdp_output_surface_get_bits_y_cb_cr");
+    if (vdp_st != VDP_STATUS_OK) {
+        talloc_free(out);
+        return NULL;
+    }
+
+    return out;
+}
+
 static struct mp_image *download_image(struct mp_hwdec_ctx *hwctx,
                                        struct mp_image *mpi,
                                        struct mp_image_pool *swpool)
@@ -43,6 +90,10 @@ static struct mp_image *download_image(struct mp_hwdec_ctx *hwctx,
     int w, h;
     mp_image_params_get_dsize(&mpi->params, &w, &h);
 
+    res = download_image_yuv(hwctx, mpi, swpool);
+    if (res)
+        return res;
+
     // Abuse this lock for our own purposes. It could use its own lock instead.
     pthread_mutex_lock(&ctx->pool_lock);
 
diff --git a/video/vdpau.h b/video/vdpau.h
index db73a87dd7..389e1c7e9a 100644
--- a/video/vdpau.h
+++ b/video/vdpau.h
@@ -23,6 +23,9 @@
 #define CHECK_VDP_ERROR(ctx, message) \
     CHECK_VDP_ERROR_ST(ctx, message, return -1;)
 
+#define CHECK_VDP_ERROR_NORETURN(ctx, message) \
+    CHECK_VDP_ERROR_ST(ctx, message, ;)
+
 #define CHECK_VDP_WARNING(ctx, message) \
     do { \
         if (vdp_st != VDP_STATUS_OK) \