vaapi: move AVHWFramesContext setup code to common code

In a way it can be reused. For now, sw_format and initial_pool_size
determination are still vaapi-specific. I'm hoping this can be eventally
moved to libavcodec in some way. Checking the supported_formats array is
not really vaapi-specific, and could be moved to the generic code path
too, but for now it would make things more complex.

hw_cuda.c can't use this, but hw_vdpau.c will in the following commit.

3 files changed, 63 insertions, 52 deletions

diff --git a/video/decode/hw_vaapi.c b/video/decode/hw_vaapi.c
index 13e38f2258..45558cd6e7 100644
--- a/video/decode/hw_vaapi.c
+++ b/video/decode/hw_vaapi.c
@@ -40,24 +40,19 @@ struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
     struct mp_hwdec_ctx *hwdev;
-
-    AVBufferRef *frames_ref;
 };
 
-
 static int init_decoder(struct lavc_ctx *ctx, int w, int h)
 {
     struct priv *p = ctx->hwdec_priv;
     // From avconv_vaapi.c. Disgusting, but apparently this is the best we get.
-    int required_sw_format = ctx->avctx->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ?
-                             AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
-
-    assert(!ctx->avctx->hw_frames_ctx);
+    int sw_format = ctx->avctx->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ?
+                    AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
 
     // The video output might not support all formats.
     // Note that supported_formats==NULL means any are accepted.
     if (p->hwdev && p->hwdev->supported_formats) {
-        int mp_format = pixfmt2imgfmt(required_sw_format);
+        int mp_format = pixfmt2imgfmt(sw_format);
         bool found = false;
         for (int n = 0; p->hwdev->supported_formats[n]; n++) {
             if (p->hwdev->supported_formats[n] == mp_format) {
@@ -72,48 +67,8 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
         }
     }
 
-    if (p->frames_ref) {
-        AVHWFramesContext *fctx = (void *)p->frames_ref->data;
-        if (fctx->width != w || fctx->height != h ||
-            fctx->sw_format != required_sw_format)
-        {
-            av_buffer_unref(&p->frames_ref);
-        }
-    }
-
-    if (!p->frames_ref) {
-        p->frames_ref = av_hwframe_ctx_alloc(p->ctx->av_device_ref);
-        if (!p->frames_ref)
-            return -1;
-
-        AVHWFramesContext *fctx = (void *)p->frames_ref->data;
-
-        fctx->format = AV_PIX_FMT_VAAPI;
-        fctx->sw_format = required_sw_format;
-        fctx->width = w;
-        fctx->height = h;
-
-        fctx->initial_pool_size = hwdec_get_max_refs(ctx) + ADDITIONAL_SURFACES;
-
-        // Some mpv downstream code uses this.
-        fctx->user_opaque = p->ctx;
-
-        va_lock(p->ctx);
-        int res = av_hwframe_ctx_init(p->frames_ref);
-        va_unlock(p->ctx);
-
-        if (res > 0) {
-            MP_ERR(ctx, "Failed to allocate hw frames.\n");
-            av_buffer_unref(&p->frames_ref);
-            return -1;
-        }
-    }
-
-    ctx->avctx->hw_frames_ctx = av_buffer_ref(p->frames_ref);
-    if (!ctx->avctx->hw_frames_ctx)
-        return -1;
-
-    return 0;
+    return hwdec_setup_hw_frames_ctx(ctx, p->ctx->av_device_ref, sw_format,
+                                hwdec_get_max_refs(ctx) + ADDITIONAL_SURFACES);
 }
 
 static void uninit(struct lavc_ctx *ctx)
@@ -123,8 +78,6 @@ static void uninit(struct lavc_ctx *ctx)
     if (!p)
         return;
 
-    av_buffer_unref(&p->frames_ref);
-
     if (!p->hwdev)
         va_destroy(p->ctx);
 
diff --git a/video/decode/lavc.h b/video/decode/lavc.h
index fe7f9937f4..fd1c7feb42 100644
--- a/video/decode/lavc.h
+++ b/video/decode/lavc.h
@@ -64,6 +64,8 @@ typedef struct lavc_ctx {
     int hwdec_fail_count;
 
     struct mp_image_pool *hwdec_swpool;
+
+    AVBufferRef *cached_hw_frames_ctx;
 } vd_ffmpeg_ctx;
 
 struct vd_lavc_hwdec {
@@ -118,6 +120,8 @@ const struct hwdec_profile_entry *hwdec_find_profile(
 bool hwdec_check_codec_support(const char *codec,
                                const struct hwdec_profile_entry *table);
 int hwdec_get_max_refs(struct lavc_ctx *ctx);
+int hwdec_setup_hw_frames_ctx(struct lavc_ctx *ctx, AVBufferRef *device_ctx,
+                              int av_sw_format, int initial_pool_size);
 
 const char *hwdec_find_decoder(const char *codec, const char *suffix);
 
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 10f9723db0..a5454056db 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -598,6 +598,7 @@ static void uninit_avctx(struct dec_video *vd)
 
     flush_all(vd);
     av_frame_free(&ctx->pic);
+    av_buffer_unref(&ctx->cached_hw_frames_ctx);
 
     if (ctx->avctx) {
         if (avcodec_close(ctx->avctx) < 0)
@@ -648,6 +649,59 @@ static void update_image_params(struct dec_video *vd, AVFrame *frame,
     params->stereo_in = vd->codec->stereo_mode;
 }
 
+// Allocate and set AVCodecContext.hw_frames_ctx. Also caches them on redundant
+// calls (useful because seeks issue get_format, which clears hw_frames_ctx).
+//  device_ctx: reference to an AVHWDeviceContext
+//  av_sw_format: AV_PIX_FMT_ for the underlying hardware frame format
+//  initial_pool_size: number of frames in the memory pool on creation
+// Return >=0 on success, <0 on error.
+int hwdec_setup_hw_frames_ctx(struct lavc_ctx *ctx, AVBufferRef *device_ctx,
+                              int av_sw_format, int initial_pool_size)
+{
+    int w = ctx->avctx->coded_width;
+    int h = ctx->avctx->coded_height;
+    int av_hw_format = imgfmt2pixfmt(ctx->hwdec_fmt);
+
+    if (ctx->cached_hw_frames_ctx) {
+        AVHWFramesContext *fctx = (void *)ctx->cached_hw_frames_ctx->data;
+        if (fctx->width != w || fctx->height != h ||
+            fctx->sw_format != av_sw_format ||
+            fctx->format != av_hw_format)
+        {
+            av_buffer_unref(&ctx->cached_hw_frames_ctx);
+        }
+    }
+
+    if (!ctx->cached_hw_frames_ctx) {
+        ctx->cached_hw_frames_ctx = av_hwframe_ctx_alloc(device_ctx);
+        if (!ctx->cached_hw_frames_ctx)
+            return -1;
+
+        AVHWFramesContext *fctx = (void *)ctx->cached_hw_frames_ctx->data;
+
+        fctx->format = av_hw_format;
+        fctx->sw_format = av_sw_format;
+        fctx->width = w;
+        fctx->height = h;
+
+        fctx->initial_pool_size = initial_pool_size;
+
+        hwdec_lock(ctx);
+        int res = av_hwframe_ctx_init(ctx->cached_hw_frames_ctx);
+        hwdec_unlock(ctx);
+
+        if (res > 0) {
+            MP_ERR(ctx, "Failed to allocate hw frames.\n");
+            av_buffer_unref(&ctx->cached_hw_frames_ctx);
+            return -1;
+        }
+    }
+
+    assert(!ctx->avctx->hw_frames_ctx);
+    ctx->avctx->hw_frames_ctx = av_buffer_ref(ctx->cached_hw_frames_ctx);
+    return ctx->avctx->hw_frames_ctx ? 0 : -1;
+}
+
 static enum AVPixelFormat get_format_hwdec(struct AVCodecContext *avctx,
                                            const enum AVPixelFormat *fmt)
 {