vd_lavc: fix use after free in some hwdecs

fd339e3f53996efd2dae9525990da433d1e1bf89 introduced a regression that caused
segfault while uniniting dxva2 decoder (and possibly vdpau too). The problem was
that it freed the avctx earlier, before calling the backend-specific uninit
which referenced it.

Revert some of the changes of that commit, and avoid calling flush by
checking whether the codec is open instead.

(Based on a PR by Kevin Mitchell.)

Signed-off-by: wm4 <wm4@nowhere>

1 files changed, 3 insertions, 8 deletions

diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 0e8cbb1967..aca42e74c0 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -457,11 +457,6 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
 
 error:
     MP_ERR(vd, "Could not open codec.\n");
-    // Free it here to avoid attempting to flush+close.
-    if (ctx->avctx) {
-        av_freep(&ctx->avctx->extradata);
-        av_freep(&ctx->avctx);
-    }
     uninit_avctx(vd);
 }
 
@@ -469,7 +464,7 @@ static void reset_avctx(struct dec_video *vd)
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
 
-    if (ctx->avctx)
+    if (ctx->avctx && avcodec_is_open(ctx->avctx))
         avcodec_flush_buffers(ctx->avctx);
     ctx->flushing = false;
 }
@@ -495,15 +490,15 @@ static void uninit_avctx(struct dec_video *vd)
     if (ctx->avctx) {
         if (avcodec_close(ctx->avctx) < 0)
             MP_ERR(vd, "Could not close codec.\n");
-
         av_freep(&ctx->avctx->extradata);
-        av_freep(&ctx->avctx);
     }
 
     if (ctx->hwdec && ctx->hwdec->uninit)
         ctx->hwdec->uninit(ctx);
     ctx->hwdec = NULL;
 
+    av_freep(&ctx->avctx);
+
     ctx->hwdec_failed = false;
     ctx->hwdec_fail_count = 0;
     ctx->max_delay_queue = 0;