mp_image: pass through unknown AVFrame side data

Useful for libavfilter. Somewhat risky, because we can't ensure the
consistency of the unknown side data (but this is a general problem with
side data, and libavfilter filters will usually get it wrong too _if_
there are conflict cases).

Fixes #5569.

2 files changed, 42 insertions, 1 deletions

diff --git a/video/mp_image.c b/video/mp_image.c
index 80c1ee266a..ecf4be7768 100644
--- a/video/mp_image.c
+++ b/video/mp_image.c
@@ -209,6 +209,9 @@ static void mp_image_destructor(void *ptr)
     av_buffer_unref(&mpi->hwctx);
     av_buffer_unref(&mpi->icc_profile);
     av_buffer_unref(&mpi->a53_cc);
+    for (int n = 0; n < mpi->num_ff_side_data; n++)
+        av_buffer_unref(&mpi->ff_side_data[n].buf);
+    talloc_free(mpi->ff_side_data);
 }
 
 int mp_chroma_div_up(int size, int shift)
@@ -329,6 +332,10 @@ struct mp_image *mp_image_new_ref(struct mp_image *img)
     fail |= !ref_buffer(&new->icc_profile);
     fail |= !ref_buffer(&new->a53_cc);
 
+    new->ff_side_data = talloc_memdup(NULL, new->ff_side_data,
+                        new->num_ff_side_data * sizeof(new->ff_side_data[0]));
+    for (int n = 0; n < new->num_ff_side_data; n++)
+        fail |= !ref_buffer(&new->ff_side_data[n].buf);
 
     if (!fail)
         return new;
@@ -897,6 +904,15 @@ struct mp_image *mp_image_from_av_frame(struct AVFrame *src)
     sd = av_frame_get_side_data(src, AV_FRAME_DATA_A53_CC);
     if (sd)
         dst->a53_cc = sd->buf;
+
+    for (int n = 0; n < src->nb_side_data; n++) {
+        sd = src->side_data[n];
+        struct mp_ff_side_data mpsd = {
+            .type = sd->type,
+            .buf = sd->buf,
+        };
+        MP_TARRAY_APPEND(NULL, dst->ff_side_data, dst->num_ff_side_data, mpsd);
+    }
 #endif
 
     if (dst->hwctx) {
@@ -908,7 +924,12 @@ struct mp_image *mp_image_from_av_frame(struct AVFrame *src)
             fns->complete_image_params(dst);
     }
 
-    return mp_image_new_ref(dst);
+    struct mp_image *res = mp_image_new_ref(dst);
+
+    // Allocated, but non-refcounted data.
+    talloc_free(dst->ff_side_data);
+
+    return res;
 }
 
 
@@ -981,6 +1002,18 @@ struct AVFrame *mp_image_to_av_frame(struct mp_image *src)
             abort();
         clm->MaxCLL = src->params.color.sig_peak * MP_REF_WHITE;
     }
+
+    // Add back side data, but only for types which are not specially handled
+    // above. Keep in mind that the types above will be out of sync anyway.
+    for (int n = 0; n < new_ref->num_ff_side_data; n++) {
+        struct mp_ff_side_data *mpsd = &new_ref->ff_side_data[n];
+        if (!av_frame_get_side_data(dst, mpsd->type)) {
+            AVFrameSideData *sd = ffmpeg_garbage(dst, mpsd->type, mpsd->buf);
+            if (!sd)
+                abort();
+            mpsd->buf = NULL;
+        }
+    }
 #endif
 
     talloc_free(new_ref);
diff --git a/video/mp_image.h b/video/mp_image.h
index f7969a4314..88e261306f 100644
--- a/video/mp_image.h
+++ b/video/mp_image.h
@@ -121,8 +121,16 @@ typedef struct mp_image {
     struct AVBufferRef *icc_profile;
     // Closed captions packet, if any (only after decoder)
     struct AVBufferRef *a53_cc;
+    // Other side data we don't care about.
+    struct mp_ff_side_data *ff_side_data;
+    int num_ff_side_data;
 } mp_image_t;
 
+struct mp_ff_side_data {
+    int type;
+    struct AVBufferRef *buf;
+};
+
 int mp_chroma_div_up(int size, int shift);
 
 int mp_image_get_alloc_size(int imgfmt, int w, int h, int stride_align);