demux: estimate total packet size, deprecate packet number limits

It's all explained in the DOCS changes. Although this option was always
kind of obscure and pointless. Until it is removed, the only reason for
setting it would be to raise the static default limit, so change its
default to INT_MAX so that it does nothing by default.

3 files changed, 32 insertions, 4 deletions

diff --git a/demux/demux.c b/demux/demux.c
index 0c45083e2d..acccdf5363 100644
--- a/demux/demux.c
+++ b/demux/demux.c
@@ -97,7 +97,8 @@ struct demux_opts {
 const struct m_sub_options demux_conf = {
     .opts = (const struct m_option[]){
         OPT_DOUBLE("demuxer-readahead-secs", min_secs, M_OPT_MIN, .min = 0),
-        OPT_INTRANGE("demuxer-max-packets", max_packs, 0, 0, INT_MAX),
+        OPT_INTRANGE("demuxer-max-packets", max_packs, 0, 0, INT_MAX,
+                     .deprecation_message = "use --demuxer-max-bytes"),
         OPT_INTRANGE("demuxer-max-bytes", max_bytes, 0, 0, INT_MAX),
         OPT_FLAG("force-seekable", force_seekable, 0),
         OPT_DOUBLE("cache-secs", min_secs_cache, M_OPT_MIN, .min = 0),
@@ -106,7 +107,7 @@ const struct m_sub_options demux_conf = {
     },
     .size = sizeof(struct demux_opts),
     .defaults = &(const struct demux_opts){
-        .max_packs = 16000,
+        .max_packs = INT_MAX,
         .max_bytes = 400 * 1024 * 1024,
         .min_secs = 1.0,
         .min_secs_cache = 10.0,
@@ -558,7 +559,7 @@ void demux_add_packet(struct sh_stream *stream, demux_packet_t *dp)
     dp->next = NULL;
 
     ds->packs++;
-    ds->bytes += dp->len;
+    ds->bytes += demux_packet_estimate_total_size(dp);
     if (ds->tail) {
         // next packet in stream
         ds->tail->next = dp;
@@ -777,7 +778,7 @@ static struct demux_packet *dequeue_packet(struct demux_stream *ds)
     pkt->next = NULL;
     if (!ds->head)
         ds->tail = NULL;
-    ds->bytes -= pkt->len;
+    ds->bytes -= demux_packet_estimate_total_size(pkt);
     ds->packs--;
 
     double ts = pkt->dts == MP_NOPTS_VALUE ? pkt->pts : pkt->dts;
diff --git a/demux/packet.c b/demux/packet.c
index 47f0cb87dc..9f9f97ea53 100644
--- a/demux/packet.c
+++ b/demux/packet.c
@@ -130,6 +130,32 @@ struct demux_packet *demux_copy_packet(struct demux_packet *dp)
     return new;
 }
 
+#define ROUND_ALLOC(s) MP_ALIGN_UP(s, 64)
+
+// Attempt to estimate the total memory consumption of the given packet.
+// This is important if we store thousands of packets and not to exceed
+// user-provided limits. Of course we can't know how much memory internal
+// fragmentation of the libc memory allocator will waste.
+// Note that this should return a "stable" value - e.g. if a new packet ref
+// is created, this should return the same value with the new ref. (This
+// implies the value is not exact and does not return the actual size of
+// memory wasted due to internal fragmentation.)
+size_t demux_packet_estimate_total_size(struct demux_packet *dp)
+{
+    size_t size = ROUND_ALLOC(sizeof(struct demux_packet));
+    size += ROUND_ALLOC(dp->len);
+    if (dp->avpacket) {
+        size += ROUND_ALLOC(sizeof(AVPacket));
+        size += ROUND_ALLOC(sizeof(AVBufferRef));
+        size += 64; // upper bound estimate on sizeof(AVBuffer)
+        size += ROUND_ALLOC(dp->avpacket->side_data_elems *
+                            sizeof(dp->avpacket->side_data[0]));
+        for (int n = 0; n < dp->avpacket->side_data_elems; n++)
+            size += ROUND_ALLOC(dp->avpacket->side_data[n].size);
+    }
+    return size;
+}
+
 int demux_packet_set_padding(struct demux_packet *dp, int start, int end)
 {
 #if LIBAVCODEC_VERSION_MICRO >= 100
diff --git a/demux/packet.h b/demux/packet.h
index d669d02376..03204c8de2 100644
--- a/demux/packet.h
+++ b/demux/packet.h
@@ -51,6 +51,7 @@ struct demux_packet *new_demux_packet_from(void *data, size_t len);
 void demux_packet_shorten(struct demux_packet *dp, size_t len);
 void free_demux_packet(struct demux_packet *dp);
 struct demux_packet *demux_copy_packet(struct demux_packet *dp);
+size_t demux_packet_estimate_total_size(struct demux_packet *dp);
 
 void demux_packet_copy_attribs(struct demux_packet *dst, struct demux_packet *src);