demux: add per-track metadata

...and ignore it. The main purpose is for retrieving per-track
replaygain tags. Other than that per-track tags are not used or accessed
by the playback core yet.

The demuxer infrastructure is still not really good with that whole
synchronization thing (at least in part due to being inherited from
mplayer's single-threaded architecture). A convoluted mechanism is
needed to transport the tags from demuxer thread to user thread. Two
factors contribute to the complexity: tags can change during playback,
and tracks (i.e. struct sh_stream) are not duplicated per thread.

In particular, we update the way replaygain tags are retrieved. We first
try to use per-track tags (common in Matroska) and global tags
(effectively formats like mp3). This part fixes #3405.

5 files changed, 143 insertions, 59 deletions

diff --git a/demux/demux.c b/demux/demux.c
index 5b4b3ec4c2..599218b690 100644
--- a/demux/demux.c
+++ b/demux/demux.c
@@ -232,6 +232,7 @@ struct sh_stream *demux_alloc_sh_stream(enum stream_type type)
         .ff_index = -1,     // may be overwritten by demuxer
         .demuxer_id = -1,   // ... same
         .codec = talloc_zero(sh, struct mp_codec_params),
+        .tags = talloc_zero(sh, struct mp_tags),
     };
     sh->codec->type = type;
     return sh;
@@ -276,6 +277,33 @@ void demux_add_sh_stream(struct demuxer *demuxer, struct sh_stream *sh)
     pthread_mutex_unlock(&in->lock);
 }
 
+// Update sh->tags (lazily). This must be called by demuxers which update
+// stream tags after init. (sh->tags can be accessed by the playback thread,
+// which means the demuxer thread cannot write or read it directly.)
+// Before init is finished, sh->tags can still be accessed freely.
+// Ownership of tags goes to the function.
+void demux_set_stream_tags(struct demuxer *demuxer, struct sh_stream *sh,
+                           struct mp_tags *tags)
+{
+    struct demux_internal *in = demuxer->in;
+    assert(demuxer == in->d_thread);
+
+    if (sh->ds) {
+        while (demuxer->num_update_stream_tags <= sh->index) {
+            MP_TARRAY_APPEND(demuxer, demuxer->update_stream_tags,
+                             demuxer->num_update_stream_tags, NULL);
+        }
+        talloc_free(demuxer->update_stream_tags[sh->index]);
+        demuxer->update_stream_tags[sh->index] = talloc_steal(demuxer, tags);
+
+        demux_changed(demuxer, DEMUX_EVENT_METADATA);
+    } else {
+        // not added yet
+        talloc_free(sh->tags);
+        sh->tags = talloc_steal(sh, tags);
+    }
+}
+
 // Return a stream with the given index. Since streams can only be added during
 // the lifetime of the demuxer, it is guaranteed that an index within the valid
 // range [0, demux_get_num_stream()) always returns a valid sh_stream pointer,
@@ -900,17 +928,18 @@ static int decode_float(char *str, float *out)
     return 0;
 }
 
-static int decode_gain(demuxer_t *demuxer, const char *tag, float *out)
+static int decode_gain(struct mp_log *log, struct mp_tags *tags,
+                       const char *tag, float *out)
 {
     char *tag_val = NULL;
     float dec_val;
 
-    tag_val = mp_tags_get_str(demuxer->metadata, tag);
+    tag_val = mp_tags_get_str(tags, tag);
     if (!tag_val)
         return -1;
 
     if (decode_float(tag_val, &dec_val)) {
-        mp_msg(demuxer->log, MSGL_ERR, "Invalid replaygain value\n");
+        mp_msg(log, MSGL_ERR, "Invalid replaygain value\n");
         return -1;
     }
 
@@ -918,14 +947,15 @@ static int decode_gain(demuxer_t *demuxer, const char *tag, float *out)
     return 0;
 }
 
-static int decode_peak(demuxer_t *demuxer, const char *tag, float *out)
+static int decode_peak(struct mp_log *log, struct mp_tags *tags,
+                       const char *tag, float *out)
 {
     char *tag_val = NULL;
     float dec_val;
 
     *out = 1.0;
 
-    tag_val = mp_tags_get_str(demuxer->metadata, tag);
+    tag_val = mp_tags_get_str(tags, tag);
     if (!tag_val)
         return 0;
 
@@ -939,38 +969,47 @@ static int decode_peak(demuxer_t *demuxer, const char *tag, float *out)
     return 0;
 }
 
-static void apply_replaygain(demuxer_t *demuxer, struct replaygain_data *rg)
-{
-    struct demux_internal *in = demuxer->in;
-    for (int n = 0; n < in->num_streams; n++) {
-        struct sh_stream *sh = in->streams[n];
-        if (sh->type == STREAM_AUDIO && !sh->codec->replaygain_data) {
-            MP_VERBOSE(demuxer, "Replaygain: Track=%f/%f Album=%f/%f\n",
-                       rg->track_gain, rg->track_peak,
-                       rg->album_gain, rg->album_peak);
-            sh->codec->replaygain_data = talloc_memdup(in, rg, sizeof(*rg));
-        }
-    }
-}
-
-static void demux_export_replaygain(demuxer_t *demuxer)
+static struct replaygain_data *decode_rgain(struct mp_log *log,
+                                            struct mp_tags *tags)
 {
     struct replaygain_data rg = {0};
 
-    if (!decode_gain(demuxer, "REPLAYGAIN_TRACK_GAIN", &rg.track_gain) &&
-        !decode_peak(demuxer, "REPLAYGAIN_TRACK_PEAK", &rg.track_peak) &&
-        !decode_gain(demuxer, "REPLAYGAIN_ALBUM_GAIN", &rg.album_gain) &&
-        !decode_peak(demuxer, "REPLAYGAIN_ALBUM_PEAK", &rg.album_peak))
+    if (!decode_gain(log, tags, "REPLAYGAIN_TRACK_GAIN", &rg.track_gain) &&
+        !decode_peak(log, tags, "REPLAYGAIN_TRACK_PEAK", &rg.track_peak) &&
+        !decode_gain(log, tags, "REPLAYGAIN_ALBUM_GAIN", &rg.album_gain) &&
+        !decode_peak(log, tags, "REPLAYGAIN_ALBUM_PEAK", &rg.album_peak))
     {
-        apply_replaygain(demuxer, &rg);
+        return talloc_memdup(NULL, &rg, sizeof(rg));
     }
 
-    if (!decode_gain(demuxer, "REPLAYGAIN_GAIN", &rg.track_gain) &&
-        !decode_peak(demuxer, "REPLAYGAIN_PEAK", &rg.track_peak))
+    if (!decode_gain(log, tags, "REPLAYGAIN_GAIN", &rg.track_gain) &&
+        !decode_peak(log, tags, "REPLAYGAIN_PEAK", &rg.track_peak))
     {
         rg.album_gain = rg.track_gain;
         rg.album_peak = rg.track_peak;
-        apply_replaygain(demuxer, &rg);
+        return talloc_memdup(NULL, &rg, sizeof(rg));
+    }
+
+    return NULL;
+}
+
+static void demux_update_replaygain(demuxer_t *demuxer)
+{
+    struct demux_internal *in = demuxer->in;
+    for (int n = 0; n < in->num_streams; n++) {
+        struct sh_stream *sh = in->streams[n];
+        if (sh->type == STREAM_AUDIO && !sh->codec->replaygain_data) {
+            struct replaygain_data *rg = decode_rgain(demuxer->log, sh->tags);
+            if (!rg)
+                rg = decode_rgain(demuxer->log, demuxer->metadata);
+            if (rg) {
+                MP_VERBOSE(demuxer, "Replaygain/%d: Track=%f/%f Album=%f/%f\n",
+                           sh->index,
+                           rg->track_gain, rg->track_peak,
+                           rg->album_gain, rg->album_peak);
+                sh->codec->replaygain_data = talloc_steal(in, rg);
+            }
+        }
     }
 }
 
@@ -998,10 +1037,25 @@ static void demux_copy(struct demuxer *dst, struct demuxer *src)
         dst->start_time = src->start_time;
         dst->priv = src->priv;
     }
+
     if (src->events & DEMUX_EVENT_METADATA) {
         talloc_free(dst->metadata);
         dst->metadata = mp_tags_dup(dst, src->metadata);
+
+        if (dst->num_update_stream_tags != src->num_update_stream_tags) {
+            talloc_free(dst->update_stream_tags);
+            dst->update_stream_tags =
+                talloc_zero_array(dst, struct mp_tags *, dst->num_update_stream_tags);
+            dst->num_update_stream_tags = src->num_update_stream_tags;
+        }
+        for (int n = 0; n < dst->num_update_stream_tags; n++) {
+            talloc_free(dst->update_stream_tags[n]);
+            dst->update_stream_tags[n] =
+                talloc_steal(dst->update_stream_tags, src->update_stream_tags[n]);
+            src->update_stream_tags[n] = NULL;
+        }
     }
+
     dst->events |= src->events;
     src->events = 0;
 }
@@ -1023,8 +1077,6 @@ void demux_changed(demuxer_t *demuxer, int events)
 
     if (demuxer->events & DEMUX_EVENT_INIT)
         demuxer_sort_chapters(demuxer);
-    if (demuxer->events & (DEMUX_EVENT_METADATA | DEMUX_EVENT_STREAMS))
-        demux_export_replaygain(demuxer);
 
     demux_copy(in->d_buffer, demuxer);
 
@@ -1047,8 +1099,28 @@ void demux_update(demuxer_t *demuxer)
     demux_copy(demuxer, in->d_buffer);
     demuxer->events |= in->events;
     in->events = 0;
-    if (in->stream_metadata && (demuxer->events & DEMUX_EVENT_METADATA))
-        mp_tags_merge(demuxer->metadata, in->stream_metadata);
+    if (demuxer->events & DEMUX_EVENT_METADATA) {
+        int num_streams = MPMIN(in->num_streams, demuxer->num_update_stream_tags);
+        for (int n = 0; n < num_streams; n++) {
+            struct mp_tags *tags = demuxer->update_stream_tags[n];
+            demuxer->update_stream_tags[n] = NULL;
+            if (tags) {
+                struct sh_stream *sh = in->streams[n];
+                talloc_free(sh->tags);
+                sh->tags = talloc_steal(sh, tags);
+            }
+        }
+
+        // Often useful audio-only files, which have metadata in the audio track
+        // metadata instead of the main metadata (especially OGG).
+        if (in->num_streams == 1)
+            mp_tags_merge(demuxer->metadata, in->streams[0]->tags);
+
+        if (in->stream_metadata)
+            mp_tags_merge(demuxer->metadata, in->stream_metadata);
+    }
+    if (demuxer->events & (DEMUX_EVENT_METADATA | DEMUX_EVENT_STREAMS))
+        demux_update_replaygain(demuxer);
     pthread_mutex_unlock(&in->lock);
 }
 
diff --git a/demux/demux.h b/demux/demux.h
index 8470047f9d..07803d2838 100644
--- a/demux/demux.h
+++ b/demux/demux.h
@@ -210,7 +210,10 @@ typedef struct demuxer {
     struct mp_log *log, *glog;
     struct demuxer_params *params;
 
-    struct demux_internal *in; // internal to demux.c
+    // internal to demux.c
+    struct demux_internal *in;
+    struct mp_tags **update_stream_tags;
+    int num_update_stream_tags;
 
     // Since the demuxer can run in its own thread, and the stream is not
     // thread-safe, only the demuxer is allowed to access the stream directly.
@@ -273,6 +276,8 @@ int demuxer_add_attachment(struct demuxer *demuxer, char *name,
                            char *type, void *data, size_t data_size);
 int demuxer_add_chapter(demuxer_t *demuxer, char *name,
                         double pts, uint64_t demuxer_id);
+void demux_set_stream_tags(struct demuxer *demuxer, struct sh_stream *sh,
+                           struct mp_tags *tags);
 
 double demuxer_get_time_length(struct demuxer *demuxer);
 
diff --git a/demux/demux_lavf.c b/demux/demux_lavf.c
index a38fb60435..e28ebd0c23 100644
--- a/demux/demux_lavf.c
+++ b/demux/demux_lavf.c
@@ -168,7 +168,6 @@ typedef struct lavf_priv {
     int num_streams;
     int cur_program;
     char *mime_type;
-    bool merge_track_metadata;
     double seek_delay;
 } lavf_priv_t;
 
@@ -502,7 +501,7 @@ static void select_tracks(struct demuxer *demuxer, int start)
     }
 }
 
-static void export_replaygain(demuxer_t *demuxer, struct mp_codec_params *c,
+static void export_replaygain(demuxer_t *demuxer, struct sh_stream *sh,
                               AVStream *st)
 {
     for (int i = 0; i < st->nb_side_data; i++) {
@@ -528,7 +527,10 @@ static void export_replaygain(demuxer_t *demuxer, struct mp_codec_params *c,
         rgain->album_peak = (av_rgain->album_peak != 0.0) ?
             av_rgain->album_peak / 100000.0f : 1.0;
 
-        c->replaygain_data = rgain;
+        // This must be run only before the stream was added, otherwise there
+        // will be race conditions with accesses from the user thread.
+        assert(!sh->ds);
+        sh->codec->replaygain_data = rgain;
     }
 }
 
@@ -575,7 +577,7 @@ static void handle_new_stream(demuxer_t *demuxer, int i)
             delay = lavc_delay / (double)codec->sample_rate;
         priv->seek_delay = MPMAX(priv->seek_delay, delay);
 
-        export_replaygain(demuxer, sh->codec, st);
+        export_replaygain(demuxer, sh, st);
 
         break;
     }
@@ -681,6 +683,7 @@ static void handle_new_stream(demuxer_t *demuxer, int i)
         if (!sh->title && sh->hls_bitrate > 0)
             sh->title = talloc_asprintf(sh, "bitrate %d", sh->hls_bitrate);
         sh->missing_timestamps = !!(priv->avif_flags & AVFMT_NOTIMESTAMPS);
+        mp_tags_copy_from_av_dictionary(sh->tags, st->metadata);
         demux_add_sh_stream(demuxer, sh);
     }
 
@@ -703,14 +706,14 @@ static void update_metadata(demuxer_t *demuxer, AVPacket *pkt)
         priv->avfc->event_flags = 0;
         demux_changed(demuxer, DEMUX_EVENT_METADATA);
     }
-    if (priv->merge_track_metadata) {
-        for (int n = 0; n < priv->num_streams; n++) {
-            AVStream *st = priv->streams[n] ? priv->avfc->streams[n] : NULL;
-            if (st && st->event_flags & AVSTREAM_EVENT_FLAG_METADATA_UPDATED) {
-                mp_tags_copy_from_av_dictionary(demuxer->metadata, st->metadata);
-                st->event_flags = 0;
-                demux_changed(demuxer, DEMUX_EVENT_METADATA);
-            }
+
+    for (int n = 0; n < priv->num_streams; n++) {
+        AVStream *st = priv->streams[n] ? priv->avfc->streams[n] : NULL;
+        if (st && st->event_flags & AVSTREAM_EVENT_FLAG_METADATA_UPDATED) {
+            st->event_flags = 0;
+            struct mp_tags *tags = talloc_zero(NULL, struct mp_tags);
+            mp_tags_copy_from_av_dictionary(tags, st->metadata);
+            demux_set_stream_tags(demuxer, priv->streams[n], tags);
         }
     }
 }
@@ -846,16 +849,6 @@ static int demux_open_lavf(demuxer_t *demuxer, enum demux_check check)
 
     add_new_streams(demuxer);
 
-    // Often useful with OGG audio-only files, which have metadata in the audio
-    // track metadata instead of the main metadata.
-    if (demux_get_num_stream(demuxer) == 1) {
-        priv->merge_track_metadata = true;
-        for (int n = 0; n < priv->num_streams; n++) {
-            if (priv->streams[n])
-                mp_tags_copy_from_av_dictionary(demuxer->metadata, avfc->streams[n]->metadata);
-        }
-    }
-
     mp_tags_copy_from_av_dictionary(demuxer->metadata, avfc->metadata);
     update_metadata(demuxer, NULL);
 
diff --git a/demux/demux_mkv.c b/demux/demux_mkv.c
index 6f45b5a5f2..da15c0fc0f 100644
--- a/demux/demux_mkv.c
+++ b/demux/demux_mkv.c
@@ -93,6 +93,7 @@ typedef struct mkv_content_encoding {
 
 typedef struct mkv_track {
     int tnum;
+    uint64_t uid;
     char *name;
     struct sh_stream *stream;
 
@@ -595,6 +596,7 @@ static void parse_trackentry(struct demuxer *demuxer,
     } else {
         MP_ERR(demuxer, "Missing track number!\n");
     }
+    track->uid = entry->track_uid;
 
     if (entry->name) {
         track->name = talloc_strdup(track, entry->name);
@@ -986,9 +988,6 @@ static void process_tags(demuxer_t *demuxer)
 
     for (int i = 0; i < tags->n_tag; i++) {
         struct ebml_tag tag = tags->tag[i];
-        if (tag.targets.target_track_uid  || tag.targets.target_attachment_uid)
-            continue;
-
         struct mp_tags *dst = NULL;
 
         if (tag.targets.target_chapter_uid) {
@@ -1009,6 +1008,19 @@ static void process_tags(demuxer_t *demuxer)
                     break;
                 }
             }
+        } else if (tag.targets.target_track_uid) {
+            for (int n = 0; n < mkv_d->num_tracks; n++) {
+                if (mkv_d->tracks[n]->uid ==
+                    tag.targets.target_track_uid)
+                {
+                    struct sh_stream *sh = mkv_d->tracks[n]->stream;
+                    if (sh)
+                        dst = sh->tags;
+                    break;
+                }
+            }
+        } else if (tag.targets.target_attachment_uid) {
+            /* ignore */
         } else {
             dst = demuxer->metadata;
         }
@@ -1924,9 +1936,9 @@ static int demux_mkv_open(demuxer_t *demuxer, enum demux_check check)
 
     MP_VERBOSE(demuxer, "All headers are parsed!\n");
 
-    process_tags(demuxer);
     display_create_tracks(demuxer);
     add_coverart(demuxer);
+    process_tags(demuxer);
 
     probe_first_timestamp(demuxer);
     if (opts->demux_mkv->probe_duration)
diff --git a/demux/stheader.h b/demux/stheader.h
index 77d0eb1da3..f9d564c230 100644
--- a/demux/stheader.h
+++ b/demux/stheader.h
@@ -46,6 +46,8 @@ struct sh_stream {
     bool forced_track;          // container forced track flag
     int hls_bitrate;
 
+    struct mp_tags *tags;
+
     bool missing_timestamps;
 
     // stream is a picture (such as album art)