player: redo hack for video keyframe seeks with external audio

If you play a video with an external audio track, and do backwards
keyframe seeks, then audio can be missing. This is because a backwards
seek can end up way before the seek target (this is just how this seek
mode works). The audio file will be seeked at the correct seek target
(since audio usually has a much higher seek granularity), which results
in silence being played until the video reaches the originally intended
seek target.

There was a hack in audio.c to deal with this. Replace it with a
different hack. The new hack probably works about as well as the old
hack, except it doesn't add weird crap to the audio resync path (which
is some of the worst code here, so this is some nice preparation for
rewriting it). As a more practical advantage, it doesn't discard the
audio demuxer packet cache. The old code did, which probably ruined
seeking in youtube DASH streams.

A non-hacky solution would be handling external files in the demuxer
layer. Then chaining the seeks would be pretty easy. But we're pretty
far from that, because it would either require intrusive changes to the
demuxer layer, or wouldn't be flexible enough to load/unload external
files at runtime. Maybe later.

4 files changed, 49 insertions, 31 deletions

diff --git a/player/audio.c b/player/audio.c
index 9b842a9003..ccddd790e7 100644
--- a/player/audio.c
+++ b/player/audio.c
@@ -283,7 +283,6 @@ void reset_audio_state(struct MPContext *mpctx)
     mpctx->delay = 0;
     mpctx->audio_drop_throttle = 0;
     mpctx->audio_stat_start = 0;
-    mpctx->audio_allow_second_chance_seek = false;
 }
 
 void uninit_audio_out(struct MPContext *mpctx)
@@ -800,27 +799,6 @@ static bool get_sync_samples(struct MPContext *mpctx, int *skip)
     }
     ptsdiff = MPCLAMP(ptsdiff, -3600, 3600);
 
-    // Heuristic: if audio is "too far" ahead, and one of them is a separate
-    // track, allow a refresh seek to the correct position to fix it.
-    if (ptsdiff > 0.2 && mpctx->audio_allow_second_chance_seek && sync_to_video) {
-        struct ao_chain *ao_c = mpctx->ao_chain;
-        if (ao_c && ao_c->track && mpctx->vo_chain && mpctx->vo_chain->track &&
-            ao_c->track->demuxer != mpctx->vo_chain->track->demuxer)
-        {
-            struct track *track = ao_c->track;
-            double pts = mpctx->video_pts;
-            if (pts != MP_NOPTS_VALUE)
-                pts += get_track_seek_offset(mpctx, track);
-            // (disable it first to make it take any effect)
-            demuxer_select_track(track->demuxer, track->stream, pts, false);
-            demuxer_select_track(track->demuxer, track->stream, pts, true);
-            reset_audio_state(mpctx);
-            MP_VERBOSE(mpctx, "retrying audio seek\n");
-            return false;
-        }
-    }
-    mpctx->audio_allow_second_chance_seek = false;
-
     int align = af_format_sample_alignment(ao_format);
     *skip = (int)(-ptsdiff * play_samplerate) / align * align;
     return true;
diff --git a/player/core.h b/player/core.h
index 041065e1f6..f2fed55366 100644
--- a/player/core.h
+++ b/player/core.h
@@ -412,9 +412,9 @@ typedef struct MPContext {
 
     struct seek_params seek;
 
-    // Allow audio to issue a second seek if audio is too far ahead (for non-hr
-    // seeks with external audio tracks).
-    bool audio_allow_second_chance_seek;
+    // Can be temporarily set to an external audio track after seeks. Then it
+    // must be seeked to the video position once video is done seeking.
+    struct track *seek_slave;
 
     /* Heuristic for relative chapter seeks: keep track which chapter
      * the user wanted to go to, even if we aren't exactly within the
diff --git a/player/loadfile.c b/player/loadfile.c
index edba9caa1b..4a886ff156 100644
--- a/player/loadfile.c
+++ b/player/loadfile.c
@@ -227,6 +227,8 @@ void reselect_demux_stream(struct MPContext *mpctx, struct track *track)
     if (pts != MP_NOPTS_VALUE)
         pts += get_track_seek_offset(mpctx, track);
     demuxer_select_track(track->demuxer, track->stream, pts, track->selected);
+    if (track == mpctx->seek_slave)
+        mpctx->seek_slave = NULL;
 }
 
 // Called from the demuxer thread if a new packet is available.
@@ -548,6 +550,9 @@ bool mp_remove_track(struct MPContext *mpctx, struct track *track)
 
     sub_destroy(track->d_sub);
 
+    if (mpctx->seek_slave == track)
+        mpctx->seek_slave = NULL;
+
     int index = 0;
     while (index < mpctx->num_tracks && mpctx->tracks[index] != track)
         index++;
diff --git a/player/playloop.c b/player/playloop.c
index 3db5818773..535bff883f 100644
--- a/player/playloop.c
+++ b/player/playloop.c
@@ -239,6 +239,7 @@ void reset_playback_state(struct MPContext *mpctx)
     mpctx->restart_complete = false;
     mpctx->paused_for_cache = false;
     mpctx->cache_buffer = 100;
+    mpctx->seek_slave = NULL;
 
 #if HAVE_ENCODING
     encode_lavc_discontinuity(mpctx->encode_lavc_ctx);
@@ -252,7 +253,7 @@ static void mp_seek(MPContext *mpctx, struct seek_params seek)
 {
     struct MPOpts *opts = mpctx->opts;
 
-    if (!mpctx->demuxer || seek.type == MPSEEK_NONE || seek.amount == MP_NOPTS_VALUE)
+    if (!mpctx->demuxer || !seek.type || seek.amount == MP_NOPTS_VALUE)
         return;
 
     bool hr_seek_very_exact = seek.exact == MPSEEK_VERY_EXACT;
@@ -326,13 +327,15 @@ static void mp_seek(MPContext *mpctx, struct seek_params seek)
 
     if (!demux_seek(mpctx->demuxer, demux_pts, demux_flags)) {
         if (!mpctx->demuxer->seekable) {
-            MP_ERR(mpctx, "Cannot seek in this file.\n");
+            MP_ERR(mpctx, "Cannot seek in this stream.\n");
             MP_ERR(mpctx, "You can force it with '--force-seekable=yes'.\n");
         }
         return;
     }
 
     // Seek external, extra files too:
+    bool has_video = false;
+    struct track *external_audio = NULL;
     for (int t = 0; t < mpctx->num_tracks; t++) {
         struct track *track = mpctx->tracks[t];
         if (track->selected && track->is_external && track->demuxer) {
@@ -342,7 +345,12 @@ static void mp_seek(MPContext *mpctx, struct seek_params seek)
             if (demux_flags & SEEK_FACTOR)
                 main_new_pos = seek_pts;
             demux_seek(track->demuxer, main_new_pos, 0);
+            if (track->type == STREAM_AUDIO && !external_audio)
+                external_audio = track;
         }
+        if (track->selected && !track->is_external && track->stream &&
+            track->type == STREAM_VIDEO && !track->stream->attached_picture)
+            has_video = true;
     }
 
     if (!(seek.flags & MPSEEK_FLAG_NOFLUSH))
@@ -352,6 +360,17 @@ static void mp_seek(MPContext *mpctx, struct seek_params seek)
     if (mpctx->recorder)
         mp_recorder_mark_discontinuity(mpctx->recorder);
 
+    // When doing keyframe seeks (hr_seek=false) backwards (no SEEK_FORWARD),
+    // then video can seek before the external audio track (because video seek
+    // granularity is coarser than audio). The result would be playing video with
+    // silence until the audio seek target is reached. Work around by blocking
+    // the demuxer (decoders can't read) and seeking to video position later.
+    if (has_video && external_audio && !hr_seek && !(demux_flags & SEEK_FORWARD)) {
+        MP_VERBOSE(mpctx, "delayed seek for aid=%d\n", external_audio->user_tid);
+        demux_block_reading(external_audio->demuxer, true);
+        mpctx->seek_slave = external_audio;
+    }
+
     /* Use the target time as "current position" for further relative
      * seeks etc until a new video frame has been decoded */
     mpctx->last_seek_pts = seek_pts;
@@ -376,9 +395,6 @@ static void mp_seek(MPContext *mpctx, struct seek_params seek)
     mp_notify(mpctx, MPV_EVENT_SEEK, NULL);
     mp_notify(mpctx, MPV_EVENT_TICK, NULL);
 
-    mpctx->audio_allow_second_chance_seek =
-        !hr_seek && !(demux_flags & SEEK_FORWARD);
-
     mpctx->ab_loop_clip = mpctx->last_seek_pts < opts->ab_loop[1];
 
     mpctx->current_seek = seek;
@@ -949,6 +965,24 @@ static void handle_playback_time(struct MPContext *mpctx)
     }
 }
 
+static void handle_delayed_audio_seek(struct MPContext *mpctx)
+{
+    if (mpctx->seek_slave) {
+        if (mpctx->video_pts != MP_NOPTS_VALUE) {
+            // We know the video position now, so seek external audio to the
+            // correct position.
+            double pts = mpctx->video_pts +
+                            get_track_seek_offset(mpctx, mpctx->seek_slave);
+            demux_seek(mpctx->seek_slave->demuxer, pts, 0);
+            mpctx->seek_slave = NULL;
+        } else if (mpctx->video_status >= STATUS_EOF) {
+            // We won't get a video position; don't stall the audio stream.
+            demux_block_reading(mpctx->seek_slave->demuxer, false);
+            mpctx->seek_slave = NULL;
+        }
+    }
+}
+
 // We always make sure audio and video buffers are filled before actually
 // starting playback. This code handles starting them at the same time.
 static void handle_playback_restart(struct MPContext *mpctx)
@@ -991,7 +1025,6 @@ static void handle_playback_restart(struct MPContext *mpctx)
         mpctx->hrseek_active = false;
         mpctx->restart_complete = true;
         mpctx->current_seek = (struct seek_params){0};
-        mpctx->audio_allow_second_chance_seek = false;
         handle_playback_time(mpctx);
         mp_notify(mpctx, MPV_EVENT_PLAYBACK_RESTART, NULL);
         update_core_idle_state(mpctx);
@@ -1099,6 +1132,8 @@ void run_playloop(struct MPContext *mpctx)
     fill_audio_out_buffers(mpctx);
     write_video(mpctx);
 
+    handle_delayed_audio_seek(mpctx);
+
     handle_playback_restart(mpctx);
 
     handle_playback_time(mpctx);