sub: change how subtitles are read

Slightly change how it is decided when a new packet should be read.
Switch to demux_read_packet_async(), and let the player "wait properly"
until required subtitle packets arrive, instead of blocking everything.
Move distinguishing the cases of passive and active reading into the
demuxer, where it belongs.

7 files changed, 87 insertions, 78 deletions

diff --git a/demux/demux.c b/demux/demux.c
index 8eb274647d..bf8f236f27 100644
--- a/demux/demux.c
+++ b/demux/demux.c
@@ -685,15 +685,33 @@ struct demux_packet *demux_read_packet(struct sh_stream *sh)
     return pkt;
 }
 
+// Sparse packets (Subtitles) interleaved with other non-sparse packets (video,
+// audio) should never be read actively, meaning the demuxer thread does not
+// try to exceed default readahead in order to find a new packet.
+static bool use_lazy_subtitle_reading(struct demux_stream *ds)
+{
+    if (ds->type != STREAM_SUB)
+        return false;
+    for (int n = 0; n < ds->in->num_streams; n++) {
+        struct demux_stream *s = ds->in->streams[n]->ds;
+        if (s->type != STREAM_SUB && s->selected && !s->eof)
+            return true;
+    }
+    return false;
+}
+
 // Poll the demuxer queue, and if there's a packet, return it. Otherwise, just
 // make the demuxer thread read packets for this stream, and if there's at
 // least one packet, call the wakeup callback.
-// Unlike demux_read_packet(), this always enables readahead (which means you
-// must not use it on interleaved subtitle streams).
+// Unlike demux_read_packet(), this always enables readahead (except for
+// interleaved subtitles).
 // Returns:
 //   < 0: EOF was reached, *out_pkt=NULL
 //  == 0: no new packet yet, but maybe later, *out_pkt=NULL
 //   > 0: new packet read, *out_pkt is set
+// Note: when reading interleaved subtitles, the demuxer won't try to forcibly
+// read ahead to get the next subtitle packet (as the next packet could be
+// minutes away). In this situation, this function will just return -1.
 int demux_read_packet_async(struct sh_stream *sh, struct demux_packet **out_pkt)
 {
     struct demux_stream *ds = sh ? sh->ds : NULL;
@@ -703,10 +721,14 @@ int demux_read_packet_async(struct sh_stream *sh, struct demux_packet **out_pkt)
         if (ds->in->threading) {
             pthread_mutex_lock(&ds->in->lock);
             *out_pkt = dequeue_packet(ds);
-            r = *out_pkt ? 1 : ((ds->eof || !ds->selected) ? -1 : 0);
-            ds->active = ds->selected; // enable readahead
-            ds->in->eof = false; // force retry
-            pthread_cond_signal(&ds->in->wakeup); // possibly read more
+            if (use_lazy_subtitle_reading(ds)) {
+                r = *out_pkt ? 1 : -1;
+            } else {
+                r = *out_pkt ? 1 : ((ds->eof || !ds->selected) ? -1 : 0);
+                ds->active = ds->selected; // enable readahead
+                ds->in->eof = false; // force retry
+                pthread_cond_signal(&ds->in->wakeup); // possibly read more
+            }
             pthread_mutex_unlock(&ds->in->lock);
         } else {
             *out_pkt = demux_read_packet(sh);
diff --git a/player/core.h b/player/core.h
index 6228732bd9..fe87658d81 100644
--- a/player/core.h
+++ b/player/core.h
@@ -515,7 +515,7 @@ void reinit_subs(struct MPContext *mpctx, int order);
 void uninit_sub(struct MPContext *mpctx, int order);
 void uninit_sub_all(struct MPContext *mpctx);
 void update_osd_msg(struct MPContext *mpctx);
-void update_subtitles(struct MPContext *mpctx);
+bool update_subtitles(struct MPContext *mpctx, double video_pts);
 
 // video.c
 void reset_video_state(struct MPContext *mpctx);
diff --git a/player/playloop.c b/player/playloop.c
index 892baf1a09..c81f720122 100644
--- a/player/playloop.c
+++ b/player/playloop.c
@@ -1036,7 +1036,8 @@ void run_playloop(struct MPContext *mpctx)
     handle_dummy_ticks(mpctx);
 
     update_osd_msg(mpctx);
-    update_subtitles(mpctx);
+    if (!mpctx->video_out)
+        update_subtitles(mpctx, mpctx->playback_pts);
 
     handle_segment_switch(mpctx, end_is_new_segment);
 
diff --git a/player/sub.c b/player/sub.c
index 42f602f754..dd1ba6d384 100644
--- a/player/sub.c
+++ b/player/sub.c
@@ -68,32 +68,14 @@ void uninit_sub_all(struct MPContext *mpctx)
     uninit_sub(mpctx, 1);
 }
 
-// When reading subtitles from a demuxer, and we read video or audio from the
-// demuxer, we should not explicitly read subtitle packets. (With external
-// subs, we have to.)
-static bool is_interleaved(struct MPContext *mpctx, struct track *track)
-{
-    if (track->is_external || !track->demuxer)
-        return false;
-
-    struct demuxer *demuxer = track->demuxer;
-    for (int t = 0; t < mpctx->num_tracks; t++) {
-        struct track *other = mpctx->tracks[t];
-        if (other != track && other->selected && other->demuxer == demuxer &&
-            (other->type == STREAM_VIDEO || other->type == STREAM_AUDIO))
-            return true;
-    }
-    return track->demuxer == mpctx->demuxer;
-}
-
-static void update_subtitle(struct MPContext *mpctx, int order)
+static bool update_subtitle(struct MPContext *mpctx, double video_pts, int order)
 {
     struct MPOpts *opts = mpctx->opts;
     struct track *track = mpctx->current_track[order][STREAM_SUB];
     struct dec_sub *dec_sub = mpctx->d_sub[order];
 
-    if (!track || !dec_sub)
-        return;
+    if (!track || !dec_sub || video_pts == MP_NOPTS_VALUE)
+        return true;
 
     if (mpctx->d_video) {
         struct mp_image_params params = mpctx->d_video->vfilter->override_params;
@@ -101,46 +83,26 @@ static void update_subtitle(struct MPContext *mpctx, int order)
             sub_control(dec_sub, SD_CTRL_SET_VIDEO_PARAMS, &params);
     }
 
-    double refpts_s = mpctx->playback_pts;
-    double curpts_s = refpts_s - opts->sub_delay;
-
-    if (!track->preloaded && track->stream) {
-        struct sh_stream *sh_stream = track->stream;
-        bool interleaved = is_interleaved(mpctx, track);
-
-        while (1) {
-            if (interleaved && !demux_has_packet(sh_stream))
-                break;
-            double subpts_s = demux_get_next_pts(sh_stream);
-            if (!demux_has_packet(sh_stream))
-                break;
-            if (subpts_s > curpts_s) {
-                MP_DBG(mpctx, "Sub early: c_pts=%5.3f s_pts=%5.3f\n",
-                       curpts_s, subpts_s);
-                // Often subs can be handled in advance
-                if (!sub_accepts_packet_in_advance(dec_sub))
-                    break;
-                // Try to avoid demuxing whole file at once
-                if (subpts_s > curpts_s + 1 && !interleaved)
-                    break;
-            }
-            struct demux_packet *pkt = demux_read_packet(sh_stream);
-            MP_DBG(mpctx, "Sub: c_pts=%5.3f s_pts=%5.3f duration=%5.3f len=%d\n",
-                   curpts_s, pkt->pts, pkt->duration, pkt->len);
-            sub_decode(dec_sub, pkt);
-            talloc_free(pkt);
-        }
+    video_pts -= opts->sub_delay;
+
+    if (!track->preloaded) {
+        if (!sub_read_packets(dec_sub, video_pts))
+            return false;
     }
 
     // Handle displaying subtitles on terminal; never done for secondary subs
     if (order == 0 && !mpctx->video_out)
-        term_osd_set_subs(mpctx, sub_get_text(dec_sub, curpts_s));
+        term_osd_set_subs(mpctx, sub_get_text(dec_sub, video_pts));
+
+    return true;
 }
 
-void update_subtitles(struct MPContext *mpctx)
+// Return true if the subtitles for the given PTS are ready; false if the player
+// should wait for new demuxer data, and then should retry.
+bool update_subtitles(struct MPContext *mpctx, double video_pts)
 {
-    update_subtitle(mpctx, 0);
-    update_subtitle(mpctx, 1);
+    return update_subtitle(mpctx, video_pts, 0) &
+           update_subtitle(mpctx, video_pts, 1);
 }
 
 static bool init_subdec(struct MPContext *mpctx, struct track *track)
diff --git a/player/video.c b/player/video.c
index 1178557e24..84981b1144 100644
--- a/player/video.c
+++ b/player/video.c
@@ -1206,6 +1206,11 @@ void write_video(struct MPContext *mpctx, double endpts)
     mpctx->time_frame -= get_relative_time(mpctx);
     update_avsync_before_frame(mpctx);
 
+    if (!update_subtitles(mpctx, mpctx->next_frames[0]->pts)) {
+        MP_WARN(mpctx, "subt wait\n");
+        return;
+    }
+
     double time_frame = MPMAX(mpctx->time_frame, -1);
     int64_t pts = mp_time_us() + (int64_t)(time_frame * 1e6);
 
@@ -1262,7 +1267,6 @@ void write_video(struct MPContext *mpctx, double endpts)
 
     mpctx->osd_force_update = true;
     update_osd_msg(mpctx);
-    update_subtitles(mpctx);
 
     vo_queue_frame(vo, frame);
 
diff --git a/sub/dec_sub.c b/sub/dec_sub.c
index b93dc890f3..2e4b27f628 100644
--- a/sub/dec_sub.c
+++ b/sub/dec_sub.c
@@ -49,6 +49,7 @@ struct dec_sub {
     struct MPOpts *opts;
 
     struct sh_stream *sh;
+    double last_pkt_pts;
 
     struct sd *sd;
 };
@@ -90,6 +91,7 @@ struct dec_sub *sub_create(struct mpv_global *global, struct demuxer *demuxer,
         sub->log = talloc_steal(sub, log),
         sub->opts = global->opts;
         sub->sh = sh;
+        sub->last_pkt_pts = MP_NOPTS_VALUE;
         mpthread_mutex_init_recursive(&sub->lock);
 
         sub->sd = talloc(NULL, struct sd);
@@ -116,20 +118,12 @@ struct dec_sub *sub_create(struct mpv_global *global, struct demuxer *demuxer,
     return NULL;
 }
 
-void sub_decode(struct dec_sub *sub, struct demux_packet *packet)
-{
-    pthread_mutex_lock(&sub->lock);
-    sub->sd->driver->decode(sub->sd, packet);
-    pthread_mutex_unlock(&sub->lock);
-}
-
 // Read all packets from the demuxer and decode/add them. Returns false if
 // there are circumstances which makes this not possible.
 bool sub_read_all_packets(struct dec_sub *sub)
 {
     pthread_mutex_lock(&sub->lock);
 
-    // Converters are assumed to always accept packets in advance
     if (!sub->sd->driver->accept_packets_in_advance) {
         pthread_mutex_unlock(&sub->lock);
         return false;
@@ -147,14 +141,40 @@ bool sub_read_all_packets(struct dec_sub *sub)
     return true;
 }
 
-bool sub_accepts_packet_in_advance(struct dec_sub *sub)
+// Read packets from the demuxer stream passed to sub_create(). Return true if
+// enough packets were read, false if the player should wait until the demuxer
+// signals new packets available (and then should retry).
+bool sub_read_packets(struct dec_sub *sub, double video_pts)
 {
-    bool res = true;
+    bool r = true;
     pthread_mutex_lock(&sub->lock);
-    if (sub->sd->driver->accepts_packet)
-        res &= sub->sd->driver->accepts_packet(sub->sd);
+    while (1) {
+        bool read_more = true;
+        if (sub->sd->driver->accepts_packet)
+            read_more = sub->sd->driver->accepts_packet(sub->sd);
+
+        if (!read_more)
+            break;
+
+        struct demux_packet *pkt;
+        int st = demux_read_packet_async(sub->sh, &pkt);
+        // Note: "wait" (st==0) happens with non-interleaved streams only, and
+        // then we should stop the playloop until a new enough packet has been
+        // seen (or the subtitle decoder's queue is full). This does not happen
+        // for interleaved subtitle streams, which never return "wait" when
+        // reading.
+        if (st <= 0) {
+            r = st < 0 || (sub->last_pkt_pts != MP_NOPTS_VALUE &&
+                           sub->last_pkt_pts >= video_pts);
+            break;
+        }
+
+        sub->sd->driver->decode(sub->sd, pkt);
+        sub->last_pkt_pts = pkt->pts;
+        talloc_free(pkt);
+    }
     pthread_mutex_unlock(&sub->lock);
-    return res;
+    return r;
 }
 
 // You must call sub_lock/sub_unlock if more than 1 thread access sub.
@@ -189,6 +209,7 @@ void sub_reset(struct dec_sub *sub)
     pthread_mutex_lock(&sub->lock);
     if (sub->sd->driver->reset)
         sub->sd->driver->reset(sub->sd);
+    sub->last_pkt_pts = MP_NOPTS_VALUE;
     pthread_mutex_unlock(&sub->lock);
 }
 
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index 56db5bc519..b3f30520e3 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -29,8 +29,7 @@ void sub_lock(struct dec_sub *sub);
 void sub_unlock(struct dec_sub *sub);
 
 bool sub_read_all_packets(struct dec_sub *sub);
-bool sub_accepts_packet_in_advance(struct dec_sub *sub);
-void sub_decode(struct dec_sub *sub, struct demux_packet *packet);
+bool sub_read_packets(struct dec_sub *sub, double video_pts);
 void sub_get_bitmaps(struct dec_sub *sub, struct mp_osd_res dim, double pts,
                      struct sub_bitmaps *res);
 char *sub_get_text(struct dec_sub *sub, double pts);