sub: move --sub-fix-timing handling to renderer

Instead of messing with the subtitle packet timestamps, do it on output.
We work on the libass event list. If there is an unwanted gap or
overlap, we render the timestamp at another position where there is no
gap or overlap.

This is somewhat more robust, and even works with demuxed subs (to some
degree - depends whether the subs are prefected soon enough).

It's active even for native ASS subs. I wonder if this is a problem with
extended type setting. If it is, the heuristic that tries to avoid
interrupting such cases has to be improved.

While it probably would be ideal to do this after the subtitle decoder,
certain aspects are at least currently handled better in this place.

2 files changed, 69 insertions, 36 deletions

diff --git a/sub/dec_sub.c b/sub/dec_sub.c
index a9267112f4..4372c0be91 100644
--- a/sub/dec_sub.c
+++ b/sub/dec_sub.c
@@ -346,37 +346,6 @@ static void multiply_timings(struct packet_list *subs, double factor)
     }
 }
 
-#define MS_TS(f_ts) ((long long)((f_ts) * 1000 + 0.5))
-
-// Remove overlaps and fill gaps between adjacent subtitle packets. This is done
-// by adjusting the duration of the earlier packet. If the gaps or overlap are
-// larger than the threshold, or if the durations are close to the threshold,
-// don't change the events.
-// The algorithm is maximally naive and doesn't work if there are multiple
-// overlapping lines. (It's not worth the trouble.)
-static void fix_overlaps_and_gaps(struct packet_list *subs)
-{
-    double threshold = SUB_GAP_THRESHOLD;
-    double keep = SUB_GAP_KEEP;
-    for (int i = 0; i < subs->num_packets - 1; i++) {
-        struct demux_packet *cur = subs->packets[i];
-        struct demux_packet *next = subs->packets[i + 1];
-        if (cur->pts != MP_NOPTS_VALUE && cur->duration > 0 &&
-            next->pts != MP_NOPTS_VALUE && next->duration > 0)
-        {
-            double end = cur->pts + cur->duration;
-            if (fabs(next->pts - end) <= threshold && cur->duration >= keep &&
-                next->duration >= keep)
-            {
-                // Conceptually: cur->duration = next->pts - cur->pts;
-                // But make sure the rounding and conversion to integers in
-                // sd_ass.c can't produce overlaps.
-                cur->duration = (MS_TS(next->pts) - MS_TS(cur->pts)) / 1000.0;
-            }
-        }
-    }
-}
-
 static void add_sub_list(struct dec_sub *sub, int at, struct packet_list *subs)
 {
     struct sd *sd = sub_get_last_sd(sub);
@@ -478,9 +447,6 @@ bool sub_read_all_packets(struct dec_sub *sub, struct sh_stream *sh)
     if (sub_speed != 1.0)
         multiply_timings(subs, sub_speed);
 
-    if (opts->sub_fix_timing)
-        fix_overlaps_and_gaps(subs);
-
     add_sub_list(sub, preprocess, subs);
 
     pthread_mutex_unlock(&sub->lock);
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 55c8d760e8..11943613f9 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -243,6 +243,72 @@ static void configure_ass(struct sd *sd, struct mp_osd_res *dim,
     ass_set_line_spacing(priv, set_line_spacing);
 }
 
+#define END(ev) ((ev)->Start + (ev)->Duration)
+
+static long long find_timestamp(struct sd *sd, double pts)
+{
+    struct sd_ass_priv *priv = sd->priv;
+    if (pts == MP_NOPTS_VALUE)
+        return 0;
+
+    long long ts = llrint(pts * 1000);
+
+    if (!sd->opts->sub_fix_timing)
+        return ts;
+
+    // Try to fix small gaps and overlaps.
+    ASS_Track *track = priv->ass_track;
+    int threshold = SUB_GAP_THRESHOLD * 1000;
+    int keep = SUB_GAP_KEEP * 1000;
+
+    // Find the "current" event.
+    ASS_Event *ev[2] = {0};
+    int n_ev = 0;
+    for (int n = 0; n < track->n_events; n++) {
+        ASS_Event *event = &track->events[n];
+        if (ts >= event->Start - threshold && ts <= END(event) + threshold) {
+            if (n_ev >= MP_ARRAY_SIZE(ev))
+                return ts; // multiple overlaps - give up (probably complex subs)
+            ev[n_ev++] = event;
+        }
+    }
+
+    if (n_ev != 2)
+        return ts;
+
+    // Simple/minor heuristic against destroying typesetting.
+    if (ev[0]->Style != ev[1]->Style)
+        return ts;
+
+    // Sort by start timestamps.
+    if (ev[0]->Start > ev[1]->Start)
+        MPSWAP(ASS_Event*, ev[0], ev[1]);
+
+    // We want to fix partial overlaps only.
+    if (END(ev[0]) >= END(ev[1]))
+        return ts;
+
+    if (ev[0]->Duration < keep || ev[1]->Duration < keep)
+        return ts;
+
+    // Gap between the events -> move ts to show the end of the first event.
+    if (ts >= END(ev[0]) && ts < ev[1]->Start && END(ev[0]) < ev[1]->Start &&
+        END(ev[0]) + threshold >= ev[1]->Start)
+        return END(ev[0]) - 1;
+
+    // Overlap -> move ts to the (exclusive) end of the first event.
+    // Relies on the fact that the ASS_Renderer has no overlap registered, even
+    // if there is one. This happens to work because we never render the
+    // overlapped state, and libass never resolves a collision.
+    if (ts >= ev[1]->Start && ts <= END(ev[0]) && END(ev[0]) > ev[1]->Start &&
+        END(ev[0]) <= ev[1]->Start + threshold)
+        return END(ev[0]);
+
+    return ts;
+}
+
+#undef END
+
 static void get_bitmaps(struct sd *sd, struct mp_osd_res dim, double pts,
                         struct sub_bitmaps *res)
 {
@@ -280,7 +346,8 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res dim, double pts,
     }
     if (no_ass)
         fill_plaintext(sd, pts);
-    mp_ass_render_frame(renderer, track, pts * 1000 + .5, &ctx->parts, res);
+    long long ts = find_timestamp(sd, pts);
+    mp_ass_render_frame(renderer, track, ts, &ctx->parts, res);
     talloc_steal(ctx, ctx->parts);
 
     if (!converted)
@@ -368,7 +435,7 @@ static char *get_text(struct sd *sd, double pts)
 
     if (pts == MP_NOPTS_VALUE)
         return NULL;
-    long long ipts = pts * 1000 + 0.5;
+    long long ipts = find_timestamp(sd, pts);
 
     struct buf b = {ctx->last_text, sizeof(ctx->last_text) - 1};