sub: move paletted image handling completely to sd_lavc.c

Until now, subtitle renderers could export SUBBITMAP_INDEXED, which is a
8 bit per pixel with palette format. sd_lavc.c was the only renderer
doing this, and the result was converted to RGBA in every use-case
(except maybe when the subtitles were hidden.)

Change it so that sd_lavc.c converts to RGBA on its own. This simplifies
everything a bit, and the palette handling can be removed from the
common code.

This is also preparation for making subtitle images refcounted. The
"caching" in img_convert.c is a PITA in this respect, and needs to be
redone. So getting rid of some img_convert.c code is a positive side-
effect. Also related to refcounted subtitles is packing them into a
single mp_image. Fewer objects to refcount is easier, and for the libass
format the same will be done. The plan is to remove manual packing from
the VOs which need single images entirely.

6 files changed, 89 insertions, 119 deletions

diff --git a/sub/img_convert.c b/sub/img_convert.c
index d86b8f4ad1..ebf9c209da 100644
--- a/sub/img_convert.c
+++ b/sub/img_convert.c
@@ -41,60 +41,6 @@ struct osd_conv_cache *osd_conv_cache_new(void)
     return talloc_zero(NULL, struct osd_conv_cache);
 }
 
-static void rgba_to_premultiplied_rgba(uint32_t *colors, size_t count)
-{
-    for (int n = 0; n < count; n++) {
-        uint32_t c = colors[n];
-        unsigned b = c & 0xFF;
-        unsigned g = (c >> 8) & 0xFF;
-        unsigned r = (c >> 16) & 0xFF;
-        unsigned a = (c >> 24) & 0xFF;
-        b = b * a / 255;
-        g = g * a / 255;
-        r = r * a / 255;
-        colors[n] = b | (g << 8) | (r << 16) | (a << 24);
-    }
-}
-
-bool osd_conv_idx_to_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs)
-{
-    struct sub_bitmaps src = *imgs;
-    if (src.format != SUBBITMAP_INDEXED)
-        return false;
-
-    imgs->format = SUBBITMAP_RGBA;
-    talloc_free(c->parts);
-    imgs->parts = c->parts = talloc_array(c, struct sub_bitmap, src.num_parts);
-
-    for (int n = 0; n < src.num_parts; n++) {
-        struct sub_bitmap *d = &imgs->parts[n];
-        struct sub_bitmap *s = &src.parts[n];
-        struct osd_bmp_indexed sb = *(struct osd_bmp_indexed *)s->bitmap;
-
-        rgba_to_premultiplied_rgba(sb.palette, 256);
-
-        *d = *s;
-        struct mp_image *image = mp_image_alloc(IMGFMT_BGRA, s->w, s->h);
-        talloc_steal(c->parts, image);
-        if (!image) {
-            // on OOM, skip the region by making it 0 sized
-            d->w = d->h = d->dw = d->dh = 0;
-            continue;
-        }
-
-        d->stride = image->stride[0];
-        d->bitmap = image->planes[0];
-
-        for (int y = 0; y < s->h; y++) {
-            uint8_t *inbmp = sb.bitmap + y * s->stride;
-            uint32_t *outbmp = (uint32_t*)((uint8_t*)d->bitmap + y * d->stride);
-            for (int x = 0; x < s->w; x++)
-                *outbmp++ = sb.palette[*inbmp++];
-        }
-    }
-    return true;
-}
-
 bool osd_conv_blur_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs,
                         double gblur)
 {
@@ -193,41 +139,6 @@ bool osd_scale_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs)
     return true;
 }
 
-static void rgba_to_gray(uint32_t *colors, size_t count)
-{
-    for (int n = 0; n < count; n++) {
-        uint32_t c = colors[n];
-        int b = c & 0xFF;
-        int g = (c >> 8) & 0xFF;
-        int r = (c >> 16) & 0xFF;
-        int a = (c >> 24) & 0xFF;
-        r = g = b = (r + g + b) / 3;
-        colors[n] = b | (g << 8) | (r << 16) | (a << 24);
-    }
-}
-
-bool osd_conv_idx_to_gray(struct osd_conv_cache *c, struct sub_bitmaps *imgs)
-{
-    struct sub_bitmaps src = *imgs;
-    if (src.format != SUBBITMAP_INDEXED)
-        return false;
-
-    talloc_free(c->parts);
-    imgs->parts = c->parts = talloc_array(c, struct sub_bitmap, src.num_parts);
-
-    for (int n = 0; n < src.num_parts; n++) {
-        struct sub_bitmap *d = &imgs->parts[n];
-        struct sub_bitmap *s = &src.parts[n];
-        struct osd_bmp_indexed sb = *(struct osd_bmp_indexed *)s->bitmap;
-
-        rgba_to_gray(sb.palette, 256);
-
-        *d = *s;
-        d->bitmap = talloc_memdup(c->parts, &sb, sizeof(sb));
-    }
-    return true;
-}
-
 static void draw_ass_rgba(unsigned char *src, int src_w, int src_h,
                           int src_stride, unsigned char *dst, size_t dst_stride,
                           int dst_x, int dst_y, uint32_t color)
diff --git a/sub/img_convert.h b/sub/img_convert.h
index 85ba5bb828..d5baadced2 100644
--- a/sub/img_convert.h
+++ b/sub/img_convert.h
@@ -11,13 +11,11 @@ struct osd_conv_cache *osd_conv_cache_new(void);
 
 // These functions convert from one OSD format to another. On success, they copy
 // the converted image data into c, and change imgs to point to the data.
-bool osd_conv_idx_to_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs);
 bool osd_conv_ass_to_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs);
 // Sub postprocessing
 bool osd_conv_blur_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs,
                         double gblur);
 bool osd_scale_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs);
-bool osd_conv_idx_to_gray(struct osd_conv_cache *c, struct sub_bitmaps *imgs);
 
 bool mp_sub_bitmaps_bb(struct sub_bitmaps *imgs, struct mp_rect *out_bb);
 
diff --git a/sub/osd.c b/sub/osd.c
index 4e269ce5aa..ea33961c9f 100644
--- a/sub/osd.c
+++ b/sub/osd.c
@@ -295,12 +295,6 @@ static void render_object(struct osd_state *osd, struct osd_object *obj,
 
     bool cached = false; // do we have a copy of all the image data?
 
-    if (out_imgs->format == SUBBITMAP_INDEXED && opts->sub_gray)
-        cached |= osd_conv_idx_to_gray(obj->cache[0], out_imgs);
-
-    if (formats[SUBBITMAP_RGBA] && out_imgs->format == SUBBITMAP_INDEXED)
-        cached |= osd_conv_idx_to_rgba(obj->cache[1], out_imgs);
-
     if (out_imgs->format == SUBBITMAP_RGBA && opts->sub_gauss != 0.0f)
         cached |= osd_conv_blur_rgba(obj->cache[2], out_imgs, opts->sub_gauss);
 
diff --git a/sub/osd.h b/sub/osd.h
index 6d5b0dfe3c..b50b72c587 100644
--- a/sub/osd.h
+++ b/sub/osd.h
@@ -29,19 +29,10 @@ enum sub_bitmap_format {
     SUBBITMAP_EMPTY = 0,// no bitmaps; always has num_parts==0
     SUBBITMAP_LIBASS,   // A8, with a per-surface blend color (libass.color)
     SUBBITMAP_RGBA,     // B8G8R8A8 (MSB=A, LSB=B), scaled, premultiplied alpha
-    SUBBITMAP_INDEXED,  // scaled, bitmap points to osd_bmp_indexed
 
     SUBBITMAP_COUNT
 };
 
-// For SUBBITMAP_INDEXED
-struct osd_bmp_indexed {
-    uint8_t *bitmap;
-    // Each entry is like a pixel in SUBBITMAP_RGBA format, but using straight
-    // alpha.
-    uint32_t palette[256];
-};
-
 struct sub_bitmap {
     void *bitmap;
     int stride;
diff --git a/sub/sd_lavc.c b/sub/sd_lavc.c
index f24a73c395..a5f093bb54 100644
--- a/sub/sd_lavc.c
+++ b/sub/sd_lavc.c
@@ -31,6 +31,7 @@
 #include "demux/stheader.h"
 #include "options/options.h"
 #include "video/mp_image.h"
+#include "video/out/bitmap_packer.h"
 #include "sd.h"
 #include "dec_sub.h"
 
@@ -39,9 +40,9 @@
 struct sub {
     bool valid;
     AVSubtitle avsub;
-    int count;
     struct sub_bitmap *inbitmaps;
-    struct osd_bmp_indexed *imgs;
+    int count;
+    struct mp_image *data;
     double pts;
     double endpts;
     int64_t id;
@@ -63,6 +64,7 @@ struct sd_lavc_priv {
     double current_pts;
     struct seekpoint *seekpoints;
     int num_seekpoints;
+    struct bitmap_packer *packer;
 };
 
 static void get_resolution(struct sd *sd, int wh[2])
@@ -144,6 +146,8 @@ static int init(struct sd *sd)
     sd->priv = priv;
     priv->displayed_id = -1;
     priv->current_pts = MP_NOPTS_VALUE;
+    priv->packer = talloc_zero(priv, struct bitmap_packer);
+    priv->packer->w_max = priv->packer->h_max = PACKER_MAX_WH;
     return 0;
 
  error:
@@ -176,6 +180,24 @@ static void alloc_sub(struct sd_lavc_priv *priv)
     priv->subs[0].id = priv->new_id++;
 }
 
+static void convert_pal(uint32_t *colors, size_t count, bool gray)
+{
+    for (int n = 0; n < count; n++) {
+        uint32_t c = colors[n];
+        int b = c & 0xFF;
+        int g = (c >> 8) & 0xFF;
+        int r = (c >> 16) & 0xFF;
+        int a = (c >> 24) & 0xFF;
+        if (gray)
+            r = g = b = (r + g + b) / 3;
+        // from straight to pre-multiplied alpha
+        b = b * a / 255;
+        g = g * a / 255;
+        r = r * a / 255;
+        colors[n] = b | (g << 8) | (r << 16) | (a << 24);
+    }
+}
+
 // Initialize sub from sub->avsub.
 static void read_sub_bitmaps(struct sd *sd, struct sub *sub)
 {
@@ -184,12 +206,16 @@ static void read_sub_bitmaps(struct sd *sd, struct sub *sub)
     AVSubtitle *avsub = &sub->avsub;
 
     MP_TARRAY_GROW(priv, sub->inbitmaps, avsub->num_rects);
-    MP_TARRAY_GROW(priv, sub->imgs, avsub->num_rects);
+
+    packer_set_size(priv->packer, avsub->num_rects);
+
+    // Assume consumers may use bilinear scaling on it (2x2 filter)
+    priv->packer->padding = 1;
 
     for (int i = 0; i < avsub->num_rects; i++) {
         struct AVSubtitleRect *r = avsub->rects[i];
         struct sub_bitmap *b = &sub->inbitmaps[sub->count];
-        struct osd_bmp_indexed *img = &sub->imgs[sub->count];
+
         if (r->type != SUBTITLE_BITMAP) {
             MP_ERR(sd, "unsupported subtitle type from libavcodec\n");
             continue;
@@ -198,6 +224,40 @@ static void read_sub_bitmaps(struct sd *sd, struct sub *sub)
             continue;
         if (r->w <= 0 || r->h <= 0)
             continue;
+
+        b->bitmap = r; // save for later (dumb hack to avoid more complexity)
+
+        priv->packer->in[sub->count] = (struct pos){r->w, r->h};
+        sub->count++;
+    }
+
+    priv->packer->count = sub->count;
+
+    if (packer_pack(priv->packer) < 0) {
+        MP_ERR(sd, "Unable to pack subtitle bitmaps.\n");
+        sub->count = 0;
+    }
+
+    if (!sub->count)
+        return;
+
+    struct pos bb[2];
+    packer_get_bb(priv->packer, bb);
+
+    if (!sub->data || sub->data->w < bb[1].x || sub->data->h < bb[1].y) {
+        talloc_free(sub->data);
+        sub->data = mp_image_alloc(IMGFMT_BGRA, priv->packer->w, priv->packer->h);
+        if (!sub->data) {
+            sub->count = 0;
+            return;
+        }
+        talloc_steal(priv, sub->data);
+    }
+
+    for (int i = 0; i < sub->count; i++) {
+        struct sub_bitmap *b = &sub->inbitmaps[i];
+        struct pos pos = priv->packer->result[i];
+        struct AVSubtitleRect *r = b->bitmap;
 #if HAVE_AV_SUBTITLE_NOPICT
         uint8_t **data = r->data;
         int *linesize = r->linesize;
@@ -205,17 +265,32 @@ static void read_sub_bitmaps(struct sd *sd, struct sub *sub)
         uint8_t **data = r->pict.data;
         int *linesize = r->pict.linesize;
 #endif
-        img->bitmap = data[0];
-        assert(r->nb_colors > 0);
-        assert(r->nb_colors * 4 <= sizeof(img->palette));
-        memcpy(img->palette, data[1], r->nb_colors * 4);
-        b->bitmap = img;
-        b->stride = linesize[0];
         b->w = r->w;
         b->h = r->h;
         b->x = r->x;
         b->y = r->y;
-        sub->count++;
+        b->stride = sub->data->stride[0];
+        b->bitmap = sub->data->planes[0] + pos.y * b->stride + pos.x * 4;
+
+        assert(r->nb_colors > 0);
+        assert(r->nb_colors <= 256);
+        uint32_t pal[256] = {0};
+        memcpy(pal, data[1], r->nb_colors * 4);
+        convert_pal(pal, 256, opts->sub_gray);
+
+        int padding = priv->packer->padding;
+        for (int y = 0; y < b->h + padding; y++) {
+            uint32_t *out = (uint32_t*)((char*)b->bitmap + y * b->stride);
+            int start = 0;
+            if (y < b->h) {
+                uint8_t *in = data[0] + y * linesize[0];
+                for (int x = 0; x < b->w; x++)
+                    *out++ = pal[*in++];
+                start = b->w;
+            }
+            for (int x = start; x < b->w + padding; x++)
+                *out++ = 0;
+        }
     }
 }
 
@@ -345,7 +420,7 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res d, double pts,
     if (priv->displayed_id != current->id)
         res->change_id++;
     priv->displayed_id = current->id;
-    res->format = SUBBITMAP_INDEXED;
+    res->format = SUBBITMAP_RGBA;
 
     double video_par = 0;
     if (priv->avctx->codec_id == AV_CODEC_ID_DVD_SUBTITLE &&
diff --git a/video/out/bitmap_packer.h b/video/out/bitmap_packer.h
index b86c3ec4f9..c9503fe42c 100644
--- a/video/out/bitmap_packer.h
+++ b/video/out/bitmap_packer.h
@@ -23,9 +23,10 @@ struct bitmap_packer {
     int asize;
 };
 
-struct ass_image;
 struct sub_bitmaps;
 
+#define PACKER_MAX_WH 65536
+
 // Clear all internal state. Leave the following fields: w_max, h_max
 void packer_reset(struct bitmap_packer *packer);