vo_opengl: use RPN expressions for user hook sizes

This replaces the previous TRANSFORM by WIDTH, HEIGHT and OFFSET where
WIDTH and HEIGHT are RPN expressions. This allows for more fine-grained
control over the output size, and also makes sure that overwriting
existing textures works more cleanly.

(Also add some more useful bstr functions)

3 files changed, 193 insertions, 9 deletions

diff --git a/video/out/opengl/user_shaders.c b/video/out/opengl/user_shaders.c
index 0c1b765400..0cf80af115 100644
--- a/video/out/opengl/user_shaders.c
+++ b/video/out/opengl/user_shaders.c
@@ -16,6 +16,54 @@
  */
 
 #include "user_shaders.h"
+#include "ctype.h"
+
+static bool parse_rpn_szexpr(struct bstr line, struct szexp out[MAX_SZEXP_SIZE])
+{
+    int pos = 0;
+
+    while (line.len > 0) {
+        struct bstr word = bstr_strip(bstr_splitchar(line, &line, ' '));
+        if (word.len == 0)
+            continue;
+
+        if (pos >= MAX_SZEXP_SIZE)
+            return false;
+
+        struct szexp *exp = &out[pos++];
+
+        if (bstr_eatend0(&word, ".w") || bstr_eatend0(&word, ".width")) {
+            exp->tag = SZEXP_VAR_W;
+            exp->val.varname = word;
+            continue;
+        }
+
+        if (bstr_eatend0(&word, ".h") || bstr_eatend0(&word, ".height")) {
+            exp->tag = SZEXP_VAR_H;
+            exp->val.varname = word;
+            continue;
+        }
+
+        switch (word.start[0]) {
+        case '+': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_ADD; continue;
+        case '-': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_SUB; continue;
+        case '*': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_MUL; continue;
+        case '/': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_DIV; continue;
+        }
+
+        if (isdigit(word.start[0])) {
+            exp->tag = SZEXP_CONST;
+            if (bstr_sscanf(word, "%f", &exp->val.cval) != 1)
+                return false;
+            continue;
+        }
+
+        // Some sort of illegal expression
+        return false;
+    }
+
+    return true;
+}
 
 // Returns false if no more shaders could be parsed
 bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
@@ -24,14 +72,19 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
     if (!body || !out || !body->start || body->len == 0)
         return false;
 
-    *out = (struct gl_user_shader){ .transform = identity_trans };
+    *out = (struct gl_user_shader){
+        .offset = identity_trans,
+        .width = {{ SZEXP_VAR_W, { .varname = bstr0("HOOKED") }}},
+        .height = {{ SZEXP_VAR_H, { .varname = bstr0("HOOKED") }}},
+    };
+
     int hook_idx = 0;
     int bind_idx = 0;
 
     // First parse all the headers
     while (true) {
         struct bstr rest;
-        struct bstr line = bstr_getline(*body, &rest);
+        struct bstr line = bstr_strip(bstr_getline(*body, &rest));
 
         // Check for the presence of the magic line beginning
         if (!bstr_eatstart0(&line, "//!"))
@@ -65,13 +118,30 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
             continue;
         }
 
-        if (bstr_eatstart0(&line, "TRANSFORM")) {
-            float sx, sy, ox, oy;
-            if (bstr_sscanf(line, "%f %f %f %f", &sx, &sy, &ox, &oy) != 4) {
-                mp_err(log, "Error while parsing TRANSFORM!\n");
+        if (bstr_eatstart0(&line, "OFFSET")) {
+            float ox, oy;
+            if (bstr_sscanf(line, "%f %f", &ox, &oy) != 2) {
+                mp_err(log, "Error while parsing OFFSET!\n");
+                return false;
+            }
+            out->offset.t[0] = ox;
+            out->offset.t[1] = oy;
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "WIDTH")) {
+            if (!parse_rpn_szexpr(line, out->width)) {
+                mp_err(log, "Error while parsing WIDTH!\n");
+                return false;
+            }
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "HEIGHT")) {
+            if (!parse_rpn_szexpr(line, out->height)) {
+                mp_err(log, "Error while parsing HEIGHT!\n");
                 return false;
             }
-            out->transform = (struct gl_transform){{{sx, 0}, {0, sy}}, {ox, oy}};
             continue;
         }
 
diff --git a/video/out/opengl/user_shaders.h b/video/out/opengl/user_shaders.h
index 051dcaaa58..0e32f53c57 100644
--- a/video/out/opengl/user_shaders.h
+++ b/video/out/opengl/user_shaders.h
@@ -24,13 +24,40 @@
 #define SHADER_API 1
 #define SHADER_MAX_HOOKS 16
 #define SHADER_MAX_BINDS 6
+#define MAX_SZEXP_SIZE 32
+
+enum szexp_op {
+    SZEXP_OP_ADD,
+    SZEXP_OP_SUB,
+    SZEXP_OP_MUL,
+    SZEXP_OP_DIV,
+};
+
+enum szexp_tag {
+    SZEXP_END = 0, // End of an RPN expression
+    SZEXP_CONST, // Push a constant value onto the stack
+    SZEXP_VAR_W, // Get the width/height of a named texture (variable)
+    SZEXP_VAR_H,
+    SZEXP_OP2, // Pop two elements and push the result of a dyadic operation
+} tag;
+
+struct szexp {
+    enum szexp_tag tag;
+    union {
+        float cval;
+        struct bstr varname;
+        enum szexp_op op;
+    } val;
+};
 
 struct gl_user_shader {
     struct bstr hook_tex[SHADER_MAX_HOOKS];
     struct bstr bind_tex[SHADER_MAX_BINDS];
     struct bstr save_tex;
     struct bstr pass_body;
-    struct gl_transform transform;
+    struct gl_transform offset;
+    struct szexp width[MAX_SZEXP_SIZE];
+    struct szexp height[MAX_SZEXP_SIZE];
     int components;
 };
 
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index f154fdf074..2a1d623f0b 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -1618,6 +1618,89 @@ static void user_hook_old(struct gl_video *p, struct img_tex tex,
     GLSLF("color = %s(HOOKED, HOOKED_pos, HOOKED_size);\n", fn_name);
 }
 
+// Returns 1.0 on failure to at least create a legal FBO
+static float eval_szexpr(struct gl_video *p, struct img_tex tex,
+                         struct szexp expr[MAX_SZEXP_SIZE])
+{
+    float stack[MAX_SZEXP_SIZE] = {0};
+    int idx = 0; // points to next element to push
+
+    for (int i = 0; i < MAX_SZEXP_SIZE; i++) {
+        switch (expr[i].tag) {
+        case SZEXP_END:
+            goto done;
+
+        case SZEXP_CONST:
+            // Since our SZEXPs are bound by MAX_SZEXP_SIZE, it should be
+            // impossible to overflow the stack
+            assert(idx < MAX_SZEXP_SIZE);
+            stack[idx++] = expr[i].val.cval;
+            continue;
+
+        case SZEXP_OP2:
+            if (idx < 2) {
+                MP_WARN(p, "Stack underflow in RPN expression!\n");
+                return 1.0;
+            }
+
+            // Pop the operands in reverse order
+            float op2 = stack[--idx], op1 = stack[--idx], res = 0.0;
+            switch (expr[i].val.op) {
+            case SZEXP_OP_ADD: res = op1 + op2; break;
+            case SZEXP_OP_SUB: res = op1 - op2; break;
+            case SZEXP_OP_MUL: res = op1 * op2; break;
+            case SZEXP_OP_DIV: res = op1 / op2; break;
+            default: abort();
+            }
+
+            if (isnan(res)) {
+                MP_WARN(p, "Illegal operation in RPN expression!\n");
+                return 1.0;
+            }
+
+            stack[idx++] = res;
+            continue;
+
+        case SZEXP_VAR_W:
+        case SZEXP_VAR_H: {
+            struct bstr name = expr[i].val.varname;
+            struct img_tex var_tex;
+
+            // HOOKED is a special case
+            if (bstr_equals0(name, "HOOKED")) {
+                var_tex = tex;
+                goto found_tex;
+            }
+
+            for (int o = 0; o < p->saved_tex_num; o++) {
+                if (bstr_equals0(name, p->saved_tex[o].name)) {
+                    var_tex = p->saved_tex[o].tex;
+                    goto found_tex;
+                }
+            }
+
+            char *errname = bstrto0(NULL, name);
+            MP_WARN(p, "Texture %s not found in RPN expression!\n", errname);
+            talloc_free(errname);
+            return 1.0;
+
+found_tex:
+            stack[idx++] = (expr[i].tag == SZEXP_VAR_W) ? var_tex.w : var_tex.h;
+            continue;
+            }
+        }
+    }
+
+done:
+    // Return the single stack element
+    if (idx != 1) {
+        MP_WARN(p, "Malformed stack after RPN expression!\n");
+        return 1.0;
+    }
+
+    return stack[0];
+}
+
 static void user_hook(struct gl_video *p, struct img_tex tex,
                       struct gl_transform *trans, void *priv)
 {
@@ -1628,7 +1711,11 @@ static void user_hook(struct gl_video *p, struct img_tex tex,
     GLSLF("// custom hook\n");
     GLSLF("color = hook();\n");
 
-    *trans = shader->transform;
+    float w = eval_szexpr(p, tex, shader->width);
+    float h = eval_szexpr(p, tex, shader->height);
+
+    *trans = (struct gl_transform){{{w / tex.w, 0}, {0, h / tex.h}}};
+    gl_transform_trans(shader->offset, trans);
 }
 
 static void user_hook_free(struct tex_hook *hook)