vo_opengl: make user hook passes optional

User hooks can now use an extra WHEN expression to specify when the
shader should be run. For example, this can be used to only run a chroma
scaling shader `WHEN CHROMA.w LUMA.w <`.

There's a slight semantics change to user shaders: When trying to bind a
texture that does not exist, a shader will now be silently skipped
(similar to when the condition is false) instead of generating an error.

This allows shader stages to depend on an optional earlier stage without
having to copy/paste the same condition everywhere.

(In other words: there's an implicit condition on all of the bound
textures existing)

4 files changed, 83 insertions, 20 deletions

diff --git a/DOCS/man/vo.rst b/DOCS/man/vo.rst
index f8c9c342f8..7eaf2d6636 100644
--- a/DOCS/man/vo.rst
+++ b/DOCS/man/vo.rst
@@ -746,10 +746,17 @@ Available video output drivers are:
         WIDTH <szexpr>, HEIGHT <szexpr>
             Specifies the size of the resulting texture for this pass.
             ``szexpr`` refers to an expression in RPN (reverse polish
-            notation), using the operators + - * /, floating point literals,
-            and references to existing texture sizes such as MAIN.width or
-            CHROMA.height. By default, these are set to HOOKED.w and HOOKED.h,
-            respectively.
+            notation), using the operators + - * / > < !, floating point
+            literals, and references to existing texture sizes such as
+            MAIN.width or CHROMA.height. By default, these are set to HOOKED.w
+            and HOOKED.h, respectively.
+
+        WHEN <szexpr>
+            Specifies a condition that needs to be true (non-zero) for the
+            shader stage to be evaluated. If it fails, it will silently be
+            omitted. (Note that a shader stage like this which has a dependency
+            on an optional hook point can still cause that hook point to be
+            saved, which has some minor overhead)
 
         OFFSET ox oy
             Indicates a pixel shift (offset) introduced by this pass. These
diff --git a/video/out/opengl/user_shaders.c b/video/out/opengl/user_shaders.c
index 4e9c1c89e8..8f915a56e3 100644
--- a/video/out/opengl/user_shaders.c
+++ b/video/out/opengl/user_shaders.c
@@ -50,6 +50,9 @@ static bool parse_rpn_szexpr(struct bstr line, struct szexp out[MAX_SZEXP_SIZE])
         case '-': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_SUB; continue;
         case '*': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_MUL; continue;
         case '/': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_DIV; continue;
+        case '!': exp->tag = SZEXP_OP1; exp->val.op = SZEXP_OP_NOT; continue;
+        case '>': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_GT;  continue;
+        case '<': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_LT;  continue;
         }
 
         if (isdigit(word.start[0])) {
@@ -77,6 +80,7 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
         .offset = identity_trans,
         .width = {{ SZEXP_VAR_W, { .varname = bstr0("HOOKED") }}},
         .height = {{ SZEXP_VAR_H, { .varname = bstr0("HOOKED") }}},
+        .cond = {{ SZEXP_CONST, { .cval = 1.0 }}},
     };
 
     int hook_idx = 0;
@@ -154,6 +158,14 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
             continue;
         }
 
+        if (bstr_eatstart0(&line, "WHEN")) {
+            if (!parse_rpn_szexpr(line, out->cond)) {
+                mp_err(log, "Error while parsing WHEN!\n");
+                return false;
+            }
+            continue;
+        }
+
         if (bstr_eatstart0(&line, "COMPONENTS")) {
             if (bstr_sscanf(line, "%d", &out->components) != 1) {
                 mp_err(log, "Error while parsing COMPONENTS!\n");
diff --git a/video/out/opengl/user_shaders.h b/video/out/opengl/user_shaders.h
index 3a651164e3..b8c287b6bd 100644
--- a/video/out/opengl/user_shaders.h
+++ b/video/out/opengl/user_shaders.h
@@ -31,6 +31,9 @@ enum szexp_op {
     SZEXP_OP_SUB,
     SZEXP_OP_MUL,
     SZEXP_OP_DIV,
+    SZEXP_OP_NOT,
+    SZEXP_OP_GT,
+    SZEXP_OP_LT,
 };
 
 enum szexp_tag {
@@ -39,6 +42,7 @@ enum szexp_tag {
     SZEXP_VAR_W, // Get the width/height of a named texture (variable)
     SZEXP_VAR_H,
     SZEXP_OP2, // Pop two elements and push the result of a dyadic operation
+    SZEXP_OP1, // Pop one element and push the result of a monadic operation
 };
 
 struct szexp {
@@ -58,6 +62,7 @@ struct gl_user_shader {
     struct gl_transform offset;
     struct szexp width[MAX_SZEXP_SIZE];
     struct szexp height[MAX_SZEXP_SIZE];
+    struct szexp cond[MAX_SZEXP_SIZE];
     int components;
 };
 
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index 52dbe19961..e23bfb4edf 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -154,6 +154,7 @@ struct tex_hook {
     void (*hook)(struct gl_video *p, struct img_tex tex, // generates GLSL
                  struct gl_transform *trans, void *priv);
     void (*free)(struct tex_hook *hook);
+    bool (*cond)(struct gl_video *p, struct img_tex tex, void *priv);
 };
 
 struct fbosurface {
@@ -1132,6 +1133,12 @@ static struct img_tex pass_hook(struct gl_video *p, const char *name,
         if (strcmp(hook->hook_tex, name) != 0)
             continue;
 
+        // Check the hook's condition
+        if (hook->cond && !hook->cond(p, tex, hook->priv)) {
+            MP_DBG(p, "Skipping hook on %s due to condition.\n", name);
+            continue;
+        }
+
         // Bind all necessary textures and add them to the prelude
         for (int t = 0; t < TEXUNIT_VIDEO_NUM; t++) {
             const char *bind_name = hook->bind_tex[t];
@@ -1149,12 +1156,11 @@ static struct img_tex pass_hook(struct gl_video *p, const char *name,
             }
 
             if (!saved_tex_find(p, bind_name, &bind_tex)) {
-                // Clean up texture bindings and just return as-is, stop
-                // all further processing of this hook
-                MP_ERR(p, "Failed running hook for %s: No saved texture named"
-                       " %s!\n", name, bind_name);
+                // Clean up texture bindings and move on to the next hook
+                MP_DBG(p, "Skipping hook on %s due to no texture named %s.\n",
+                       name, bind_name);
                 p->pass_tex_num -= t;
-                return tex;
+                goto next_hook;
             }
 
             hook_prelude(p, bind_name, pass_bind(p, bind_tex), bind_tex);
@@ -1197,6 +1203,8 @@ static struct img_tex pass_hook(struct gl_video *p, const char *name,
         }
 
         saved_tex_store(p, store_name, saved_tex);
+
+next_hook: ;
     }
 
     return tex;
@@ -1615,9 +1623,10 @@ static void user_hook_old(struct gl_video *p, struct img_tex tex,
     GLSLF("color = %s(HOOKED_raw, HOOKED_pos, HOOKED_size);\n", fn_name);
 }
 
-// Returns 1.0 on failure to at least create a legal FBO
-static float eval_szexpr(struct gl_video *p, struct img_tex tex,
-                         struct szexp expr[MAX_SZEXP_SIZE])
+// Returns whether successful. 'result' is left untouched on failure
+static bool eval_szexpr(struct gl_video *p, struct img_tex tex,
+                        struct szexp expr[MAX_SZEXP_SIZE],
+                        float *result)
 {
     float stack[MAX_SZEXP_SIZE] = {0};
     int idx = 0; // points to next element to push
@@ -1634,10 +1643,22 @@ static float eval_szexpr(struct gl_video *p, struct img_tex tex,
             stack[idx++] = expr[i].val.cval;
             continue;
 
+        case SZEXP_OP1:
+            if (idx < 1) {
+                MP_WARN(p, "Stack underflow in RPN expression!\n");
+                return false;
+            }
+
+            switch (expr[i].val.op) {
+            case SZEXP_OP_NOT: stack[idx-1] = !stack[idx-1]; break;
+            default: abort();
+            }
+            continue;
+
         case SZEXP_OP2:
             if (idx < 2) {
                 MP_WARN(p, "Stack underflow in RPN expression!\n");
-                return 1.0;
+                return false;
             }
 
             // Pop the operands in reverse order
@@ -1649,12 +1670,14 @@ static float eval_szexpr(struct gl_video *p, struct img_tex tex,
             case SZEXP_OP_SUB: res = op1 - op2; break;
             case SZEXP_OP_MUL: res = op1 * op2; break;
             case SZEXP_OP_DIV: res = op1 / op2; break;
+            case SZEXP_OP_GT:  res = op1 > op2; break;
+            case SZEXP_OP_LT:  res = op1 < op2; break;
             default: abort();
             }
 
-            if (isnan(res)) {
+            if (!isfinite(res)) {
                 MP_WARN(p, "Illegal operation in RPN expression!\n");
-                return 1.0;
+                return false;
             }
 
             stack[idx++] = res;
@@ -1679,7 +1702,7 @@ static float eval_szexpr(struct gl_video *p, struct img_tex tex,
             }
 
             MP_WARN(p, "Texture %.*s not found in RPN expression!\n", BSTR_P(name));
-            return 1.0;
+            return false;
 
 found_tex:
             stack[idx++] = (expr[i].tag == SZEXP_VAR_W) ? var_tex.w : var_tex.h;
@@ -1692,10 +1715,21 @@ done:
     // Return the single stack element
     if (idx != 1) {
         MP_WARN(p, "Malformed stack after RPN expression!\n");
-        return 1.0;
+        return false;
     }
 
-    return stack[0];
+    *result = stack[0];
+    return true;
+}
+
+static bool user_hook_cond(struct gl_video *p, struct img_tex tex, void *priv)
+{
+    struct gl_user_shader *shader = priv;
+    assert(shader);
+
+    float res = false;
+    eval_szexpr(p, tex, shader->cond, &res);
+    return res;
 }
 
 static void user_hook(struct gl_video *p, struct img_tex tex,
@@ -1708,8 +1742,12 @@ static void user_hook(struct gl_video *p, struct img_tex tex,
     GLSLF("// custom hook\n");
     GLSLF("color = hook();\n");
 
-    float w = eval_szexpr(p, tex, shader->width);
-    float h = eval_szexpr(p, tex, shader->height);
+    // Make sure we at least create a legal FBO on failure, since it's better
+    // to do this and display an error message than just crash OpenGL
+    float w = 1.0, h = 1.0;
+
+    eval_szexpr(p, tex, shader->width, &w);
+    eval_szexpr(p, tex, shader->height, &h);
 
     *trans = (struct gl_transform){{{w / tex.w, 0}, {0, h / tex.h}}};
     gl_transform_trans(shader->offset, trans);
@@ -1757,6 +1795,7 @@ static void pass_hook_user_shaders(struct gl_video *p, char **shaders)
                 .components = out.components,
                 .hook = user_hook,
                 .free = user_hook_free,
+                .cond = user_hook_cond,
             };
 
             for (int i = 0; i < SHADER_MAX_HOOKS; i++) {