vo_opengl: fix 10-bit video prescaling

The nnedi3 prescaler requires a normalized range to work properly,
but the original implementation did the range normalization after
the first step of the first pass. This could lead to severe quality
degradation when debanding is not enabled for NNEDI3.

Fix this issue by passing `tex_mul` into the shader code.

Fixes #2464

5 files changed, 21 insertions, 24 deletions

diff --git a/video/out/opengl/nnedi3.c b/video/out/opengl/nnedi3.c
index f757324608..04131078e3 100644
--- a/video/out/opengl/nnedi3.c
+++ b/video/out/opengl/nnedi3.c
@@ -81,7 +81,7 @@ const float* get_nnedi3_weights(const struct nnedi3_opts *conf, int *size)
 }
 
 void pass_nnedi3(GL *gl, struct gl_shader_cache *sc, int planes, int tex_num,
-                 int step, const struct nnedi3_opts *conf,
+                 int step, float tex_mul, const struct nnedi3_opts *conf,
                  struct gl_transform *transform)
 {
     assert(0 <= step && step < 2);
@@ -115,23 +115,23 @@ void pass_nnedi3(GL *gl, struct gl_shader_cache *sc, int planes, int tex_num,
         GLSLH(#pragma optionNV(fastprecision on))
     }
 
-    GLSLHF("float nnedi3(sampler2D tex, vec2 pos, vec2 tex_size, int plane) {\n");
+    GLSLHF("float nnedi3(sampler2D tex, vec2 pos, vec2 tex_size, int plane, float tex_mul) {\n");
 
     if (step == 0) {
         *transform = (struct gl_transform){{{1.0,0.0}, {0.0,2.0}}, {0.0,-0.5}};
 
         GLSLH(if (fract(pos.y * tex_size.y) < 0.5)
-                  return texture(tex, pos + vec2(0, 0.25) / tex_size)[plane];)
+                  return texture(tex, pos + vec2(0, 0.25) / tex_size)[plane] * tex_mul;)
         GLSLHF("#define GET(i, j) "
-               "(texture(tex, pos+vec2((i)-(%f),(j)-(%f)+0.25)/tex_size)[plane])\n",
+               "(texture(tex, pos+vec2((i)-(%f),(j)-(%f)+0.25)/tex_size)[plane]*tex_mul)\n",
                width / 2.0 - 1, (height - 1) / 2.0);
     } else {
         *transform = (struct gl_transform){{{2.0,0.0}, {0.0,1.0}}, {-0.5,0.0}};
 
         GLSLH(if (fract(pos.x * tex_size.x) < 0.5)
-                  return texture(tex, pos + vec2(0.25, 0) / tex_size)[plane];)
+                  return texture(tex, pos + vec2(0.25, 0) / tex_size)[plane] * tex_mul;)
         GLSLHF("#define GET(i, j) "
-               "(texture(tex, pos+vec2((j)-(%f)+0.25,(i)-(%f))/tex_size)[plane])\n",
+               "(texture(tex, pos+vec2((j)-(%f)+0.25,(i)-(%f))/tex_size)[plane]*tex_mul)\n",
                (height - 1) / 2.0, width / 2.0 - 1);
     }
 
@@ -213,7 +213,7 @@ void pass_nnedi3(GL *gl, struct gl_shader_cache *sc, int planes, int tex_num,
     GLSL(vec4 color = vec4(1.0);)
 
     for (int i = 0; i < planes; i++) {
-        GLSLF("color[%d] = nnedi3(texture%d, texcoord%d, texture_size%d, %d);\n",
-              i, tex_num, tex_num, tex_num, i);
+        GLSLF("color[%d] = nnedi3(texture%d, texcoord%d, texture_size%d, %d, %f);\n",
+              i, tex_num, tex_num, tex_num, i, tex_mul);
     }
 }
diff --git a/video/out/opengl/nnedi3.h b/video/out/opengl/nnedi3.h
index 8498b9de7a..be69f34c78 100644
--- a/video/out/opengl/nnedi3.h
+++ b/video/out/opengl/nnedi3.h
@@ -41,7 +41,7 @@ extern const struct m_sub_options nnedi3_conf;
 const float* get_nnedi3_weights(const struct nnedi3_opts *conf, int *size);
 
 void pass_nnedi3(GL *gl, struct gl_shader_cache *sc, int planes, int tex_num,
-                 int step, const struct nnedi3_opts *conf,
+                 int step, float tex_mul, const struct nnedi3_opts *conf,
                  struct gl_transform *transform);
 
 #endif
diff --git a/video/out/opengl/superxbr.c b/video/out/opengl/superxbr.c
index a88c058a38..c5b40fe36f 100644
--- a/video/out/opengl/superxbr.c
+++ b/video/out/opengl/superxbr.c
@@ -77,7 +77,7 @@ const struct m_sub_options superxbr_conf = {
 */
 
 void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
-                   int step, const struct superxbr_opts *conf,
+                   int step, float tex_mul, const struct superxbr_opts *conf,
                    struct gl_transform *transform)
 {
     assert(0 <= step && step < 2);
@@ -99,7 +99,7 @@ void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
         GLSLHF("#define weight1 (%f*1.29633/10.0)\n", conf->sharpness);
         GLSLHF("#define weight2 (%f*1.75068/10.0/2.0)\n", conf->sharpness);
 
-        GLSLH(#define Get(x, y) (texture(tex, pos + (vec2(x, y) - vec2(0.25, 0.25)) / tex_size)[plane]))
+        GLSLH(#define Get(x, y) (texture(tex, pos + (vec2(x, y) - vec2(0.25, 0.25)) / tex_size)[plane] * tex_mul))
     } else {
         *transform = (struct gl_transform){{{1.0,0.0}, {0.0,1.0}}, {0.0,0.0}};
 
@@ -113,7 +113,7 @@ void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
         GLSLHF("#define weight1 (%f*1.75068/10.0)\n", conf->sharpness);
         GLSLHF("#define weight2 (%f*1.29633/10.0/2.0)\n", conf->sharpness);
 
-        GLSLH(#define Get(x, y) (texture(tex, pos + (vec2((x) + (y) - 1, (y) - (x))) / tex_size)[plane]))
+        GLSLH(#define Get(x, y) (texture(tex, pos + (vec2((x) + (y) - 1, (y) - (x))) / tex_size)[plane] * tex_mul))
     }
     GLSLH(float df(float A, float B)
           {
@@ -140,7 +140,7 @@ void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
                       wp3*(df(i1,e2)+df(i3,e4)+df(e1,i2)+df(e3,i4)));
           })
 
-    GLSLHF("float superxbr(sampler2D tex, vec2 pos, vec2 tex_size, int plane) {\n");
+    GLSLHF("float superxbr(sampler2D tex, vec2 pos, vec2 tex_size, int plane, float tex_mul) {\n");
 
     if (step == 0) {
         GLSLH(vec2 dir = fract(pos * tex_size) - 0.5;)
@@ -152,11 +152,11 @@ void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
                   return 0.0;)
 
         GLSLH(if (dir.x < 0 || dir.y < 0 || dist.x < 1 || dist.y < 1)
-                  return texture(tex, pos - dir / tex_size)[plane];)
+                  return texture(tex, pos - dir / tex_size)[plane] * tex_mul;)
     } else {
         GLSLH(vec2 dir = fract(pos * tex_size / 2) - 0.5;)
         GLSLH(if (dir.x * dir.y > 0)
-                  return texture(tex, pos)[plane];)
+                  return texture(tex, pos)[plane] * tex_mul;)
     }
 
     GLSLH(float P0 = Get(-1,-1);
@@ -228,7 +228,7 @@ void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
     GLSL(vec4 color = vec4(1.0);)
 
     for (int i = 0; i < planes; i++) {
-        GLSLF("color[%d] = superxbr(texture%d, texcoord%d, texture_size%d, %d);\n",
-              i, tex_num, tex_num, tex_num, i);
+        GLSLF("color[%d] = superxbr(texture%d, texcoord%d, texture_size%d, %d, %f);\n",
+              i, tex_num, tex_num, tex_num, i, tex_mul);
     }
 }
diff --git a/video/out/opengl/superxbr.h b/video/out/opengl/superxbr.h
index 46f15fd269..08cec82839 100644
--- a/video/out/opengl/superxbr.h
+++ b/video/out/opengl/superxbr.h
@@ -30,7 +30,7 @@ extern const struct superxbr_opts superxbr_opts_def;
 extern const struct m_sub_options superxbr_conf;
 
 void pass_superxbr(struct gl_shader_cache *sc, int planes, int tex_num,
-                   int step, const struct superxbr_opts *conf,
+                   int step, float tex_mul, const struct superxbr_opts *conf,
                    struct gl_transform *transform);
 
 #endif
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index 14ad3dee79..5ed831bfe5 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -1210,20 +1210,17 @@ static void pass_prescale(struct gl_video *p, int src_tex_num, int dst_tex_num,
             switch(p->opts.prescale) {
             case 1:
                 pass_superxbr(p->sc, planes, tex_num, step,
-                              p->opts.superxbr_opts, &transform);
+                              tex_mul, p->opts.superxbr_opts, &transform);
                 break;
             case 2:
                 pass_nnedi3(p->gl, p->sc, planes, tex_num, step,
-                            p->opts.nnedi3_opts, &transform);
+                            tex_mul, p->opts.nnedi3_opts, &transform);
                 break;
             default:
                 abort();
             }
 
-            if (tex_mul != 1.0) {
-                GLSLF("color *= %f;\n", tex_mul);
-                tex_mul = 1.0;
-            }
+            tex_mul = 1.0;
 
             gl_transform_trans(transform, offset);