vd_lavc: sort -copy hwdec modes to end of list

Otherwise, if e.g. "nvdec" didn't work, but "nvdec-copy" did, it would
never try "vdpau", which is actually the next non-copy mode on the
autprobe list. It's really expected that it selects "vdpau". Fix this by
sorting the -copy modes to the end of the final hwdec list.

But we still don't want preferred -copy modes like "nvdec-copy" to be
sorted after fragile non-preferred modes like "cuda", and --hwdec=auto
should prefer "nvdec-copy" over it, so make sure the copying mode does
not get precedence over preferred vs. non-preferred mode.

Also simplify the existing auto_pos sorting condition, and fix the
fallback sort order (although that doesn't matter too much).

1 files changed, 12 insertions, 6 deletions

diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index d3fe630498..b9732c1fc0 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -224,11 +224,17 @@ static int hwdec_compare(const void *p1, const void *p2)
     if (h1 == h2)
         return 0;
 
-    if (h1->auto_pos > h2->auto_pos)
-        return 1;
-    if (h1->auto_pos < h2->auto_pos)
-        return -1;
-    return h1->rank < h2->rank ? 1 :-1;
+    // Strictly put non-preferred hwdecs to the end of the list.
+    if ((h1->auto_pos == INT_MAX) != (h2->auto_pos == INT_MAX))
+        return h1->auto_pos == INT_MAX ? 1 : -1;
+    // List non-copying entries first, so --hwdec=auto takes them.
+    if (h1->copying != h2->copying)
+        return h1->copying ? 1 : -1;
+    // Order by autoprobe preferrence order.
+    if (h1->auto_pos != h2->auto_pos)
+        return h1->auto_pos > h2->auto_pos ? 1 : -1;
+    // Fallback sort order to make sorting stable.
+    return h1->rank > h2->rank ? 1 :-1;
 }
 
 static bool test_decoder_suffix(const char *name, const char *suffix)
@@ -250,7 +256,7 @@ static void add_hwdec_item(struct hwdec_info **infos, int *num_infos,
              info.codec->name, info.method_name);
 
     info.rank = *num_infos;
-    info.auto_pos = MP_ARRAY_SIZE(hwdec_autoprobe_order);
+    info.auto_pos = INT_MAX;
     for (int x = 0; hwdec_autoprobe_order[x]; x++) {
         if (strcmp(hwdec_autoprobe_order[x], info.method_name) == 0)
             info.auto_pos = x;